responseListener) {
// Decrement the stream number by one when the call is closed.
@Override
public void onClose(Status status, Metadata trailers) {
- if (!decremented.getAndSet(true)) {
- delegateChannelRef.activeStreamsCountDecr(startNanos, status, false);
- }
+ finishCount(status, false);
// If the operation completed successfully, bind/unbind the affinity key.
if (keys != null && status.getCode() == Status.Code.OK) {
if (affinity.getCommand() == AffinityConfig.Command.UNBIND) {
@@ -219,7 +266,8 @@ public void onMessage(RespT message) {
* A simple wrapper of ClientCall.
*
* It defines the callback function to manage the number of active streams of a ChannelRef
- * everytime a call is started/closed.
+ * every time a call is created/closed. Stream capacity is reserved in the constructor, before
+ * {@link #start(Listener, Metadata)}, and remains reserved until close or cancel.
*/
public static class SimpleGcpClientCall extends ForwardingClientCall {
@@ -230,7 +278,8 @@ public static class SimpleGcpClientCall extends ForwardingClientCal
private final boolean unbindOnComplete;
private long startNanos = 0;
- private final AtomicBoolean decremented = new AtomicBoolean(false);
+ // 0 = not counted, 1 = counted, 2 = finished.
+ private final AtomicInteger countState = new AtomicInteger();
protected SimpleGcpClientCall(
GcpManagedChannel delegateChannel,
@@ -244,8 +293,16 @@ protected SimpleGcpClientCall(
// Set the actual channel ID in callOptions so downstream interceptors can access it.
CallOptions callOptionsWithChannelId =
callOptions.withOption(GcpManagedChannel.CHANNEL_ID_KEY, channelRef.getId());
- this.delegateCall =
- channelRef.getChannel().newCall(methodDescriptor, callOptionsWithChannelId);
+ startNanos = System.nanoTime();
+ channelRef.activeStreamsCountIncr();
+ countState.set(1);
+ try {
+ this.delegateCall =
+ channelRef.getChannel().newCall(methodDescriptor, callOptionsWithChannelId);
+ } catch (RuntimeException | Error failure) {
+ finishCount(Status.fromThrowable(failure), true);
+ throw failure;
+ }
}
@Override
@@ -255,16 +312,12 @@ protected ClientCall delegate() {
@Override
public void start(Listener responseListener, Metadata headers) {
- startNanos = System.nanoTime();
-
Listener listener =
new ForwardingClientCallListener.SimpleForwardingClientCallListener(
responseListener) {
@Override
public void onClose(Status status, Metadata trailers) {
- if (!decremented.getAndSet(true)) {
- channelRef.activeStreamsCountDecr(startNanos, status, false);
- }
+ finishCount(status, false);
// Unbind the affinity key when the caller explicitly requests it
// (e.g., on terminal RPCs like Commit or Rollback) to prevent
// unbounded growth of the affinity map.
@@ -281,20 +334,28 @@ public void onMessage(RespT message) {
}
};
- channelRef.activeStreamsCountIncr();
- delegateCall.start(listener, headers);
+ try {
+ delegateCall.start(listener, headers);
+ } catch (RuntimeException | Error failure) {
+ finishCount(Status.fromThrowable(failure), true);
+ throw failure;
+ }
}
@Override
public void cancel(String message, Throwable cause) {
- if (!decremented.getAndSet(true)) {
- channelRef.activeStreamsCountDecr(startNanos, Status.CANCELLED, true);
- }
+ finishCount(Status.CANCELLED, true);
// Always unbind on cancel — the transaction is being abandoned.
if (affinityKey != null) {
delegateChannel.unbind(Collections.singletonList(affinityKey));
}
delegateCall.cancel(message, cause);
}
+
+ private void finishCount(Status status, boolean fromClientSide) {
+ if (countState.compareAndSet(1, 2)) {
+ channelRef.activeStreamsCountDecr(startNanos, status, fromClientSide);
+ }
+ }
}
}
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java
index da3a3ce3cdf3..dd911f39f31e 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java
@@ -28,6 +28,9 @@
import com.google.cloud.grpc.proto.MethodConfig;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
import com.google.errorprone.annotations.concurrent.GuardedBy;
import com.google.protobuf.Descriptors.FieldDescriptor;
import com.google.protobuf.MessageOrBuilder;
@@ -72,8 +75,12 @@
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.function.IntUnaryOperator;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -82,6 +89,21 @@
/** A channel management factory that implements grpc.Channel APIs. */
public class GcpManagedChannel extends ManagedChannel {
+
+ private static final class PendingPrime {
+ private final ManagedChannel channel;
+ private int attempt = -1;
+ private boolean invoking;
+ private boolean finished;
+ @Nullable private ListenableFuture future;
+ @Nullable private ScheduledFuture timeoutTask;
+ @Nullable private ScheduledFuture retryTask;
+
+ private PendingPrime(ManagedChannel channel) {
+ this.channel = channel;
+ }
+ }
+
private static final Logger logger = Logger.getLogger(GcpManagedChannel.class.getName());
static final AtomicInteger channelPoolIndex = new AtomicInteger();
@@ -111,7 +133,12 @@ public class GcpManagedChannel extends ManagedChannel {
public static final CallOptions.Key CHANNEL_AFFINITY_REF_KEY =
CallOptions.Key.create("GcpChannelAffinityRef");
- /** Opaque sticky channel reference for callers that should not depend on {@link ChannelRef}. */
+ /**
+ * Opaque caller-owned channel reference for transaction-lifetime stickiness.
+ *
+ * The reference remains on a draining channel until its delegate shuts down. Call {@link
+ * #useDifferentChannelOnNextCall()} to move the next RPC to another active channel.
+ */
public static final class ChannelAffinityRef {
private static final int USE_DIFFERENT_CHANNEL_ON_NEXT_CALL_MASK = 1 << 31;
private static final int CHANNEL_ID_MASK = ~USE_DIFFERENT_CHANNEL_ON_NEXT_CALL_MASK;
@@ -127,6 +154,11 @@ public void useDifferentChannelOnNextCall() {
state.getAndUpdate(value -> value | USE_DIFFERENT_CHANNEL_ON_NEXT_CALL_MASK);
}
+ @VisibleForTesting
+ void setChannelIdForTest(int channelId) {
+ state.set(stateFromChannelId(channelId));
+ }
+
private static int channelIdFromState(int state) {
int encodedChannelId = state & CHANNEL_ID_MASK;
return encodedChannelId == 0 ? NO_CHANNEL_ID : encodedChannelId - 1;
@@ -156,6 +188,16 @@ private static int stateFromChannelId(int channelId) {
private int minRpcPerChannel = 0;
private int maxRpcPerChannel = 0;
private Duration scaleDownInterval = Duration.ZERO;
+ private Duration scaleUpCooldown = Duration.ofSeconds(10);
+ private int scaleDownConsecutiveLowLoadChecks = 3;
+ private int maxScaleUpPercent = 30;
+ private int maxScaleDownChannels = 2;
+ private Duration drainIdleGrace = Duration.ofMinutes(1);
+ private int errorPenaltyStep = 5;
+ private Duration errorPenaltyDuration = Duration.ofSeconds(5);
+ @Nullable private GcpChannelPrimer channelPrimer;
+ private Duration channelPrimeTimeout = Duration.ofSeconds(10);
+ private int channelPrimeMaxAttempts = 3;
private boolean isDynamicScalingEnabled = false;
private int maxConcurrentStreamsLowWatermark = DEFAULT_MAX_STREAM;
private GcpManagedChannelOptions.ChannelPickStrategy channelPickStrategy =
@@ -177,7 +219,23 @@ private static int stateFromChannelId(int channelId) {
private final Map channelIdToChannelRef = new ConcurrentHashMap<>();
// A set of channels that we removed from the pool and wait for their RPCs to be completed before
// we can shut them down.
- final Set removedChannelRefs = new HashSet<>();
+ final Set removedChannelRefs = ConcurrentHashMap.newKeySet();
+
+ @GuardedBy("this")
+ private final Map> drainTasks = new HashMap<>();
+
+ // One-slot scale-up signal. At most one worker mutates pool size at a time.
+ private final AtomicBoolean scaleUpSignalPending = new AtomicBoolean();
+ private final AtomicBoolean scaleUpWorkerRunning = new AtomicBoolean();
+ private final AtomicLong totalErrorPenaltyLoad = new AtomicLong();
+ private final AtomicInteger inFlightPrimeCount = new AtomicInteger();
+
+ @GuardedBy("this")
+ private final Set pendingPrimes = new HashSet<>();
+
+ private volatile long lastScaleUpNanos = Long.MIN_VALUE;
+ private int consecutiveLowLoadChecks;
+ private volatile boolean shuttingDown;
private final ExecutorService stateNotificationExecutor =
Executors.newCachedThreadPool(
@@ -242,7 +300,6 @@ private static int stateFromChannelId(int channelId) {
private AtomicInteger maxActiveStreams = new AtomicInteger();
private AtomicInteger minTotalActiveStreams = new AtomicInteger();
private AtomicInteger maxTotalActiveStreams = new AtomicInteger();
- private AtomicInteger maxTotalActiveStreamsForScaleDown = new AtomicInteger();
private long minOkCalls = 0;
private long maxOkCalls = 0;
private final AtomicLong totalOkCalls = new AtomicLong();
@@ -265,15 +322,89 @@ private static int stateFromChannelId(int channelId) {
private AtomicLong maxUnresponsiveDrops = new AtomicLong();
private AtomicLong scaleUpCount = new AtomicLong();
private AtomicLong scaleDownCount = new AtomicLong();
+ private final AtomicLong scaleUpPrimeFailures = new AtomicLong();
// Clock supplier for nanoTime, injectable for testing.
private Supplier nanoClock = System::nanoTime;
+ private IntUnaryOperator candidateIndexPicker =
+ bound -> ThreadLocalRandom.current().nextInt(bound);
+ @Nullable private volatile Consumer pickerValidationHookForTest;
+ @Nullable private volatile Runnable inactiveMappingRemovedHookForTest;
@VisibleForTesting
void setNanoClock(Supplier nanoClock) {
this.nanoClock = nanoClock;
}
+ @VisibleForTesting
+ void setPickerValidationHookForTest(Consumer hook) {
+ pickerValidationHookForTest = hook;
+ }
+
+ @VisibleForTesting
+ void setCandidateIndexPickerForTest(IntUnaryOperator candidateIndexPicker) {
+ this.candidateIndexPicker = candidateIndexPicker;
+ }
+
+ @VisibleForTesting
+ void setInactiveMappingRemovedHookForTest(Runnable hook) {
+ inactiveMappingRemovedHookForTest = hook;
+ }
+
+ private boolean validatePickedChannel(ChannelRef channelRef) {
+ Consumer hook = pickerValidationHookForTest;
+ if (hook != null) {
+ pickerValidationHookForTest = null;
+ hook.accept(channelRef);
+ }
+ return channelRef.isActive();
+ }
+
+ @VisibleForTesting
+ Map> fallbackMapForTest() {
+ return fallbackMap;
+ }
+
+ @VisibleForTesting
+ int channelIdMapSizeForTest() {
+ return channelIdToChannelRef.size();
+ }
+
+ @VisibleForTesting
+ int readyChannelCountForTest() {
+ return readyChannels.get();
+ }
+
+ @VisibleForTesting
+ int totalAffinityCountForTest() {
+ return totalAffinityCount.get();
+ }
+
+ @VisibleForTesting
+ long scaleUpPrimeFailuresForTest() {
+ return scaleUpPrimeFailures.get();
+ }
+
+ @VisibleForTesting
+ boolean scaleUpWorkerRunningForTest() {
+ return scaleUpWorkerRunning.get();
+ }
+
+ @VisibleForTesting
+ int inFlightPrimeCountForTest() {
+ return inFlightPrimeCount.get();
+ }
+
+ @VisibleForTesting
+ static long primeBackoffMillisForTest(int attempt) {
+ return primeBackoffMillis(attempt);
+ }
+
+ @VisibleForTesting
+ synchronized int drainTaskCountForTest() {
+ return drainTasks.size();
+ }
+
private static ScheduledThreadPoolExecutor createSharedBackgroundService() {
ScheduledThreadPoolExecutor executor =
new ScheduledThreadPoolExecutor(
@@ -349,8 +480,9 @@ public GcpManagedChannel(
}
}
- private void cleanupAffinityKeys() {
- final long cutoff = System.nanoTime() - affinityKeyLifetime.toNanos();
+ @VisibleForTesting
+ void cleanupAffinityKeys() {
+ final long cutoff = nanoClock.get() - affinityKeyLifetime.toNanos();
affinityKeyLastUsed.forEach(
(String key, Long time) -> {
if (time < cutoff) {
@@ -359,69 +491,74 @@ private void cleanupAffinityKeys() {
});
}
- private synchronized void checkScaleDown() {
- if (!isDynamicScalingEnabled) {
+ /**
+ * Evaluates instantaneous active load; consecutive checks provide the low-load history rather
+ * than retaining a maximum observed between checks.
+ */
+ @VisibleForTesting
+ synchronized void checkScaleDown() {
+ if (!isDynamicScalingEnabled || shuttingDown) {
return;
}
- // Use and reset maxTotalActiveStreamsForScaleDown.
- int maxTotalActiveStreamsCount =
- maxTotalActiveStreamsForScaleDown.getAndSet(totalActiveStreams.get());
- // Number of channels to support maximum seen (since last check) concurrent streams
- // with lowest desired utilization (minRpcPerChannel).
- int desiredSize =
- maxTotalActiveStreamsCount / minRpcPerChannel
- + ((maxTotalActiveStreamsCount % minRpcPerChannel == 0) ? 0 : 1);
-
- int scaleDownTo = Math.max(minSize, desiredSize);
- // Remove those extra channels that are the oldest.
- removeOldestChannels(channelRefs.size() - scaleDownTo);
-
- // Shutdown removed channels where all RPCs are completed.
- List completedChRefs =
- removedChannelRefs.stream()
- .filter(chRef -> (chRef.getActiveStreamsCount() == 0))
- .collect(Collectors.toList());
- removedChannelRefs.removeAll(completedChRefs);
- for (ChannelRef channelRef : completedChRefs) {
- channelRef.getChannel().shutdown();
- // Remove channel from broken channels map.
- fallbackMap.remove(channelRef.getId());
- channelIdToChannelRef.remove(channelRef.getId());
+ int channelCount = channelRefs.size();
+ if (channelCount <= minSize) {
+ consecutiveLowLoadChecks = 0;
+ return;
+ }
+ long activeLoad = activeLoad(channelRefs);
+ if (activeLoad > (long) minRpcPerChannel * channelCount) {
+ consecutiveLowLoadChecks = 0;
+ return;
}
+ if (++consecutiveLowLoadChecks < scaleDownConsecutiveLowLoadChecks) {
+ return;
+ }
+ consecutiveLowLoadChecks = 0;
+
+ int desiredSize = Math.max(minSize, ceilDiv(activeLoad, targetRpcPerChannel()));
+ int removeCount = Math.min(maxScaleDownChannels, Math.max(0, channelCount - desiredSize));
+ removeChannels(removeCount);
}
- private void removeOldestChannels(int num) {
+ private void removeChannels(int num) {
if (num <= 0) {
return;
}
- // Select longest connected channels (or disconnected channels).
+ // Drain least-loaded channels first; the oldest allocation breaks ties.
final List channelsToRemove =
channelRefs.stream()
- .sorted(Comparator.comparing(ChannelRef::getConnectedSinceNanos))
+ .sorted(
+ Comparator.comparingInt(ChannelRef::getActiveStreamsCount)
+ .thenComparingLong(ChannelRef::getCreatedNanos))
.limit(num)
.collect(Collectors.toList());
- // Remove from active channels.
+ for (ChannelRef channelRef : channelsToRemove) {
+ // Stop new picks before publishing the shorter active list.
+ channelRef.deactivateAndAccountReadiness();
+ }
channelRefs.removeAll(channelsToRemove);
+ // Normal unbind updates both affinity maps, per-channel count, and aggregate count.
+ List keysToUnbind =
+ affinityKeyToChannelRef.entrySet().stream()
+ .filter(entry -> channelsToRemove.contains(entry.getValue()))
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toList());
+ if (!keysToUnbind.isEmpty()) {
+ unbind(keysToUnbind);
+ }
+
for (ChannelRef channelRef : channelsToRemove) {
- channelRef.resetAffinityCount();
- channelRef.deactivate();
- if (channelRef.getState() == ConnectivityState.READY) {
- decReadyChannels(false);
+ channelRef.clearErrorPenalty();
+ removedChannelRefs.add(channelRef);
+ if (channelRef.getActiveStreamsCount() == 0) {
+ scheduleDrain(channelRef);
}
}
- // Remove affinity keys mapping for the channels.
- affinityKeyToChannelRef
- .keySet()
- .removeIf(key -> channelsToRemove.contains(affinityKeyToChannelRef.get(key)));
-
- // Keep them aside to wait for all RPCs to complete.
- removedChannelRefs.addAll(channelsToRemove);
-
// Track minimum number of channels for metrics.
minChannels.accumulateAndGet(getNumberOfChannels(), Math::min);
scaleDownCount.addAndGet(channelsToRemove.size());
@@ -430,6 +567,85 @@ private void removeOldestChannels(int num) {
executeStateChangeCallbacks();
}
+ /** Drain task bookkeeping is guarded by the pool monitor. */
+ @VisibleForTesting
+ synchronized void scheduleDrain(ChannelRef channelRef) {
+ if (channelRef.isActive() || channelRef.getActiveStreamsCount() != 0 || shuttingDown) {
+ return;
+ }
+ long elapsed = Math.max(0, nanoClock.get() - channelRef.getLastActivityNanos());
+ long delay = Math.max(0, drainIdleGrace.toNanos() - elapsed);
+ ScheduledFuture task;
+ try {
+ task = SHARED_BACKGROUND_SERVICE.schedule(() -> finishDrain(channelRef), delay, NANOSECONDS);
+ } catch (RejectedExecutionException e) {
+ logger.fine(log("Drain task rejected: %s", e.getMessage()));
+ return;
+ }
+ ScheduledFuture previous = drainTasks.put(channelRef, task);
+ if (previous != null) {
+ previous.cancel(false);
+ }
+ }
+
+ @VisibleForTesting
+ synchronized void finishDrain(ChannelRef channelRef) {
+ ScheduledFuture drainTask = drainTasks.remove(channelRef);
+ if (drainTask != null) {
+ drainTask.cancel(false);
+ }
+ if (channelRef.isActive()
+ || channelRef.getActiveStreamsCount() != 0
+ || !removedChannelRefs.contains(channelRef)
+ || shuttingDown) {
+ return;
+ }
+ long elapsed = Math.max(0, nanoClock.get() - channelRef.getLastActivityNanos());
+ if (elapsed < drainIdleGrace.toNanos()) {
+ scheduleDrain(channelRef);
+ return;
+ }
+ if (removedChannelRefs.remove(channelRef)) {
+ channelRef.clearErrorPenalty();
+ channelRef.getChannel().shutdown();
+ fallbackMap.remove(channelRef.getId());
+ channelIdToChannelRef.remove(channelRef.getId(), channelRef);
+ }
+ }
+
+ private static int ceilDiv(long numerator, int denominator) {
+ if (numerator == 0) {
+ return 0;
+ }
+ return (int) Math.min(Integer.MAX_VALUE, 1 + ((numerator - 1) / denominator));
+ }
+
+ private int targetRpcPerChannel() {
+ return Math.max(1, (minRpcPerChannel + maxRpcPerChannel) / 2);
+ }
+
+ private long activeLoad(List refs) {
+ long load = 0;
+ for (int i = 0; i < refs.size(); i++) {
+ ChannelRef channelRef = candidateAt(refs, i);
+ if (channelRef != null && channelRef.isActive()) {
+ load += channelRef.getActiveStreamsCount();
+ }
+ }
+ return load;
+ }
+
+ private long pickerLoad(List refs) {
+ long load = 0;
+ for (int i = 0; i < refs.size(); i++) {
+ ChannelRef channelRef = candidateAt(refs, i);
+ if (channelRef != null && channelRef.isActive()) {
+ load += channelRef.getPickerLoad();
+ }
+ }
+ return load;
+ }
+
private Supplier log(Supplier messageSupplier) {
return () -> String.format("%s: %s", metricPoolIndex, messageSupplier.get());
}
@@ -458,6 +674,16 @@ private void initOptions() {
minRpcPerChannel = poolOptions.getMinRpcPerChannel();
maxRpcPerChannel = poolOptions.getMaxRpcPerChannel();
scaleDownInterval = poolOptions.getScaleDownInterval();
+ scaleUpCooldown = poolOptions.getScaleUpCooldown();
+ scaleDownConsecutiveLowLoadChecks = poolOptions.getScaleDownConsecutiveLowLoadChecks();
+ maxScaleUpPercent = poolOptions.getMaxScaleUpPercent();
+ maxScaleDownChannels = poolOptions.getMaxScaleDownChannels();
+ drainIdleGrace = poolOptions.getDrainIdleGrace();
+ errorPenaltyStep = poolOptions.getErrorPenaltyStep();
+ errorPenaltyDuration = poolOptions.getErrorPenaltyDuration();
+ channelPrimer = poolOptions.getChannelPrimer();
+ channelPrimeTimeout = poolOptions.getChannelPrimeTimeout();
+ channelPrimeMaxAttempts = poolOptions.getChannelPrimeMaxAttempts();
isDynamicScalingEnabled =
minRpcPerChannel > 0 && maxRpcPerChannel > 0 && !scaleDownInterval.isZero();
channelPickStrategy = poolOptions.getChannelPickStrategy();
@@ -484,7 +710,13 @@ private synchronized void initScaleDownChecker(Duration scaleDownInterval) {
scaleDownTask =
SHARED_BACKGROUND_SERVICE.scheduleAtFixedRate(
- this::checkScaleDown,
+ () -> {
+ try {
+ checkScaleDown();
+ } catch (Throwable failure) {
+ logger.log(Level.WARNING, log("Scale-down check failed"), failure);
+ }
+ },
scaleDownInterval.toMillis(),
scaleDownInterval.toMillis(),
MILLISECONDS);
@@ -770,6 +1002,13 @@ private void initMetrics() {
this,
GcpManagedChannel::reportScaleUp,
GcpManagedChannel::reportScaleDown);
+
+ createDerivedLongCumulativeTimeSeries(
+ GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES,
+ "The number of scaled-up channels rejected because priming failed or timed out.",
+ GcpMetricsConstants.COUNT,
+ this,
+ GcpManagedChannel::reportScaleUpPrimeFailures);
}
private void setupOtelCommonAttributes(GcpMetricsOptions metricsOptions) {
@@ -1025,6 +1264,14 @@ private void initOtelMetrics(Meter meter) {
m.record(reportScaleUp(), withDirection(GcpMetricsConstants.DIRECTION_UP));
m.record(reportScaleDown(), withDirection(GcpMetricsConstants.DIRECTION_DOWN));
});
+
+ meter
+ .gaugeBuilder(metricPrefix + GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES)
+ .ofLongs()
+ .setDescription(
+ "The number of scaled-up channels rejected because priming failed or timed out.")
+ .setUnit(GcpMetricsConstants.COUNT)
+ .buildWithCallback(m -> m.record(reportScaleUpPrimeFailures(), otelCommonAttributes));
}
private void logGauge(String key, long value) {
@@ -1053,6 +1300,7 @@ void logMetrics() {
reportMaxAllowedChannels();
reportScaleUp();
reportScaleDown();
+ reportScaleUpPrimeFailures();
reportNumChannelDisconnect();
reportNumChannelConnect();
reportMinReadinessTime();
@@ -1417,6 +1665,12 @@ private long reportScaleDown() {
return value;
}
+ private long reportScaleUpPrimeFailures() {
+ long value = scaleUpPrimeFailures.get();
+ logCumulative(GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES, value);
+ return value;
+ }
+
private void incReadyChannels(boolean connected) {
if (connected) {
numChannelConnect.incrementAndGet();
@@ -1478,9 +1732,12 @@ public void notifyWhenStateChanged(ConnectivityState source, Runnable callback)
private class ChannelStateMonitor implements Runnable {
private final ChannelRef channelRef;
private final ManagedChannel channel;
- private ConnectivityState currentState;
+ private volatile ConnectivityState currentState;
private long connectingStartNanos;
- private long connectedSinceNanos;
+ private volatile long connectedSinceNanos;
+
+ @GuardedBy("channelRef")
+ private boolean readyAccounted;
private ChannelStateMonitor(ManagedChannel channel, ChannelRef channelRef) {
this.channelRef = channelRef;
@@ -1496,15 +1753,26 @@ public ConnectivityState getCurrentState() {
return currentState;
}
+ private void accountReadyIfNeeded() {
+ if (currentState == ConnectivityState.READY && !readyAccounted) {
+ readyAccounted = true;
+ incReadyChannels(false);
+ }
+ }
+
+ private void unaccountReadyIfNeeded() {
+ if (readyAccounted) {
+ readyAccounted = false;
+ decReadyChannels(false);
+ }
+ }
+
@Override
public void run() {
if (channel == null) {
return;
}
- // Is the channel in the pool?
- boolean isActive = channelRefs.contains(this.channelRef);
-
// Keep minSize channels always connected.
boolean requestConnection =
channelRefs.size() < minSize
@@ -1515,35 +1783,39 @@ public void run() {
.anyMatch(id -> (id == channelRef.getId()));
ConnectivityState newState = channel.getState(requestConnection);
- if (logger.isLoggable(Level.FINER)) {
- logger.finer(
- log(
- "Channel %d state change detected: %s -> %s",
- channelRef.getId(), currentState, newState));
- }
- if (newState == ConnectivityState.READY && currentState != ConnectivityState.READY) {
- connectedSinceNanos = System.nanoTime();
- if (isActive) {
- incReadyChannels(true);
- if (connectingStartNanos > 0) {
- saveReadinessTime(System.nanoTime() - connectingStartNanos);
+ boolean isActive;
+ synchronized (channelRef) {
+ isActive = channelRef.isActive() && channelRefs.contains(channelRef);
+ if (logger.isLoggable(Level.FINER)) {
+ logger.finer(
+ log(
+ "Channel %d state change detected: %s -> %s",
+ channelRef.getId(), currentState, newState));
+ }
+ if (newState == ConnectivityState.READY && currentState != ConnectivityState.READY) {
+ connectedSinceNanos = nanoClock.get();
+ if (isActive && !readyAccounted) {
+ readyAccounted = true;
+ incReadyChannels(true);
+ if (connectingStartNanos > 0) {
+ saveReadinessTime(nanoClock.get() - connectingStartNanos);
+ }
}
+ connectingStartNanos = 0;
}
- connectingStartNanos = 0;
- }
- if (isActive
- && newState != ConnectivityState.READY
- && currentState == ConnectivityState.READY) {
- decReadyChannels(true);
- }
- if (newState == ConnectivityState.CONNECTING
- && currentState != ConnectivityState.CONNECTING) {
- connectingStartNanos = System.nanoTime();
- }
- if (newState != ConnectivityState.READY) {
- connectedSinceNanos = 0;
+ if (newState != ConnectivityState.READY && readyAccounted) {
+ readyAccounted = false;
+ decReadyChannels(true);
+ }
+ if (newState == ConnectivityState.CONNECTING
+ && currentState != ConnectivityState.CONNECTING) {
+ connectingStartNanos = nanoClock.get();
+ }
+ if (newState != ConnectivityState.READY) {
+ connectedSinceNanos = 0;
+ }
+ currentState = newState;
}
- currentState = newState;
processChannelStateChange(channelRef.getId(), newState);
if (isActive) {
@@ -1573,6 +1845,11 @@ void processChannelStateChange(int channelId, ConnectivityState state) {
if (!fallbackEnabled) {
return;
}
+ ChannelRef channelRef = channelIdToChannelRef.get(channelId);
+ if (channelRef == null || !channelRef.isActive()) {
+ fallbackMap.remove(channelId);
+ return;
+ }
if (state == ConnectivityState.READY || state == ConnectivityState.IDLE) {
// Ready
fallbackMap.remove(channelId);
@@ -1599,11 +1876,25 @@ public int getStreamsLowWatermark() {
}
public int getMinActiveStreams() {
- return channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).min().orElse(0);
+ int minimum = Integer.MAX_VALUE;
+ for (int i = 0; i < channelRefs.size(); i++) {
+ ChannelRef channelRef = candidateAt(channelRefs, i);
+ if (channelRef != null && channelRef.isActive()) {
+ minimum = Math.min(minimum, channelRef.getActiveStreamsCount());
+ }
+ }
+ return minimum == Integer.MAX_VALUE ? 0 : minimum;
}
public int getMaxActiveStreams() {
- return channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).max().orElse(0);
+ int maximum = 0;
+ for (int i = 0; i < channelRefs.size(); i++) {
+ ChannelRef channelRef = candidateAt(channelRefs, i);
+ if (channelRef != null && channelRef.isActive()) {
+ maximum = Math.max(maximum, channelRef.getActiveStreamsCount());
+ }
+ }
+ return maximum;
}
/**
@@ -1637,10 +1928,13 @@ protected ChannelRef getChannelRefForBind() {
* @return {@link ChannelRef}
*/
protected synchronized ChannelRef getChannelRefRoundRobin() {
+ ChannelRef first = createFirstChannel();
+ if (first != null) {
+ return first;
+ }
if (!isDynamicScalingEnabled && channelRefs.size() < maxSize) {
return createNewChannel();
}
- maybeDynamicUpscale();
bindingIndex++;
if (bindingIndex >= channelRefs.size()) {
bindingIndex = 0;
@@ -1659,12 +1953,15 @@ protected synchronized ChannelRef getChannelRefRoundRobin() {
* Otherwise pick the one with the smallest number of streams.
*/
protected ChannelRef getChannelRef(@Nullable String key) {
- maybeDynamicUpscale();
if (key == null || key.isEmpty()) {
return pickLeastBusyChannel(/* forFallback= */ false);
}
ChannelRef mappedChannel = affinityKeyToChannelRef.get(key);
- affinityKeyLastUsed.put(key, System.nanoTime());
+ affinityKeyLastUsed.put(key, nanoClock.get());
+ if (mappedChannel != null && !mappedChannel.isActive()) {
+ unbindInactiveMapping(key, mappedChannel);
+ mappedChannel = null;
+ }
if (mappedChannel == null) {
ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ false);
bind(channelRef, Collections.singletonList(key));
@@ -1682,17 +1979,20 @@ protected ChannelRef getChannelRef(@Nullable String key) {
// Channel is not ready. Look up if the affinity key mapped to another channel.
Integer channelId = tempMap.get(key);
if (channelId != null && !fallbackMap.containsKey(channelId)) {
- // Fallback channel is ready.
- if (logger.isLoggable(Level.FINEST)) {
- logger.finest(log("Using fallback channel: %d -> %d", mappedChannel.getId(), channelId));
+ ChannelRef fallbackChannel = channelIdToChannelRef.get(channelId);
+ if (fallbackChannel != null && fallbackChannel.isActive()) {
+ if (logger.isLoggable(Level.FINEST)) {
+ logger.finest(log("Using fallback channel: %d -> %d", mappedChannel.getId(), channelId));
+ }
+ fallbacksSucceeded.incrementAndGet();
+ return fallbackChannel;
}
- fallbacksSucceeded.incrementAndGet();
- return channelRefs.get(channelId);
+ tempMap.remove(key, channelId);
}
// No temp mapping for this key or fallback channel is also broken.
ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ true);
if (!fallbackMap.containsKey(channelRef.getId())
- && channelRef.getActiveStreamsCount() < DEFAULT_MAX_STREAM) {
+ && channelRef.getActiveStreamsCount() < maxConcurrentStreamsLowWatermark) {
// Got a ready and not an overloaded channel.
if (channelRef.getId() != mappedChannel.getId()) {
if (logger.isLoggable(Level.FINEST)) {
@@ -1710,16 +2010,20 @@ protected ChannelRef getChannelRef(@Nullable String key) {
fallbacksFailed.incrementAndGet();
if (channelId != null) {
// Stick with previous mapping if fallback has failed.
- return channelRefs.get(channelId);
+ ChannelRef fallbackChannel = channelIdToChannelRef.get(channelId);
+ if (fallbackChannel != null && fallbackChannel.isActive()) {
+ return fallbackChannel;
+ }
}
return mappedChannel;
}
/**
- * Pick a {@link ChannelRef} using a caller-owned reference instead of grpc-gcp's affinity map.
+ * Picks a {@link ChannelRef} using a caller-owned reference instead of grpc-gcp's affinity map. A
+ * reference remains sticky while its delegate is open, including while the channel drains, and
+ * re-resolves after delegate shutdown or an explicit request to use a different channel.
*/
protected ChannelRef getChannelRefByAffinityRef(ChannelAffinityRef affinityRef) {
- maybeDynamicUpscale();
// Retry if another thread updates the caller-owned affinity ref while we are picking a channel.
while (true) {
int state = affinityRef.state.get();
@@ -1730,7 +2034,7 @@ protected ChannelRef getChannelRefByAffinityRef(ChannelAffinityRef affinityRef)
channelId == ChannelAffinityRef.NO_CHANNEL_ID
? null
: channelIdToChannelRef.get(channelId);
- if (!useDifferentChannel && channelRef != null && channelRef.isActive()) {
+ if (!useDifferentChannel && channelRef != null && !channelRef.getChannel().isShutdown()) {
return channelRef;
}
@@ -1747,19 +2051,17 @@ protected ChannelRef getChannelRefByAffinityRef(ChannelAffinityRef affinityRef)
private ChannelRef pickLeastBusyChannelDifferentFrom(@Nullable ChannelRef excludedChannelRef) {
ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ false);
- if (excludedChannelRef == null || channelRefs.size() <= 1) {
- return channelRef;
- }
- if (channelRef != excludedChannelRef && channelRef.isActive()) {
+ if (excludedChannelRef == null || channelRef != excludedChannelRef) {
return channelRef;
}
ChannelRef leastBusyChannelRef = null;
int leastBusyStreams = Integer.MAX_VALUE;
- for (ChannelRef candidate : channelRefs) {
- if (candidate == excludedChannelRef || !candidate.isActive()) {
+ for (int i = 0; i < channelRefs.size(); i++) {
+ ChannelRef candidate = candidateAt(channelRefs, i);
+ if (candidate == null || !candidate.isActive() || candidate == excludedChannelRef) {
continue;
}
- int streams = candidate.getActiveStreamsCount();
+ int streams = candidate.getPickerLoad();
if (leastBusyChannelRef == null || streams < leastBusyStreams) {
leastBusyChannelRef = candidate;
leastBusyStreams = streams;
@@ -1772,16 +2074,19 @@ private ChannelRef pickLeastBusyChannelDifferentFrom(@Nullable ChannelRef exclud
// If we have a ready channel not in the pool that we wait for completing its RPCs,
// then re-use that channel instead.
@VisibleForTesting
- ChannelRef createNewChannel() {
+ synchronized ChannelRef createNewChannel() {
Optional reusedChannelRef = pickChannelForReuse();
if (reusedChannelRef.isPresent()) {
ChannelRef chRef = reusedChannelRef.get();
- channelRefs.add(chRef);
+ ScheduledFuture drainTask = drainTasks.remove(chRef);
+ if (drainTask != null) {
+ drainTask.cancel(false);
+ }
removedChannelRefs.remove(chRef);
+ channelRefs.add(chRef);
channelIdToChannelRef.put(chRef.getId(), chRef);
- chRef.activate();
+ chRef.activateAndAccountReadiness();
logger.finer(log("Channel %d reused.", chRef.getId()));
- incReadyChannels(false);
maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
return chRef;
}
@@ -1789,22 +2094,20 @@ ChannelRef createNewChannel() {
ChannelRef channelRef = new ChannelRef(delegateChannelBuilder.build());
channelRefs.add(channelRef);
channelIdToChannelRef.put(channelRef.getId(), channelRef);
+ channelRef.activateAndAccountReadiness();
logger.finer(log("Channel %d created.", channelRef.getId()));
maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
return channelRef;
}
private Optional pickChannelForReuse() {
- // Pick the most recently connected, if any.
- Optional chRef =
- removedChannelRefs.stream().max(Comparator.comparing(ChannelRef::getConnectedSinceNanos));
-
- // Make sure it is ready, because connectedSinceNanos may be 0.
- if (chRef.isPresent() && chRef.get().getState() != ConnectivityState.READY) {
- return Optional.empty();
- }
-
- return chRef;
+ // Pick the most recently connected reusable READY channel, if any.
+ return removedChannelRefs.stream()
+ .filter(
+ channelRef ->
+ channelRef.getState() == ConnectivityState.READY
+ && !channelRef.getChannel().isShutdown())
+ .max(Comparator.comparing(ChannelRef::getConnectedSinceNanos));
}
// Returns first newly created channel or null if there are already some channels in the pool.
@@ -1814,7 +2117,7 @@ private ChannelRef createFirstChannel() {
return null;
}
synchronized (this) {
- if (channelRefs.isEmpty()) {
+ if (channelRefs.isEmpty() && !shuttingDown) {
return createNewChannel();
}
}
@@ -1836,27 +2139,364 @@ private ChannelRef tryCreateNewChannel() {
return null;
}
- private void maybeDynamicUpscale() {
- if (!isDynamicScalingEnabled || channelRefs.size() >= maxSize) {
+ private void maybeSignalScaleUp(ChannelRef selectedChannel) {
+ if (!selectedChannel.isActive()
+ || !isDynamicScalingEnabled
+ || shuttingDown
+ || channelRefs.size() >= maxSize) {
return;
}
+ int activeChannels = channelRefs.size();
+ if (activeChannels == 0) {
+ return;
+ }
+ long totalLoad = (long) totalActiveStreams.get() + totalErrorPenaltyLoad.get();
+ if (selectedChannel.getPickerLoad() <= maxRpcPerChannel
+ && ((double) totalLoad / activeChannels) <= maxRpcPerChannel) {
+ return;
+ }
+ signalScaleUp();
+ }
- if ((totalActiveStreams.get() / channelRefs.size()) >= maxRpcPerChannel) {
- dynamicUpscale();
+ private void signalScaleUp() {
+ scaleUpSignalPending.set(true);
+ if (!scaleUpWorkerRunning.compareAndSet(false, true)) {
+ return;
+ }
+ try {
+ SHARED_BACKGROUND_SERVICE.execute(this::runScaleUpWorker);
+ } catch (RejectedExecutionException e) {
+ scaleUpWorkerRunning.set(false);
+ logger.fine(log("Scale-up task rejected: %s", e.getMessage()));
+ }
+ }
+
+ private void runScaleUpWorker() {
+ try {
+ do {
+ scaleUpSignalPending.set(false);
+ try {
+ dynamicUpscale();
+ } catch (Throwable failure) {
+ logger.log(Level.WARNING, log("Scale-up failed"), failure);
+ }
+ } while (scaleUpSignalPending.get() && !shuttingDown);
+ } finally {
+ scaleUpWorkerRunning.set(false);
+ // Close the race where a signal arrives between the final test and clearing running.
+ if (scaleUpSignalPending.get() && !shuttingDown) {
+ signalScaleUp();
+ }
}
}
- private synchronized void dynamicUpscale() {
- if (!isDynamicScalingEnabled || channelRefs.size() >= maxSize) {
+ private void dynamicUpscale() {
+ final int channelsToBuild;
+ int reused = 0;
+ synchronized (this) {
+ if (!isDynamicScalingEnabled || shuttingDown || channelRefs.size() >= maxSize) {
+ return;
+ }
+ long now = nanoClock.get();
+ if (lastScaleUpNanos != Long.MIN_VALUE
+ && now - lastScaleUpNanos < scaleUpCooldown.toNanos()) {
+ return;
+ }
+ List activeChannels = new ArrayList<>(channelRefs);
+ int active = activeChannels.size();
+ if (active == 0) {
+ return;
+ }
+ int desired = ceilDiv(pickerLoad(activeChannels), targetRpcPerChannel());
+ int add = desired - active;
+ // Small pools may add two channels per event before percentage growth dominates.
+ int percentCap = Math.max(2, ceilDiv((long) active * maxScaleUpPercent, 100));
+ add = Math.min(add, percentCap);
+ add = Math.min(add, maxSize - active);
+ if (add <= 0) {
+ return;
+ }
+ while (reused < add && reuseDrainingChannel() != null) {
+ reused++;
+ }
+ channelsToBuild = add - reused;
+ // Claim cooldown before delegate construction or asynchronous priming begins.
+ lastScaleUpNanos = now;
+ }
+
+ scaleUpCount.addAndGet(reused);
+ List builtChannels = new ArrayList<>(channelsToBuild);
+ try {
+ for (int i = 0; i < channelsToBuild; i++) {
+ builtChannels.add(delegateChannelBuilder.build());
+ }
+ } catch (Throwable failure) {
+ builtChannels.forEach(ManagedChannel::shutdownNow);
+ throw failure;
+ }
+
+ if (channelPrimer != null) {
+ builtChannels.forEach(this::startPrime);
return;
}
- if ((totalActiveStreams.get() / channelRefs.size()) >= maxRpcPerChannel) {
- createNewChannel();
- scaleUpCount.incrementAndGet();
+ int added = 0;
+ List surplus = new ArrayList<>();
+ synchronized (this) {
+ for (ManagedChannel channel : builtChannels) {
+ if (shuttingDown || channelRefs.size() >= maxSize) {
+ surplus.add(channel);
+ } else {
+ addBuiltChannel(channel);
+ added++;
+ }
+ }
+ }
+ surplus.forEach(ManagedChannel::shutdownNow);
+ scaleUpCount.addAndGet(added);
+ }
+
+ /** Starts bounded asynchronous priming without holding the scale-up worker. */
+ private void startPrime(ManagedChannel channel) {
+ PendingPrime pendingPrime = new PendingPrime(channel);
+ synchronized (this) {
+ if (shuttingDown) {
+ channel.shutdownNow();
+ return;
+ }
+ pendingPrimes.add(pendingPrime);
+ inFlightPrimeCount.incrementAndGet();
+ }
+ try {
+ SHARED_BACKGROUND_SERVICE.execute(() -> startPrimeAttempt(pendingPrime, 0));
+ } catch (RejectedExecutionException failure) {
+ rejectPendingPrime(pendingPrime, failure);
+ }
+ }
+
+ private void startPrimeAttempt(PendingPrime pendingPrime, int attempt) {
+ synchronized (this) {
+ if (pendingPrime.finished) {
+ return;
+ }
+ if (shuttingDown) {
+ finishPendingPrime(pendingPrime);
+ pendingPrime.channel.shutdownNow();
+ return;
+ }
+ pendingPrime.attempt = attempt;
+ pendingPrime.invoking = true;
+ pendingPrime.retryTask = null;
+ }
+
+ ListenableFuture future;
+ try {
+ future = channelPrimer.prime(pendingPrime.channel);
+ if (future == null) {
+ throw new NullPointerException("Channel primer returned null");
+ }
+ } catch (Throwable failure) {
+ finishPrimeFailure(pendingPrime, attempt, null, failure, false);
+ return;
+ }
+
+ boolean cancelFuture = false;
+ Throwable schedulingFailure = null;
+ synchronized (this) {
+ if (pendingPrime.finished
+ || pendingPrime.attempt != attempt
+ || !pendingPrime.invoking
+ || shuttingDown) {
+ cancelFuture = true;
+ } else {
+ pendingPrime.invoking = false;
+ pendingPrime.future = future;
+ try {
+ pendingPrime.timeoutTask =
+ SHARED_BACKGROUND_SERVICE.schedule(
+ () ->
+ finishPrimeFailure(
+ pendingPrime,
+ attempt,
+ future,
+ new TimeoutException("Channel priming timed out"),
+ true),
+ channelPrimeTimeout.toNanos(),
+ NANOSECONDS);
+ } catch (RejectedExecutionException failure) {
+ schedulingFailure = failure;
+ }
+ }
+ }
+ if (cancelFuture) {
+ future.cancel(true);
+ pendingPrime.channel.shutdownNow();
+ return;
+ }
+ if (schedulingFailure != null) {
+ finishPrimeFailure(pendingPrime, attempt, future, schedulingFailure, true);
+ return;
+ }
+
+ Futures.addCallback(
+ future,
+ new FutureCallback() {
+ @Override
+ public void onSuccess(@Nullable Void unused) {
+ finishPrimeSuccess(pendingPrime, attempt, future);
+ }
+
+ @Override
+ public void onFailure(Throwable failure) {
+ finishPrimeFailure(pendingPrime, attempt, future, failure, false);
+ }
+ },
+ SHARED_BACKGROUND_SERVICE);
+ }
+
+ private void finishPrimeSuccess(
+ PendingPrime pendingPrime, int attempt, ListenableFuture future) {
+ boolean surplus;
+ synchronized (this) {
+ if (!isCurrentPrimeAttempt(pendingPrime, attempt, future)) {
+ return;
+ }
+ cancelPrimeTimeout(pendingPrime);
+ pendingPrime.future = null;
+ finishPendingPrime(pendingPrime);
+ surplus = shuttingDown || channelRefs.size() >= maxSize;
+ if (!surplus) {
+ addBuiltChannel(pendingPrime.channel);
+ scaleUpCount.incrementAndGet();
+ }
+ }
+ if (surplus) {
+ pendingPrime.channel.shutdownNow();
+ }
+ }
+
+ private void finishPrimeFailure(
+ PendingPrime pendingPrime,
+ int attempt,
+ @Nullable ListenableFuture future,
+ Throwable failure,
+ boolean cancelFuture) {
+ boolean finalFailure = false;
+ boolean publishFailure = false;
+ synchronized (this) {
+ if (pendingPrime.finished
+ || pendingPrime.attempt != attempt
+ || (future == null ? !pendingPrime.invoking : pendingPrime.future != future)) {
+ return;
+ }
+ pendingPrime.invoking = false;
+ pendingPrime.future = null;
+ cancelPrimeTimeout(pendingPrime);
+ if (shuttingDown || attempt + 1 >= channelPrimeMaxAttempts) {
+ finishPendingPrime(pendingPrime);
+ finalFailure = true;
+ publishFailure = !shuttingDown;
+ } else {
+ long delayMillis = primeBackoffMillis(attempt);
+ try {
+ pendingPrime.retryTask =
+ SHARED_BACKGROUND_SERVICE.schedule(
+ () -> startPrimeAttempt(pendingPrime, attempt + 1), delayMillis, MILLISECONDS);
+ } catch (RejectedExecutionException rejected) {
+ failure.addSuppressed(rejected);
+ finishPendingPrime(pendingPrime);
+ finalFailure = true;
+ publishFailure = true;
+ }
+ }
+ }
+ if (cancelFuture && future != null) {
+ future.cancel(true);
+ }
+ if (finalFailure) {
+ pendingPrime.channel.shutdownNow();
+ if (publishFailure) {
+ scaleUpPrimeFailures.incrementAndGet();
+ logger.log(Level.WARNING, log("Scaled-up channel priming failed"), failure);
+ }
}
}
+ private void rejectPendingPrime(PendingPrime pendingPrime, Throwable failure) {
+ boolean publishFailure;
+ synchronized (this) {
+ if (pendingPrime.finished) {
+ return;
+ }
+ finishPendingPrime(pendingPrime);
+ publishFailure = !shuttingDown;
+ }
+ pendingPrime.channel.shutdownNow();
+ if (publishFailure) {
+ scaleUpPrimeFailures.incrementAndGet();
+ logger.log(Level.WARNING, log("Scaled-up channel priming failed"), failure);
+ }
+ }
+
+ @GuardedBy("this")
+ private boolean isCurrentPrimeAttempt(
+ PendingPrime pendingPrime, int attempt, ListenableFuture future) {
+ return !pendingPrime.finished
+ && pendingPrime.attempt == attempt
+ && pendingPrime.future == future;
+ }
+
+ @GuardedBy("this")
+ private void cancelPrimeTimeout(PendingPrime pendingPrime) {
+ if (pendingPrime.timeoutTask != null) {
+ pendingPrime.timeoutTask.cancel(false);
+ pendingPrime.timeoutTask = null;
+ }
+ }
+
+ @GuardedBy("this")
+ private void finishPendingPrime(PendingPrime pendingPrime) {
+ if (!pendingPrime.finished) {
+ pendingPrime.finished = true;
+ pendingPrimes.remove(pendingPrime);
+ inFlightPrimeCount.decrementAndGet();
+ }
+ }
+
+ private static long primeBackoffMillis(int attempt) {
+ return Math.min(100L << Math.min(attempt, 12), 5000L);
+ }
+
+ @GuardedBy("this")
+ @Nullable
+ private ChannelRef reuseDrainingChannel() {
+ Optional reusable = pickChannelForReuse();
+ if (!reusable.isPresent()) {
+ return null;
+ }
+ ChannelRef channelRef = reusable.get();
+ ScheduledFuture drainTask = drainTasks.remove(channelRef);
+ if (drainTask != null) {
+ drainTask.cancel(false);
+ }
+ removedChannelRefs.remove(channelRef);
+ channelRefs.add(channelRef);
+ channelIdToChannelRef.put(channelRef.getId(), channelRef);
+ channelRef.activateAndAccountReadiness();
+ maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
+ return channelRef;
+ }
+
+ @GuardedBy("this")
+ private ChannelRef addBuiltChannel(ManagedChannel channel) {
+ ChannelRef channelRef = new ChannelRef(channel);
+ channelRefs.add(channelRef);
+ channelIdToChannelRef.put(channelRef.getId(), channelRef);
+ channelRef.activateAndAccountReadiness();
+ maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
+ return channelRef;
+ }
+
// This is pre-dynamic scaling functionality where we only scale up when the minimum number of
// streams on any channel reached maxConcurrentStreamsLowWatermark.
// If dynamic scaling is enabled we do not use this logic.
@@ -1876,16 +2516,19 @@ private boolean shouldScaleUp(int minStreams) {
* be provided if available.
*/
private ChannelRef pickLeastBusyChannel(boolean forFallback) {
- ChannelRef first = createFirstChannel();
- if (first != null) {
- return first;
- }
-
- if (!fallbackEnabled) {
- return pickLeastBusyNoFallback();
+ // Retries cover deactivation after selection.
+ for (int attempt = 0; attempt < 3; attempt++) {
+ ChannelRef first = createFirstChannel();
+ if (first != null) {
+ return first;
+ }
+ ChannelRef picked =
+ fallbackEnabled ? pickLeastBusyWithFallback(forFallback) : pickLeastBusyNoFallback();
+ if (validatePickedChannel(picked)) {
+ return picked;
+ }
}
-
- return pickLeastBusyWithFallback(forFallback);
+ return leastLoadedActiveChannel(channelRefs);
}
/**
@@ -1893,6 +2536,17 @@ private ChannelRef pickLeastBusyChannel(boolean forFallback) {
* GcpManagedChannelOptions.ChannelPickStrategy}.
*/
private ChannelRef pickLeastBusyNoFallback() {
+ // Retries cover deactivation after selection.
+ for (int attempt = 0; attempt < 3; attempt++) {
+ ChannelRef candidate = pickLeastBusyNoFallbackOnce();
+ if (candidate.isActive()) {
+ return candidate;
+ }
+ }
+ return leastLoadedActiveChannel(channelRefs);
+ }
+
+ private ChannelRef pickLeastBusyNoFallbackOnce() {
ChannelRef channelCandidate;
int minStreams;
@@ -1905,15 +2559,8 @@ private ChannelRef pickLeastBusyNoFallback() {
// Global min would delay scale-up; sampled min would be noisy.
minStreams = getMaxActiveStreams();
} else {
- channelCandidate = channelRefs.get(0);
- minStreams = channelCandidate.getActiveStreamsCount();
- for (ChannelRef channelRef : channelRefs) {
- int cnt = channelRef.getActiveStreamsCount();
- if (cnt < minStreams) {
- minStreams = cnt;
- channelCandidate = channelRef;
- }
- }
+ channelCandidate = leastLoadedActiveChannel(channelRefs);
+ minStreams = channelCandidate.getPickerLoad();
}
if (shouldScaleUp(minStreams)) {
@@ -1927,37 +2574,41 @@ private ChannelRef pickLeastBusyNoFallback() {
}
/**
- * Fallback-enabled channel selection. Always uses a full linear scan because the fallback logic
- * needs to filter channels by readiness state and max stream limits.
+ * Fallback-enabled channel selection. Uses allocation-free scans because fallback selection must
+ * filter channels by readiness state and max stream limits.
*/
private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
- // Full scan to collect eligible ("ready") channels not in fallbackMap and under max streams.
- List readyCandidates = new ArrayList<>();
- ChannelRef overallCandidate = channelRefs.get(0);
- int overallMinStreams = overallCandidate.getActiveStreamsCount();
+ ChannelRef overallCandidate = null;
+ int overallMinStreams = Integer.MAX_VALUE;
int readyMaxStreams = 0;
+ int readyCount = 0;
- for (ChannelRef channelRef : channelRefs) {
- int cnt = channelRef.getActiveStreamsCount();
- if (cnt < overallMinStreams) {
- overallMinStreams = cnt;
+ for (int i = 0; i < channelRefs.size(); i++) {
+ ChannelRef channelRef = candidateAt(channelRefs, i);
+ if (channelRef == null || !channelRef.isActive()) {
+ continue;
+ }
+ int count = channelRef.getPickerLoad();
+ if (overallCandidate == null || count < overallMinStreams) {
+ overallMinStreams = count;
overallCandidate = channelRef;
}
- if (!fallbackMap.containsKey(channelRef.getId()) && cnt < DEFAULT_MAX_STREAM) {
- readyCandidates.add(channelRef);
- if (cnt > readyMaxStreams) {
- readyMaxStreams = cnt;
- }
+ if (isReadyCandidate(channelRef, count)) {
+ readyCount++;
+ readyMaxStreams = Math.max(readyMaxStreams, count);
}
}
- // For scale-up, use maxStreams among ready channels (consistent with non-fallback path).
- int scaleUpStreams = readyCandidates.isEmpty() ? Integer.MAX_VALUE : readyMaxStreams;
+ if (overallCandidate == null) {
+ return leastLoadedActiveChannel(channelRefs);
+ }
+
+ int scaleUpStreams = readyCount == 0 ? Integer.MAX_VALUE : readyMaxStreams;
if (shouldScaleUp(scaleUpStreams)) {
ChannelRef newChannel = tryCreateNewChannel();
if (newChannel != null) {
scaleUpCount.incrementAndGet();
- if (!forFallback && readyCandidates.isEmpty()) {
+ if (!forFallback && readyCount == 0) {
if (logger.isLoggable(Level.FINEST)) {
logger.finest(log("Fallback to newly created channel %d", newChannel.getId()));
}
@@ -1967,9 +2618,8 @@ private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
}
}
- if (!readyCandidates.isEmpty()) {
- // Apply power-of-two among eligible channels to avoid thundering herd.
- ChannelRef readyCandidate = pickFromCandidates(readyCandidates);
+ if (readyCount > 0) {
+ ChannelRef readyCandidate = pickReadyCandidate(readyCount);
if (!forFallback && readyCandidate.getId() != overallCandidate.getId()) {
if (logger.isLoggable(Level.FINEST)) {
logger.finest(
@@ -1991,46 +2641,124 @@ private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
return overallCandidate;
}
+ private boolean isReadyCandidate(ChannelRef channelRef, int pickerLoad) {
+ return channelRef.isActive()
+ && !fallbackMap.containsKey(channelRef.getId())
+ && pickerLoad < maxConcurrentStreamsLowWatermark;
+ }
+
+ private ChannelRef pickReadyCandidate(int readyCount) {
+ for (int attempt = 0; attempt < 2 * readyCount; attempt++) {
+ ChannelRef first = readyCandidateAt(candidateIndexPicker.applyAsInt(readyCount));
+ ChannelRef second = readyCandidateAt(candidateIndexPicker.applyAsInt(readyCount));
+ if (first == null || second == null) {
+ continue;
+ }
+ ChannelRef picked = pickLessBusy(first, second);
+ if (picked.isActive()) {
+ return picked;
+ }
+ }
+
+ ChannelRef best = null;
+ int bestLoad = Integer.MAX_VALUE;
+ for (int i = 0; i < channelRefs.size(); i++) {
+ ChannelRef candidate = candidateAt(channelRefs, i);
+ if (candidate == null) {
+ continue;
+ }
+ int load = candidate.getPickerLoad();
+ if (isReadyCandidate(candidate, load) && (best == null || load < bestLoad)) {
+ best = candidate;
+ bestLoad = load;
+ }
+ }
+ return best == null ? leastLoadedActiveChannel(channelRefs) : best;
+ }
+
+ @Nullable
+ private ChannelRef readyCandidateAt(int readyIndex) {
+ int seen = 0;
+ for (int i = 0; i < channelRefs.size(); i++) {
+ ChannelRef candidate = candidateAt(channelRefs, i);
+ if (candidate != null && isReadyCandidate(candidate, candidate.getPickerLoad())) {
+ if (seen == readyIndex) {
+ return candidate;
+ }
+ seen++;
+ }
+ }
+ return null;
+ }
+
/**
* Picks a channel from the given candidate list using the configured strategy.
*
- * For {@code POWER_OF_TWO}: samples two distinct random candidates and picks the less busy
- * one. On tie, prefers the channel with more recent activity (warmer) to preserve connection
- * warmth under low traffic.
+ *
For {@code POWER_OF_TWO}: samples twice with replacement and picks the less busy sample. The
+ * first sample wins ties. Draining samples are retried up to twice the candidate count before a
+ * full active-channel scan.
*
*
For {@code LINEAR_SCAN}: deterministic scan picking the first least-busy channel.
*/
- private ChannelRef pickFromCandidates(List candidates) {
- if (candidates.size() == 1) {
- return candidates.get(0);
- }
+ @VisibleForTesting
+ ChannelRef pickFromCandidates(List candidates) {
+ int size = candidates.size();
if (channelPickStrategy == GcpManagedChannelOptions.ChannelPickStrategy.POWER_OF_TWO) {
- ThreadLocalRandom random = ThreadLocalRandom.current();
- int i = random.nextInt(candidates.size());
- int j = random.nextInt(candidates.size() - 1);
- if (j >= i) {
- j++;
- }
- ChannelRef a = candidates.get(i);
- ChannelRef b = candidates.get(j);
- int aStreams = a.getActiveStreamsCount();
- int bStreams = b.getActiveStreamsCount();
- if (aStreams < bStreams) return a;
- if (bStreams < aStreams) return b;
- // Tie: prefer the warmer channel (more recent activity).
- return a.lastResponseNanos >= b.lastResponseNanos ? a : b;
- }
- // LINEAR_SCAN: pick the least busy.
- ChannelRef best = candidates.get(0);
- int bestStreams = best.getActiveStreamsCount();
- for (int k = 1; k < candidates.size(); k++) {
- int cnt = candidates.get(k).getActiveStreamsCount();
- if (cnt < bestStreams) {
- bestStreams = cnt;
- best = candidates.get(k);
- }
- }
- return best;
+ for (int attempt = 0; attempt < 2 * size; attempt++) {
+ ChannelRef first = candidateAt(candidates, candidateIndexPicker.applyAsInt(size));
+ ChannelRef second = candidateAt(candidates, candidateIndexPicker.applyAsInt(size));
+ if (first == null || second == null || !first.isActive() || !second.isActive()) {
+ continue;
+ }
+ ChannelRef picked = pickLessBusy(first, second);
+ if (picked.isActive()) {
+ return picked;
+ }
+ }
+ }
+ return leastLoadedActiveChannel(candidates);
+ }
+
+ private ChannelRef leastLoadedActiveChannel(List candidates) {
+ ChannelRef best = null;
+ int bestLoad = Integer.MAX_VALUE;
+ for (int i = 0; i < candidates.size(); i++) {
+ ChannelRef candidate = candidateAt(candidates, i);
+ if (candidate == null || !candidate.isActive()) {
+ continue;
+ }
+ int load = candidate.getPickerLoad();
+ if (best == null || load < bestLoad) {
+ best = candidate;
+ bestLoad = load;
+ }
+ }
+ if (best != null) {
+ return best;
+ }
+ ChannelRef first = createFirstChannel();
+ if (first != null) {
+ return first;
+ }
+ if (shuttingDown) {
+ throw Status.UNAVAILABLE.withDescription("Channel pool is shut down").asRuntimeException();
+ }
+ throw Status.UNAVAILABLE.withDescription("No available channels").asRuntimeException();
+ }
+
+ @Nullable
+ private static ChannelRef candidateAt(List candidates, int index) {
+ try {
+ return candidates.get(index);
+ } catch (IndexOutOfBoundsException ignored) {
+ // CopyOnWriteArrayList may shrink between size() and get() during scale-down.
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ ChannelRef pickLessBusy(ChannelRef first, ChannelRef second) {
+ return first.getPickerLoad() <= second.getPickerLoad() ? first : second;
}
@Override
@@ -2050,6 +2778,9 @@ public String authority() {
* If method-affinity is specified, we will use the GcpClientCall to fetch the affinitykey and
* bind/unbind the channel, otherwise we just need the SimpleGcpClientCall to keep track of the
* number of streams in each channel.
+ *
+ *
A returned simple call reserves one unit of pool load immediately. If never started, callers
+ * must invoke {@link ClientCall#cancel(String, Throwable)} to release that reservation.
*/
@Override
public ClientCall newCall(
@@ -2096,7 +2827,9 @@ private String keyFromOptsCtx(CallOptions callOptions) {
return key;
}
- private synchronized void cancelBackgroundTasks() {
+ private synchronized List cancelBackgroundTasks() {
+ shuttingDown = true;
+ scaleUpSignalPending.set(false);
if (cleanupTask != null) {
cleanupTask.cancel(false);
cleanupTask = null;
@@ -2109,22 +2842,49 @@ private synchronized void cancelBackgroundTasks() {
logMetricsTask.cancel(false);
logMetricsTask = null;
}
+ drainTasks.values().forEach(task -> task.cancel(false));
+ drainTasks.clear();
+ List primesToCancel = new ArrayList<>(pendingPrimes);
+ primesToCancel.forEach(this::finishPendingPrime);
+ return primesToCancel;
+ }
+
+ private void cancelPendingPrimes(List pendingPrimes, boolean force) {
+ for (PendingPrime pendingPrime : pendingPrimes) {
+ if (pendingPrime.future != null) {
+ pendingPrime.future.cancel(true);
+ }
+ if (pendingPrime.timeoutTask != null) {
+ pendingPrime.timeoutTask.cancel(false);
+ }
+ if (pendingPrime.retryTask != null) {
+ pendingPrime.retryTask.cancel(false);
+ }
+ if (force) {
+ pendingPrime.channel.shutdownNow();
+ } else {
+ pendingPrime.channel.shutdown();
+ }
+ }
}
@Override
public ManagedChannel shutdownNow() {
logger.finer(log("Shutdown now started."));
- for (ChannelRef channelRef : channelRefs) {
+ List primesToCancel = cancelBackgroundTasks();
+ cancelPendingPrimes(primesToCancel, true);
+ List activeSnapshot = new ArrayList<>(channelRefs);
+ List removedSnapshot = new ArrayList<>(removedChannelRefs);
+ for (ChannelRef channelRef : activeSnapshot) {
if (!channelRef.getChannel().isTerminated()) {
channelRef.getChannel().shutdownNow();
}
}
- for (ChannelRef channelRef : removedChannelRefs) {
+ for (ChannelRef channelRef : removedSnapshot) {
if (!channelRef.getChannel().isTerminated()) {
channelRef.getChannel().shutdownNow();
}
}
- cancelBackgroundTasks();
if (!stateNotificationExecutor.isTerminated()) {
stateNotificationExecutor.shutdownNow();
}
@@ -2134,13 +2894,16 @@ public ManagedChannel shutdownNow() {
@Override
public ManagedChannel shutdown() {
logger.finer(log("Shutdown started."));
- for (ChannelRef channelRef : channelRefs) {
+ List primesToCancel = cancelBackgroundTasks();
+ cancelPendingPrimes(primesToCancel, false);
+ List activeSnapshot = new ArrayList<>(channelRefs);
+ List removedSnapshot = new ArrayList<>(removedChannelRefs);
+ for (ChannelRef channelRef : activeSnapshot) {
channelRef.getChannel().shutdown();
}
- for (ChannelRef channelRef : removedChannelRefs) {
+ for (ChannelRef channelRef : removedSnapshot) {
channelRef.getChannel().shutdown();
}
- cancelBackgroundTasks();
stateNotificationExecutor.shutdown();
return this;
}
@@ -2252,10 +3015,13 @@ public ConnectivityState getState(boolean requestConnection) {
* One channel can be mapped to more than one keys. But one key can only be mapped to one
* channel.
*/
- protected void bind(ChannelRef channelRef, List affinityKeys) {
+ protected synchronized void bind(ChannelRef channelRef, List affinityKeys) {
if (channelRef == null || affinityKeys == null) {
return;
}
+ if (!channelRef.isActive()) {
+ channelRef = pickLeastBusyChannel(/* forFallback= */ false);
+ }
if (logger.isLoggable(Level.FINEST)) {
logger.finest(
log(
@@ -2266,13 +3032,25 @@ protected void bind(ChannelRef channelRef, List affinityKeys) {
while (affinityKeyToChannelRef.putIfAbsent(affinityKey, channelRef) != null) {
unbind(Collections.singletonList(affinityKey));
}
- affinityKeyLastUsed.put(affinityKey, System.nanoTime());
+ affinityKeyLastUsed.put(affinityKey, nanoClock.get());
channelRef.affinityCountIncr();
}
}
+ private synchronized void unbindInactiveMapping(String affinityKey, ChannelRef mappedChannel) {
+ if (affinityKeyToChannelRef.remove(affinityKey, mappedChannel)) {
+ Runnable hook = inactiveMappingRemovedHookForTest;
+ if (hook != null) {
+ inactiveMappingRemovedHookForTest = null;
+ hook.run();
+ }
+ affinityKeyLastUsed.remove(affinityKey);
+ mappedChannel.affinityCountDecr();
+ }
+ }
+
/** Unbind channel with affinity key. */
- protected void unbind(List affinityKeys) {
+ protected synchronized void unbind(List affinityKeys) {
if (affinityKeys == null) {
return;
}
@@ -2408,7 +3186,13 @@ protected class ChannelRef {
// activeStreamsCount are mutated from the GcpClientCall concurrently using the
// `activeStreamsCountIncr()` and `activeStreamsCountDecr()` methods.
private final AtomicInteger activeStreamsCount;
- private long lastResponseNanos = nanoClock.get();
+ private final long createdNanos = nanoClock.get();
+ private volatile long lastActivityNanos = createdNanos;
+ private volatile long lastResponseNanos = createdNanos;
+
+ private final AtomicInteger errorPenaltyLoad = new AtomicInteger();
+ private final AtomicLong errorPenaltyExpiresAtNanos = new AtomicLong();
+
private final AtomicInteger deadlineExceededCount = new AtomicInteger();
private final AtomicLong okCalls = new AtomicLong();
private final AtomicLong errCalls = new AtomicLong();
@@ -2431,6 +3215,14 @@ protected long getConnectedSinceNanos() {
return channelStateMonitor.getConnectedSinceNanos();
}
+ protected long getCreatedNanos() {
+ return createdNanos;
+ }
+
+ protected long getLastActivityNanos() {
+ return lastActivityNanos;
+ }
+
protected ConnectivityState getState() {
return channelStateMonitor.getCurrentState();
}
@@ -2447,12 +3239,29 @@ protected boolean isActive() {
return active;
}
- private void activate() {
- active = true;
+ private void activateAndAccountReadiness() {
+ synchronized (this) {
+ active = true;
+ channelStateMonitor.accountReadyIfNeeded();
+ }
+ }
+
+ private void deactivateAndAccountReadiness() {
+ synchronized (this) {
+ channelStateMonitor.unaccountReadyIfNeeded();
+ active = false;
+ }
+ }
+
+ @VisibleForTesting
+ void deactivateForTest() {
+ deactivateAndAccountReadiness();
}
- private void deactivate() {
- active = false;
+ @VisibleForTesting
+ void setActiveStreamsForTest(int streams) {
+ int previous = activeStreamsCount.getAndSet(streams);
+ totalActiveStreams.addAndGet(streams - previous);
}
protected void affinityCountIncr() {
@@ -2472,14 +3281,16 @@ protected void resetAffinityCount() {
}
protected void activeStreamsCountIncr() {
+ lastActivityNanos = nanoClock.get();
int actStreams = activeStreamsCount.incrementAndGet();
maxActiveStreams.accumulateAndGet(actStreams, Math::max);
int totalActStreams = totalActiveStreams.incrementAndGet();
maxTotalActiveStreams.accumulateAndGet(totalActStreams, Math::max);
- maxTotalActiveStreamsForScaleDown.accumulateAndGet(totalActStreams, Math::max);
+ maybeSignalScaleUp(this);
}
protected void activeStreamsCountDecr(long startNanos, Status status, boolean fromClientSide) {
+ lastActivityNanos = nanoClock.get();
int actStreams = activeStreamsCount.decrementAndGet();
minActiveStreams.accumulateAndGet(actStreams, Math::min);
int totalActStreams = totalActiveStreams.decrementAndGet();
@@ -2494,6 +3305,10 @@ protected void activeStreamsCountDecr(long startNanos, Status status, boolean fr
if (unresponsiveDetectionEnabled) {
detectUnresponsiveConnection(startNanos, status, fromClientSide);
}
+ applyErrorPenalty(status);
+ if (!active && actStreams == 0) {
+ scheduleDrain(this);
+ }
}
protected void messageReceived() {
@@ -2509,6 +3324,65 @@ protected int getActiveStreamsCount() {
return activeStreamsCount.get();
}
+ protected int getPickerLoad() {
+ return getActiveStreamsCount() + currentErrorPenalty();
+ }
+
+ @VisibleForTesting
+ int currentErrorPenalty() {
+ long expiry = errorPenaltyExpiresAtNanos.get();
+ if (expiry == 0) {
+ return 0;
+ }
+ if (nanoClock.get() < expiry) {
+ return errorPenaltyLoad.get();
+ }
+ synchronized (this) {
+ expiry = errorPenaltyExpiresAtNanos.get();
+ if (expiry == 0) {
+ return 0;
+ }
+ if (nanoClock.get() < expiry) {
+ return errorPenaltyLoad.get();
+ }
+ errorPenaltyExpiresAtNanos.set(0);
+ // Single net aggregate update for this multi-atomic state transition.
+ totalErrorPenaltyLoad.addAndGet(-errorPenaltyLoad.get());
+ return 0;
+ }
+ }
+
+ private synchronized void applyErrorPenalty(Status status) {
+ if (!isDynamicScalingEnabled
+ || !active
+ || errorPenaltyStep == 0
+ || errorPenaltyDuration.isNegative()
+ || (status.getCode() != Code.UNAVAILABLE
+ && status.getCode() != Code.RESOURCE_EXHAUSTED)) {
+ return;
+ }
+ long now = nanoClock.get();
+ long expiry = errorPenaltyExpiresAtNanos.get();
+ int previousContribution = expiry == 0 ? 0 : errorPenaltyLoad.get();
+ int current = now < expiry ? previousContribution : 0;
+ int next = Math.min(maxRpcPerChannel, current + errorPenaltyStep);
+ errorPenaltyLoad.set(next);
+ errorPenaltyExpiresAtNanos.set(now + errorPenaltyDuration.toNanos());
+ // Single net aggregate update while this channel's fields are locked.
+ long addedPenalty = (long) next - previousContribution;
+ totalErrorPenaltyLoad.addAndGet(addedPenalty);
+ if (addedPenalty > 0) {
+ maybeSignalScaleUp(this);
+ }
+ }
+
+ private synchronized void clearErrorPenalty() {
+ if (errorPenaltyExpiresAtNanos.getAndSet(0) != 0) {
+ // Single net aggregate update for this multi-atomic state transition.
+ totalErrorPenaltyLoad.addAndGet(-errorPenaltyLoad.get());
+ }
+ }
+
protected long getAndResetOkCalls() {
return okCalls.getAndSet(0);
}
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java
index 94a180b6a6e7..b94e04ef7631 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java
@@ -49,13 +49,12 @@ public enum ChannelPickStrategy {
LINEAR_SCAN,
/**
- * Picks two channels at random and returns the one with fewer active streams. Ties are broken
- * by preferring the more recently active channel (warmth-preserving).
+ * Samples two channels at random with replacement and returns the one with lower picker load.
+ * The first sample wins ties.
*
- * This is the default strategy. It avoids the thundering herd problem while keeping warm
- * channels preferred under low traffic. The trade-off is that it may not always find the global
- * minimum, but in practice the difference is negligible because stream counts are inherently
- * racy.
+ *
This is the default strategy. It avoids the thundering herd problem. The trade-off is that
+ * it may not always find the global minimum, but in practice the difference is negligible
+ * because stream counts are inherently racy.
*/
POWER_OF_TWO,
}
@@ -208,10 +207,30 @@ public static class GcpChannelPoolOptions {
// Minimum desired average concurrent calls per channel.
private final int minRpcPerChannel;
- // Maximim desired average concurrent calls per channel.
+ // Maximum desired average concurrent calls per channel.
private final int maxRpcPerChannel;
// How often to check for a possibility to scale down.
private final Duration scaleDownInterval;
+ // Minimum interval between successful scale-up operations.
+ private final Duration scaleUpCooldown;
+ // Number of consecutive low-load observations required before scaling down.
+ private final int scaleDownConsecutiveLowLoadChecks;
+ // Maximum percentage of the active pool that one scale-up operation may add.
+ private final int maxScaleUpPercent;
+ // Maximum number of channels that one scale-down operation may remove.
+ private final int maxScaleDownChannels;
+ // How long an empty draining channel remains available for reuse.
+ private final Duration drainIdleGrace;
+ // Load added after a retryable channel error.
+ private final int errorPenaltyStep;
+ // How long the retryable-error penalty remains in effect.
+ private final Duration errorPenaltyDuration;
+ // Optional hook that warms a scaled-up channel before publication.
+ @Nullable private final GcpChannelPrimer channelPrimer;
+ // Maximum time to wait for one channel-primer future.
+ private final Duration channelPrimeTimeout;
+ // Maximum number of channel-primer attempts before rejecting a channel.
+ private final int channelPrimeMaxAttempts;
// Use round-robin channel selection for affinity binding calls.
private final boolean useRoundRobinOnBind;
@@ -229,6 +248,16 @@ public GcpChannelPoolOptions(Builder builder) {
minRpcPerChannel = builder.minRpcPerChannel;
maxRpcPerChannel = builder.maxRpcPerChannel;
scaleDownInterval = builder.scaleDownInterval;
+ scaleUpCooldown = builder.scaleUpCooldown;
+ scaleDownConsecutiveLowLoadChecks = builder.scaleDownConsecutiveLowLoadChecks;
+ maxScaleUpPercent = builder.maxScaleUpPercent;
+ maxScaleDownChannels = builder.maxScaleDownChannels;
+ drainIdleGrace = builder.drainIdleGrace;
+ errorPenaltyStep = builder.errorPenaltyStep;
+ errorPenaltyDuration = builder.errorPenaltyDuration;
+ channelPrimer = builder.channelPrimer;
+ channelPrimeTimeout = builder.channelPrimeTimeout;
+ channelPrimeMaxAttempts = builder.channelPrimeMaxAttempts;
concurrentStreamsLowWatermark = builder.concurrentStreamsLowWatermark;
useRoundRobinOnBind = builder.useRoundRobinOnBind;
affinityKeyLifetime = builder.affinityKeyLifetime;
@@ -260,6 +289,47 @@ public Duration getScaleDownInterval() {
return scaleDownInterval;
}
+ public Duration getScaleUpCooldown() {
+ return scaleUpCooldown;
+ }
+
+ public int getScaleDownConsecutiveLowLoadChecks() {
+ return scaleDownConsecutiveLowLoadChecks;
+ }
+
+ public int getMaxScaleUpPercent() {
+ return maxScaleUpPercent;
+ }
+
+ public int getMaxScaleDownChannels() {
+ return maxScaleDownChannels;
+ }
+
+ public Duration getDrainIdleGrace() {
+ return drainIdleGrace;
+ }
+
+ public int getErrorPenaltyStep() {
+ return errorPenaltyStep;
+ }
+
+ public Duration getErrorPenaltyDuration() {
+ return errorPenaltyDuration;
+ }
+
+ @Nullable
+ public GcpChannelPrimer getChannelPrimer() {
+ return channelPrimer;
+ }
+
+ public Duration getChannelPrimeTimeout() {
+ return channelPrimeTimeout;
+ }
+
+ public int getChannelPrimeMaxAttempts() {
+ return channelPrimeMaxAttempts;
+ }
+
public int getConcurrentStreamsLowWatermark() {
return concurrentStreamsLowWatermark;
}
@@ -293,8 +363,35 @@ public static GcpChannelPoolOptions.Builder newBuilder(GcpChannelPoolOptions opt
@Override
public String toString() {
return String.format(
- "{maxSize: %d, minSize: %d, concurrentStreamsLowWatermark: %d, useRoundRobinOnBind: %s}",
- getMaxSize(), getMinSize(), getConcurrentStreamsLowWatermark(), isUseRoundRobinOnBind());
+ "{maxSize: %d, minSize: %d, initSize: %d, minRpcPerChannel: %d, "
+ + "maxRpcPerChannel: %d, scaleDownInterval: %s, scaleUpCooldown: %s, "
+ + "scaleDownConsecutiveLowLoadChecks: %d, maxScaleUpPercent: %d, "
+ + "maxScaleDownChannels: %d, drainIdleGrace: %s, errorPenaltyStep: %d, "
+ + "errorPenaltyDuration: %s, concurrentStreamsLowWatermark: %d, "
+ + "useRoundRobinOnBind: %s, affinityKeyLifetime: %s, cleanupInterval: %s, "
+ + "channelPickStrategy: %s, channelPrimer: %s, channelPrimeTimeout: %s, "
+ + "channelPrimeMaxAttempts: %d}",
+ getMaxSize(),
+ getMinSize(),
+ getInitSize(),
+ getMinRpcPerChannel(),
+ getMaxRpcPerChannel(),
+ getScaleDownInterval(),
+ getScaleUpCooldown(),
+ getScaleDownConsecutiveLowLoadChecks(),
+ getMaxScaleUpPercent(),
+ getMaxScaleDownChannels(),
+ getDrainIdleGrace(),
+ getErrorPenaltyStep(),
+ getErrorPenaltyDuration(),
+ getConcurrentStreamsLowWatermark(),
+ isUseRoundRobinOnBind(),
+ getAffinityKeyLifetime(),
+ getCleanupInterval(),
+ getChannelPickStrategy(),
+ getChannelPrimer(),
+ getChannelPrimeTimeout(),
+ getChannelPrimeMaxAttempts());
}
public static class Builder {
@@ -304,6 +401,16 @@ public static class Builder {
private int minRpcPerChannel = 0;
private int maxRpcPerChannel = 0;
private Duration scaleDownInterval = Duration.ZERO;
+ private Duration scaleUpCooldown = Duration.ofSeconds(10);
+ private int scaleDownConsecutiveLowLoadChecks = 3;
+ private int maxScaleUpPercent = 30;
+ private int maxScaleDownChannels = 2;
+ private Duration drainIdleGrace = Duration.ofMinutes(1);
+ private int errorPenaltyStep = 5;
+ private Duration errorPenaltyDuration = Duration.ofSeconds(5);
+ @Nullable private GcpChannelPrimer channelPrimer;
+ private Duration channelPrimeTimeout = Duration.ofSeconds(10);
+ private int channelPrimeMaxAttempts = 3;
private int concurrentStreamsLowWatermark = GcpManagedChannel.DEFAULT_MAX_STREAM;
private boolean useRoundRobinOnBind = false;
private Duration affinityKeyLifetime = Duration.ZERO;
@@ -323,6 +430,16 @@ public Builder(GcpChannelPoolOptions options) {
this.minRpcPerChannel = options.getMinRpcPerChannel();
this.maxRpcPerChannel = options.getMaxRpcPerChannel();
this.scaleDownInterval = options.getScaleDownInterval();
+ this.scaleUpCooldown = options.getScaleUpCooldown();
+ this.scaleDownConsecutiveLowLoadChecks = options.getScaleDownConsecutiveLowLoadChecks();
+ this.maxScaleUpPercent = options.getMaxScaleUpPercent();
+ this.maxScaleDownChannels = options.getMaxScaleDownChannels();
+ this.drainIdleGrace = options.getDrainIdleGrace();
+ this.errorPenaltyStep = options.getErrorPenaltyStep();
+ this.errorPenaltyDuration = options.getErrorPenaltyDuration();
+ this.channelPrimer = options.getChannelPrimer();
+ this.channelPrimeTimeout = options.getChannelPrimeTimeout();
+ this.channelPrimeMaxAttempts = options.getChannelPrimeMaxAttempts();
this.concurrentStreamsLowWatermark = options.getConcurrentStreamsLowWatermark();
this.useRoundRobinOnBind = options.isUseRoundRobinOnBind();
this.affinityKeyLifetime = options.getAffinityKeyLifetime();
@@ -375,23 +492,14 @@ public Builder setInitSize(int initSize) {
/**
* Enables dynamic scaling functionality.
*
- *
When the average number of concurrent calls per channel reaches maxRpcPerChannel
- * the pool will create and add a new channel unless already at max size.
+ *
After a call is counted, load above maxRpcPerChannel on its selected
+ * channel or across the pool average signals a background scale-up worker.
*
*
Every scaleDownInterval a check for downscaling is performed. Based on the
- * maximum total concurrent calls observed since the last check, the desired number of
- * channels is calculated as:
- *
- *
(max_total_concurrent_calls / minRpcPerChannel) rounded up.
- *
- *
If the calculated desired number of channels is lower than the current number of
- * channels, the pool will be downscaled to the desired number or min size (whichever is
- * greater).
- *
- *
When downscaling, channels with the oldest connections are selected. Then the selected
- * channels are removed from the pool but are not instructed to shutdown until all calls are
- * completed. In a case when the pool is scaling up and there is a ready channel awaiting
- * calls completion, the channel will be re-used instead of creating a new channel.
+ * current active-call average, consecutive low-load observations, and midpoint target are
+ * used to decide bounded scale-down. Least-loaded channels drain without new picks and close
+ * after their calls complete and the configured idle grace expires. A READY draining channel
+ * can be reused by a later scale-up before it closes.
*
* @param minRpcPerChannel minimum desired average concurrent calls per channel.
* @param maxRpcPerChannel maximum desired average concurrent calls per channel.
@@ -403,6 +511,9 @@ public Builder setDynamicScaling(
minRpcPerChannel > 0, "Minimum RPCs per channel must be positive.");
Preconditions.checkArgument(
maxRpcPerChannel > 0, "Maximum RPCs per channel must be positive.");
+ Preconditions.checkArgument(
+ minRpcPerChannel <= maxRpcPerChannel,
+ "Minimum RPCs per channel must not exceed maximum RPCs per channel.");
Preconditions.checkArgument(
!scaleDownInterval.isNegative() && !scaleDownInterval.isZero(),
"Scale down interval must be positive.");
@@ -424,6 +535,109 @@ public Builder disableDynamicScaling() {
return this;
}
+ /**
+ * Sets the minimum interval between successful scale-up operations. Zero uses the 10-second
+ * default.
+ */
+ public Builder setScaleUpCooldown(Duration scaleUpCooldown) {
+ Preconditions.checkNotNull(scaleUpCooldown, "Scale up cooldown must not be null.");
+ Preconditions.checkArgument(
+ !scaleUpCooldown.isNegative(), "Scale up cooldown must not be negative.");
+ this.scaleUpCooldown = scaleUpCooldown.isZero() ? Duration.ofSeconds(10) : scaleUpCooldown;
+ return this;
+ }
+
+ /** Sets the number of consecutive low-load checks required before scaling down. */
+ public Builder setScaleDownConsecutiveLowLoadChecks(int checks) {
+ Preconditions.checkArgument(checks > 0, "Scale down checks must be positive.");
+ this.scaleDownConsecutiveLowLoadChecks = checks;
+ return this;
+ }
+
+ /**
+ * Sets the maximum percentage of active channels added by one scale-up operation. The
+ * percentage cap has a two-channel floor before desired-size and maximum-size clamps.
+ */
+ public Builder setMaxScaleUpPercent(int percent) {
+ Preconditions.checkArgument(
+ percent > 0 && percent <= 100, "Scale up percent must be in (0, 100].");
+ this.maxScaleUpPercent = percent;
+ return this;
+ }
+
+ /** Sets the maximum number of channels removed by one scale-down operation. */
+ public Builder setMaxScaleDownChannels(int channels) {
+ Preconditions.checkArgument(channels > 0, "Scale down channel limit must be positive.");
+ this.maxScaleDownChannels = channels;
+ return this;
+ }
+
+ /** Sets how long an empty draining channel remains available for reuse. */
+ public Builder setDrainIdleGrace(Duration drainIdleGrace) {
+ Preconditions.checkNotNull(drainIdleGrace, "Drain idle grace must not be null.");
+ Preconditions.checkArgument(
+ !drainIdleGrace.isNegative(), "Drain idle grace must not be negative.");
+ this.drainIdleGrace = drainIdleGrace;
+ return this;
+ }
+
+ /**
+ * Sets the load penalty added after each retryable channel error. Zero uses the default of 5.
+ * Set a negative {@link #setErrorPenaltyDuration(Duration) penalty duration} to disable
+ * penalties.
+ */
+ public Builder setErrorPenaltyStep(int errorPenaltyStep) {
+ Preconditions.checkArgument(
+ errorPenaltyStep >= 0, "Error penalty step must not be negative.");
+ this.errorPenaltyStep = errorPenaltyStep == 0 ? 5 : errorPenaltyStep;
+ return this;
+ }
+
+ /**
+ * Sets how long retryable-error penalty load remains in effect. A negative duration disables
+ * penalties.
+ */
+ public Builder setErrorPenaltyDuration(Duration errorPenaltyDuration) {
+ Preconditions.checkNotNull(
+ errorPenaltyDuration, "Error penalty duration must not be null.");
+ this.errorPenaltyDuration = errorPenaltyDuration;
+ return this;
+ }
+
+ /**
+ * Sets the optional hook that primes newly built scale-up channels concurrently. Each channel
+ * is published individually when its own primer future succeeds. A {@code null} primer
+ * disables priming and preserves the existing scale-up path.
+ */
+ public Builder setChannelPrimer(@Nullable GcpChannelPrimer channelPrimer) {
+ this.channelPrimer = channelPrimer;
+ return this;
+ }
+
+ /**
+ * Sets the maximum time allowed for each channel-primer attempt. Zero uses the 10-second
+ * default.
+ */
+ public Builder setChannelPrimeTimeout(Duration channelPrimeTimeout) {
+ Preconditions.checkNotNull(channelPrimeTimeout, "Channel prime timeout must not be null.");
+ Preconditions.checkArgument(
+ !channelPrimeTimeout.isNegative(), "Channel prime timeout must not be negative.");
+ this.channelPrimeTimeout =
+ channelPrimeTimeout.isZero() ? Duration.ofSeconds(10) : channelPrimeTimeout;
+ return this;
+ }
+
+ /**
+ * Sets the maximum number of attempts to prime one scaled-up channel. Zero uses the default
+ * of 3. Retry backoff is exponential from 100 ms and capped at 5 s.
+ */
+ public Builder setChannelPrimeMaxAttempts(int channelPrimeMaxAttempts) {
+ Preconditions.checkArgument(
+ channelPrimeMaxAttempts >= 0, "Channel prime max attempts must not be negative.");
+ this.channelPrimeMaxAttempts = channelPrimeMaxAttempts == 0 ? 3 : channelPrimeMaxAttempts;
+ return this;
+ }
+
/**
* Sets the concurrent streams low watermark. If every channel in the pool has at least this
* amount of concurrent streams then a new channel will be created in the pool unless the pool
@@ -482,8 +696,8 @@ public Builder setCleanupInterval(Duration cleanupInterval) {
* Sets the strategy for picking the least busy channel from the pool.
*
*
Defaults to {@link ChannelPickStrategy#POWER_OF_TWO} which avoids the thundering herd
- * problem by randomly sampling two channels and picking the less busy one, with ties broken
- * by channel warmth (most recently active).
+ * problem by sampling two channels with replacement and picking the less busy one. The first
+ * sample wins ties.
*
*
Use {@link ChannelPickStrategy#LINEAR_SCAN} to restore the legacy behavior of scanning
* all channels and always picking the one with the fewest active streams.
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java
index 94db0da5c354..262559ba41f7 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java
@@ -77,4 +77,5 @@ class GcpMetricsConstants {
public static String METRIC_ENDPOINT_SWITCH = "endpoint_switch";
public static String METRIC_CURRENT_ENDPOINT = "current_endpoint";
public static String METRIC_CHANNEL_POOL_SCALING = "channel_pool_scaling";
+ public static String METRIC_SCALE_UP_PRIME_FAILURES = "scale_up_prime_failures";
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java
index ce113e95c457..f63b6c3745cf 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java
@@ -17,12 +17,16 @@
package com.google.cloud.grpc;
import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.cloud.grpc.proto.AffinityConfig;
import io.grpc.CallOptions;
import io.grpc.ClientCall;
import io.grpc.ConnectivityState;
@@ -64,6 +68,9 @@ public T parse(InputStream stream) {
.setResponseMarshaller(new FakeMarshaller<>())
.build();
+ private static final MethodDescriptor STREAMING_METHOD_DESCRIPTOR =
+ METHOD_DESCRIPTOR.toBuilder().setType(MethodDescriptor.MethodType.SERVER_STREAMING).build();
+
@Mock private ManagedChannel delegateChannel;
@Mock private ClientCall delegateCall;
@@ -162,4 +169,137 @@ public void simpleCallUnbindsAffinityKeyOnCancel() {
assertThat(channelRef.getAffinityCount()).isEqualTo(0);
verify(delegateCall).cancel("cancelled", null);
}
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void unaryCallCountsExactlyOnceForWholeLifetime() {
+ GcpClientCall.SimpleGcpClientCall call =
+ new GcpClientCall.SimpleGcpClientCall<>(
+ gcpChannel, channelRef, METHOD_DESCRIPTOR, CallOptions.DEFAULT);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+
+ ArgumentCaptor> listenerCaptor =
+ (ArgumentCaptor>)
+ (ArgumentCaptor) ArgumentCaptor.forClass(ClientCall.Listener.class);
+ verify(delegateCall).start(listenerCaptor.capture(), any(Metadata.class));
+ listenerCaptor.getValue().onClose(Status.OK, new Metadata());
+ call.cancel("late cancel", null);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void streamingCallCountsOnceUntilTerminalClose() {
+ when(delegateChannel.newCall(eq(STREAMING_METHOD_DESCRIPTOR), any(CallOptions.class)))
+ .thenReturn(delegateCall);
+ GcpClientCall.SimpleGcpClientCall call =
+ new GcpClientCall.SimpleGcpClientCall<>(
+ gcpChannel, channelRef, STREAMING_METHOD_DESCRIPTOR, CallOptions.DEFAULT);
+
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ call.sendMessage("request");
+ call.request(10);
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+
+ ArgumentCaptor> listenerCaptor =
+ (ArgumentCaptor>)
+ (ArgumentCaptor) ArgumentCaptor.forClass(ClientCall.Listener.class);
+ verify(delegateCall).start(listenerCaptor.capture(), any(Metadata.class));
+ listenerCaptor.getValue().onClose(Status.OK, new Metadata());
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void simpleCallCancelledBeforeStartNeverDecrementsBelowZero() {
+ GcpClientCall.SimpleGcpClientCall call =
+ new GcpClientCall.SimpleGcpClientCall<>(
+ gcpChannel, channelRef, METHOD_DESCRIPTOR, CallOptions.DEFAULT);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+ call.cancel("before start", null);
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ call.start(new ClientCall.Listener() {}, new Metadata());
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void affinityCallCancelledBeforeFirstMessageDoesNotLeakOrDoubleDecrement() {
+ gcpChannel.channelRefs.add(channelRef);
+ GcpClientCall call =
+ new GcpClientCall<>(
+ gcpChannel,
+ METHOD_DESCRIPTOR,
+ CallOptions.DEFAULT,
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BOUND)
+ .setAffinityKey("name")
+ .build());
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ call.cancel("before message", null);
+ call.halfClose();
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+
+ call.sendMessage("request");
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ ArgumentCaptor> listenerCaptor =
+ (ArgumentCaptor>)
+ (ArgumentCaptor) ArgumentCaptor.forClass(ClientCall.Listener.class);
+ verify(delegateCall).start(listenerCaptor.capture(), any(Metadata.class));
+ verify(delegateCall).cancel("before message", null);
+ verify(delegateCall, never()).halfClose();
+ verify(delegateCall, never()).sendMessage(any());
+ listenerCaptor.getValue().onClose(Status.CANCELLED, new Metadata());
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void affinityQueuedCallFailureClearsQueueAndReleasesCountOnce() {
+ gcpChannel.channelRefs.add(channelRef);
+ IllegalStateException failure = new IllegalStateException("queued start failed");
+ doThrow(failure).when(delegateCall).start(any(), any(Metadata.class));
+ GcpClientCall call =
+ new GcpClientCall<>(
+ gcpChannel,
+ METHOD_DESCRIPTOR,
+ CallOptions.DEFAULT,
+ AffinityConfig.newBuilder().setCommand(AffinityConfig.Command.BOUND).build());
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ call.request(1);
+ assertThat(call.queuedCallCountForTest()).isEqualTo(2);
+
+ IllegalStateException thrown =
+ assertThrows(IllegalStateException.class, () -> call.sendMessage("request"));
+
+ assertThat(thrown).isSameInstanceAs(failure);
+ assertThat(call.queuedCallCountForTest()).isEqualTo(0);
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ call.cancel("late cancel", null);
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void affinityCallPropagatesSelectedChannelIdInCallOptions() {
+ gcpChannel.channelRefs.add(channelRef);
+ GcpClientCall call =
+ new GcpClientCall<>(
+ gcpChannel,
+ METHOD_DESCRIPTOR,
+ CallOptions.DEFAULT,
+ AffinityConfig.newBuilder().setCommand(AffinityConfig.Command.BOUND).build());
+
+ call.sendMessage("request");
+
+ ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(CallOptions.class);
+ verify(delegateChannel).newCall(eq(METHOD_DESCRIPTOR), optionsCaptor.capture());
+ assertThat(optionsCaptor.getValue().getOption(GcpManagedChannel.CHANNEL_ID_KEY))
+ .isEqualTo(channelRef.getId());
+ }
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelDynamicPoolTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelDynamicPoolTest.java
new file mode 100644
index 000000000000..9d3d140fef88
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelDynamicPoolTest.java
@@ -0,0 +1,1226 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelAffinityRef;
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.SettableFuture;
+import io.grpc.ConnectivityState;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Dynamic channel pool behavior tests. */
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelDynamicPoolTest {
+ private final ExecutorService stateExecutor = Executors.newSingleThreadExecutor();
+ private GcpManagedChannel pool;
+
+ @After
+ public void tearDown() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ stateExecutor.shutdownNow();
+ }
+
+ @Test
+ public void hotChannelAloneSignalsScaleUpAfterCallIsCounted() throws Exception {
+ pool = newPool(2, 2, 4, 2, 5, Duration.ofSeconds(30), builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+
+ for (int i = 0; i < 7; i++) {
+ hot.activeStreamsCountIncr();
+ }
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(pool.channelRefs.get(1).getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void poolAverageSignalsScaleUpWhenSelectedChannelIsBelowMaximum() throws Exception {
+ pool = newPool(2, 2, 4, 2, 5, Duration.ofSeconds(30), builder());
+ ChannelRef selected = pool.channelRefs.get(0);
+ pool.channelRefs.get(1).setActiveStreamsForTest(12);
+
+ selected.activeStreamsCountIncr();
+
+ assertThat(selected.getPickerLoad()).isAtMost(5);
+ awaitCondition(() -> pool.getNumberOfChannels() == 4);
+ }
+
+ @Test
+ public void scaleUpBuildsOnBackgroundWorkerNotCallerThread() throws Exception {
+ AtomicReference scaleUpThread = new AtomicReference<>();
+ AtomicInteger builds = new AtomicInteger();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() > 2) {
+ scaleUpThread.set(Thread.currentThread().getName());
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ });
+ pool = newPool(2, 2, 4, 2, 5, Duration.ofSeconds(30), delegate);
+ String callerThread = Thread.currentThread().getName();
+
+ for (int i = 0; i < 7; i++) {
+ pool.channelRefs.get(0).activeStreamsCountIncr();
+ }
+
+ awaitCondition(() -> scaleUpThread.get() != null);
+ assertThat(scaleUpThread.get()).startsWith("gcp-mc-bg-");
+ assertThat(scaleUpThread.get()).isNotEqualTo(callerThread);
+ }
+
+ @Test
+ public void successfulPrimerDelaysPublicationAndRunsOnBackgroundWorker() throws Exception {
+ SettableFuture primeFuture = SettableFuture.create();
+ AtomicReference primerThread = new AtomicReference<>();
+ AtomicReference primingChannel =
+ new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primerThread.set(Thread.currentThread().getName());
+ primingChannel.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return primeFuture;
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> primingChannel.get() != null);
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ assertThat(primerThread.get()).startsWith("gcp-mc-bg-");
+ primeFuture.set(null);
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ }
+
+ @Test
+ public void scaleUpStartsAllChannelPrimersConcurrently() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(3);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(10);
+ primeFutures.forEach(future -> future.set(null));
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> pool.getNumberOfChannels() == 13);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(13);
+ }
+
+ @Test
+ public void primedChannelIsPublishedBeforeRestOfBatchCompletes() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ Map, GcpManagedChannelTest.FakeManagedChannel> primingChannels =
+ new java.util.concurrent.ConcurrentHashMap<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primingChannels.put(future, (GcpManagedChannelTest.FakeManagedChannel) channel);
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ SettableFuture firstFuture = primeFutures.get(0);
+ GcpManagedChannelTest.FakeManagedChannel firstChannel = primingChannels.get(firstFuture);
+ firstFuture.set(null);
+ awaitCondition(() -> pool.getNumberOfChannels() == 11);
+ assertThat(
+ pool.channelRefs.stream()
+ .anyMatch(channelRef -> channelRef.getChannel() == firstChannel))
+ .isTrue();
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(2);
+ assertThat(primeFutures.get(1).isDone()).isFalse();
+ assertThat(primeFutures.get(2).isDone()).isFalse();
+ primeFutures.get(1).set(null);
+ primeFutures.get(2).set(null);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> pool.getNumberOfChannels() == 13);
+ }
+
+ @Test
+ public void failedPrimeDoesNotDelayOtherChannels() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ primeFutures.get(0).setException(new IllegalStateException("prime failed"));
+ primeFutures.get(1).set(null);
+ primeFutures.get(2).set(null);
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> pool.getNumberOfChannels() == 12);
+ assertThat(pool.scaleUpPrimeFailuresForTest()).isEqualTo(1);
+ }
+
+ @Test
+ public void timedOutPrimeDoesNotDelayOtherChannels() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(1), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ primeFutures.get(1).set(null);
+ primeFutures.get(2).set(null);
+ awaitCondition(() -> pool.getNumberOfChannels() == 12);
+ assertThat(primeFutures.get(0).isDone()).isFalse();
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(1);
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ }
+
+ @Test
+ public void shutdownClosesEveryUnpublishedPrimingChannel() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List primingChannels = new CopyOnWriteArrayList<>();
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primingChannels.add((GcpManagedChannelTest.FakeManagedChannel) channel);
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+ hot.activeStreamsCountIncr();
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+
+ pool.shutdownNow();
+
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ assertThat(primingChannels).hasSize(3);
+ assertThat(primingChannels.stream().allMatch(channel -> channel.isShutdown())).isTrue();
+ assertThat(primeFutures.stream().allMatch(Future::isCancelled)).isTrue();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(10);
+ }
+
+ @Test
+ public void failedPrimerRejectsChannelAndLaterScaleUpStillWorks() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference rejected = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ if (primeCalls.incrementAndGet() == 1) {
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return Futures.immediateFailedFuture(new IllegalStateException("prime failed"));
+ }
+ return Futures.immediateVoidFuture();
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 1, builder());
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(primeCalls.get()).isEqualTo(2);
+ }
+
+ @Test
+ public void primerTimeoutRejectsAndClosesChannel() throws Exception {
+ SettableFuture neverCompletes = SettableFuture.create();
+ AtomicReference rejected = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return neverCompletes;
+ };
+ pool = newPrimedPool(primer, Duration.ofMillis(20), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ }
+
+ @Test
+ public void primerRetriesUntilSuccess() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ GcpChannelPrimer primer =
+ channel ->
+ primeCalls.incrementAndGet() < 3
+ ? Futures.immediateFailedFuture(new IllegalStateException("prime failed"))
+ : Futures.immediateVoidFuture();
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 3, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(primeCalls.get()).isEqualTo(3);
+ assertThat(pool.scaleUpPrimeFailuresForTest()).isEqualTo(0);
+ }
+
+ @Test
+ public void primerRetriesExhaustedRejectsChannel() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference rejected = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ primeCalls.incrementAndGet();
+ return Futures.immediateFailedFuture(new IllegalStateException("prime failed"));
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 3, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ assertThat(primeCalls.get()).isEqualTo(3);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ }
+
+ @Test
+ public void primerBackoffIsCappedForManyAttempts() {
+ List backoffs = new CopyOnWriteArrayList<>();
+
+ for (int attempt = 0; attempt < 49; attempt++) {
+ backoffs.add(GcpManagedChannel.primeBackoffMillisForTest(attempt));
+ }
+
+ assertThat(backoffs).hasSize(49);
+ assertThat(backoffs.stream().mapToLong(Long::longValue).max().orElse(0)).isAtMost(5_000L);
+ assertThat(backoffs.stream().mapToLong(Long::longValue).sum()).isEqualTo(221_300L);
+ }
+
+ @Test
+ public void reusableDrainingChannelSkipsPrimer() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ GcpChannelPrimer primer =
+ channel -> {
+ primeCalls.incrementAndGet();
+ return Futures.immediateVoidFuture();
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), builder());
+ for (ChannelRef ref : pool.channelRefs) {
+ ((GcpManagedChannelTest.FakeManagedChannel) ref.getChannel())
+ .setState(ConnectivityState.READY);
+ }
+ awaitCondition(() -> pool.readyChannelCountForTest() == 2);
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(1);
+
+ ChannelRef active = pool.channelRefs.get(0);
+ active.setActiveStreamsForTest(3);
+ active.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 2);
+ assertThat(primeCalls.get()).isEqualTo(0);
+ }
+
+ @Test
+ public void keyedBindCompletesWhileScaleUpDelegateBuildIsBlocked() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ CountDownLatch scaleUpBuildStarted = new CountDownLatch(1);
+ CountDownLatch releaseScaleUpBuild = new CountDownLatch(1);
+ ExecutorService binder = Executors.newSingleThreadExecutor();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() > 2) {
+ scaleUpBuildStarted.countDown();
+ try {
+ releaseScaleUpBuild.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ });
+ try {
+ pool = newPool(2, 2, 4, 2, 5, Duration.ofSeconds(30), delegate);
+ for (int i = 0; i < 7; i++) {
+ pool.channelRefs.get(0).activeStreamsCountIncr();
+ }
+ awaitCondition(() -> scaleUpBuildStarted.getCount() == 0);
+
+ Future bind =
+ binder.submit(() -> pool.bind(pool.channelRefs.get(0), Collections.singletonList("key")));
+ await().atMost(Duration.ofSeconds(1)).until(bind::isDone);
+ assertThat(pool.affinityKeyToChannelRef).containsKey("key");
+ } finally {
+ releaseScaleUpBuild.countDown();
+ binder.shutdownNow();
+ }
+ }
+
+ @Test
+ public void inactiveMappingCleanupIsAtomicWithConcurrentBind() throws Exception {
+ AtomicLong clock = new AtomicLong(1);
+ pool = affinityPool(Duration.ofNanos(1), builder());
+ pool.setNanoClock(clock::get);
+ ChannelRef inactive = pool.channelRefs.get(0);
+ ChannelRef rebound = pool.channelRefs.get(1);
+ String key = "session";
+ pool.bind(inactive, Collections.singletonList(key));
+ inactive.deactivateForTest();
+
+ CountDownLatch mappingRemoved = new CountDownLatch(1);
+ CountDownLatch bindAttempted = new CountDownLatch(1);
+ AtomicReference bindingThread = new AtomicReference<>();
+ pool.setInactiveMappingRemovedHookForTest(
+ () -> {
+ mappingRemoved.countDown();
+ try {
+ assertThat(bindAttempted.await(5, TimeUnit.SECONDS)).isTrue();
+ // Binding must wait for inactive cleanup to remove the matching timestamp.
+ await()
+ .atMost(Duration.ofSeconds(5))
+ .until(() -> bindingThread.get().getState() == Thread.State.BLOCKED);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ });
+ ExecutorService callers = Executors.newFixedThreadPool(2);
+ try {
+ Future resolver = callers.submit(() -> pool.getChannelRef(key));
+ assertThat(mappingRemoved.await(5, TimeUnit.SECONDS)).isTrue();
+ Future binder =
+ callers.submit(
+ () -> {
+ bindingThread.set(Thread.currentThread());
+ bindAttempted.countDown();
+ pool.bind(rebound, Collections.singletonList(key));
+ });
+
+ resolver.get(5, TimeUnit.SECONDS);
+ binder.get(5, TimeUnit.SECONDS);
+ boolean hasMapping = pool.affinityKeyToChannelRef.containsKey(key);
+ boolean hasLastUsed = pool.affinityKeyLastUsed.containsKey(key);
+ assertThat(hasMapping).isEqualTo(hasLastUsed);
+ assertThat(hasMapping).isTrue();
+
+ clock.addAndGet(2);
+ pool.cleanupAffinityKeys();
+ assertThat(pool.affinityKeyToChannelRef).doesNotContainKey(key);
+ assertThat(pool.affinityKeyLastUsed).doesNotContainKey(key);
+ } finally {
+ callers.shutdownNow();
+ }
+ }
+
+ @Test
+ public void shutdownPoolPickerCompletesWithUnavailable() throws Exception {
+ ExecutorService picker = Executors.newSingleThreadExecutor();
+ try {
+ pool = newPool(0, 0, 2, 1, 3, Duration.ofSeconds(30), builder());
+ pool.shutdownNow();
+
+ Future pick = picker.submit(() -> pool.getChannelRef(null));
+ await().atMost(Duration.ofSeconds(1)).until(pick::isDone);
+ try {
+ pick.get();
+ throw new AssertionError("picker unexpectedly returned a channel");
+ } catch (ExecutionException expected) {
+ assertThat(expected.getCause()).isInstanceOf(StatusRuntimeException.class);
+ assertThat(((StatusRuntimeException) expected.getCause()).getStatus().getCode())
+ .isEqualTo(Status.Code.UNAVAILABLE);
+ }
+ } finally {
+ picker.shutdownNow();
+ }
+ }
+
+ @Test
+ public void failedScaleUpBuildDoesNotKillFutureScaleUps() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() == 3) {
+ throw new IllegalStateException("one build failure");
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ });
+ pool = newPool(2, 2, 4, 2, 5, Duration.ofSeconds(30), delegate);
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+
+ for (int i = 0; i < 7; i++) {
+ hot.activeStreamsCountIncr();
+ }
+ awaitCondition(() -> builds.get() == 3);
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(builds.get()).isAtLeast(4);
+ }
+
+ @Test
+ public void shutdownReturnsWhileScaleUpBuildIsBlockedAndClosesSurplus() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ AtomicReference surplus = new AtomicReference<>();
+ CountDownLatch scaleUpBuildStarted = new CountDownLatch(1);
+ CountDownLatch releaseScaleUpBuild = new CountDownLatch(1);
+ ExecutorService shutdownExecutor = Executors.newSingleThreadExecutor();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() > 2) {
+ scaleUpBuildStarted.countDown();
+ try {
+ releaseScaleUpBuild.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ GcpManagedChannelTest.FakeManagedChannel channel =
+ new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ surplus.set(channel);
+ return channel;
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ });
+ try {
+ pool = newPool(2, 2, 4, 2, 5, Duration.ofSeconds(30), delegate);
+ for (int i = 0; i < 7; i++) {
+ pool.channelRefs.get(0).activeStreamsCountIncr();
+ }
+ awaitCondition(() -> scaleUpBuildStarted.getCount() == 0);
+
+ Future shutdown = shutdownExecutor.submit(pool::shutdownNow);
+ await().atMost(Duration.ofSeconds(1)).until(shutdown::isDone);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+
+ releaseScaleUpBuild.countDown();
+ awaitCondition(() -> surplus.get() != null && surplus.get().isShutdown());
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ } finally {
+ releaseScaleUpBuild.countDown();
+ shutdownExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void burstSignalsCoalesceWhileScaleUpWorkerIsBusy() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ CountDownLatch scaleUpBuildStarted = new CountDownLatch(1);
+ CountDownLatch releaseScaleUpBuild = new CountDownLatch(1);
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() == 3) {
+ scaleUpBuildStarted.countDown();
+ try {
+ releaseScaleUpBuild.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ });
+ pool = newPool(2, 2, 10, 2, 5, Duration.ofSeconds(30), delegate);
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ for (int i = 0; i < 7; i++) {
+ hot.activeStreamsCountIncr();
+ }
+ assertThat(scaleUpBuildStarted.await(5, TimeUnit.SECONDS)).isTrue();
+
+ for (int i = 7; i < 30; i++) {
+ hot.activeStreamsCountIncr();
+ }
+ clock.incrementAndGet();
+ releaseScaleUpBuild.countDown();
+
+ // First handling adds one; the single buffered follow-up adds two.
+ awaitCondition(() -> pool.getNumberOfChannels() == 5);
+ await()
+ .during(Duration.ofMillis(100))
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> pool.getNumberOfChannels() == 5);
+ }
+
+ @Test
+ public void removedChannelsDrainThenShutdown() throws Exception {
+ pool = newPool(2, 1, 2, 1, 3, Duration.ofMillis(20), Duration.ofMillis(30), builder());
+ long startNanos = System.nanoTime();
+ for (ChannelRef ref : pool.channelRefs) {
+ ref.activeStreamsCountIncr();
+ }
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 1);
+ assertThat(pool.removedChannelRefs).hasSize(1);
+ ChannelRef draining = pool.removedChannelRefs.iterator().next();
+ assertThat(draining.getChannel().isShutdown()).isFalse();
+ draining.activeStreamsCountDecr(startNanos, Status.OK, false);
+ awaitCondition(() -> pool.removedChannelRefs.isEmpty() && pool.channelIdMapSizeForTest() == 1);
+ }
+
+ @Test
+ public void concurrentDrainSchedulingKeepsOneTaskPerChannel() throws Exception {
+ pool = newPool(1, 1, 1, 1, 3, Duration.ofSeconds(30), Duration.ofMinutes(1), builder());
+ ChannelRef draining = pool.channelRefs.get(0);
+ draining.deactivateForTest();
+ CountDownLatch schedulersReady = new CountDownLatch(2);
+ CountDownLatch start = new CountDownLatch(1);
+ ExecutorService schedulers = Executors.newFixedThreadPool(2);
+ try {
+ Future first =
+ schedulers.submit(
+ () -> {
+ schedulersReady.countDown();
+ start.await();
+ pool.scheduleDrain(draining);
+ return null;
+ });
+ Future second =
+ schedulers.submit(
+ () -> {
+ schedulersReady.countDown();
+ start.await();
+ pool.scheduleDrain(draining);
+ return null;
+ });
+ assertThat(schedulersReady.await(5, TimeUnit.SECONDS)).isTrue();
+ start.countDown();
+ first.get(5, TimeUnit.SECONDS);
+ second.get(5, TimeUnit.SECONDS);
+
+ assertThat(pool.drainTaskCountForTest()).isEqualTo(1);
+ } finally {
+ schedulers.shutdownNow();
+ }
+ }
+
+ @Test
+ public void errorPenaltyAccumulatesDecaysAndBiasesPicker() {
+ AtomicLong clock = new AtomicLong(1_000_000_000L);
+ pool = newPool(2, 2, 2, 2, 10, Duration.ofSeconds(30), builder());
+ pool.setNanoClock(clock::get);
+ ChannelRef penalized = pool.channelRefs.get(0);
+ ChannelRef healthy = pool.channelRefs.get(1);
+
+ penalized.activeStreamsCountIncr();
+ penalized.activeStreamsCountDecr(clock.get(), Status.UNAVAILABLE, false);
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(5);
+ assertThat(pool.pickLessBusy(penalized, healthy)).isSameInstanceAs(healthy);
+
+ penalized.activeStreamsCountIncr();
+ penalized.activeStreamsCountDecr(clock.get(), Status.RESOURCE_EXHAUSTED, false);
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(10);
+
+ penalized.activeStreamsCountIncr();
+ penalized.activeStreamsCountDecr(clock.get(), Status.UNAVAILABLE, false);
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(10);
+
+ clock.addAndGet(Duration.ofSeconds(6).toNanos());
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(0);
+ assertThat(pool.pickLessBusy(penalized, healthy)).isSameInstanceAs(penalized);
+ }
+
+ @Test
+ public void pickerDoesNotRetainActiveChannelSnapshotHelper() {
+ try {
+ GcpManagedChannel.class.getDeclaredMethod("activeChannelSnapshot");
+ throw new AssertionError("picker still allocates active-channel snapshots");
+ } catch (NoSuchMethodException expected) {
+ // Method removal keeps the picker on the CopyOnWriteArrayList hot path.
+ }
+ }
+
+ @Test
+ public void powerOfTwoTieKeepsFirstSample() {
+ pool = newPool(2, 2, 2, 2, 5, Duration.ofSeconds(30), builder());
+ ChannelRef first = pool.channelRefs.get(0);
+ ChannelRef second = pool.channelRefs.get(1);
+
+ assertThat(pool.pickLessBusy(first, second)).isSameInstanceAs(first);
+ assertThat(pool.pickLessBusy(second, first)).isSameInstanceAs(second);
+ }
+
+ @Test
+ public void powerOfTwoUsesCandidateRetryBoundBeforeFullScan() {
+ pool = newPool(4, 4, 4, 2, 5, Duration.ofSeconds(30), builder());
+ pool.channelRefs.get(0).deactivateForTest();
+ pool.channelRefs.get(1).deactivateForTest();
+ ChannelRef leastLoaded = pool.channelRefs.get(2);
+ ChannelRef finalSample = pool.channelRefs.get(3);
+ finalSample.setActiveStreamsForTest(10);
+ int[] samples = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3};
+ AtomicInteger nextSample = new AtomicInteger();
+ pool.setCandidateIndexPickerForTest(bound -> samples[nextSample.getAndIncrement()]);
+
+ ChannelRef picked = pool.pickFromCandidates(pool.channelRefs);
+
+ assertThat(picked).isSameInstanceAs(finalSample);
+ assertThat(picked).isNotSameInstanceAs(leastLoaded);
+ assertThat(nextSample.get()).isEqualTo(4 * pool.channelRefs.size());
+ }
+
+ @Test
+ public void pickerRetriesWhenChannelDeactivatesBeforeValidation() {
+ pool = newPool(2, 2, 2, 2, 5, Duration.ofSeconds(30), builder());
+ AtomicReference deactivated = new AtomicReference<>();
+ pool.setPickerValidationHookForTest(
+ candidate -> {
+ deactivated.set(candidate);
+ candidate.deactivateForTest();
+ });
+
+ ChannelRef picked = pool.getChannelRef(null);
+ picked.activeStreamsCountIncr();
+
+ assertThat(picked).isNotSameInstanceAs(deactivated.get());
+ assertThat(deactivated.get().getActiveStreamsCount()).isEqualTo(0);
+ assertThat(picked.getActiveStreamsCount()).isEqualTo(1);
+ }
+
+ @Test
+ public void fallbackUsesChannelIdMapAfterPoolHasIndexGap() {
+ pool = fallbackPool(3, 100);
+ ChannelRef removed = pool.channelRefs.get(0);
+ ChannelRef mapped = pool.channelRefs.get(1);
+ ChannelRef fallback = pool.channelRefs.get(2);
+ String key = "session";
+ pool.bind(mapped, Collections.singletonList(key));
+ pool.processChannelStateChange(mapped.getId(), ConnectivityState.TRANSIENT_FAILURE);
+ pool.fallbackMapForTest().get(mapped.getId()).put(key, fallback.getId());
+ pool.channelRefs.remove(removed);
+
+ assertThat(pool.getChannelRef(key)).isSameInstanceAs(fallback);
+ }
+
+ @Test
+ public void fallbackEligibilityUsesConfiguredWatermark() {
+ pool = fallbackPool(2, 1);
+ ChannelRef mapped = pool.channelRefs.get(0);
+ ChannelRef atWatermark = pool.channelRefs.get(1);
+ String key = "session";
+ pool.bind(mapped, Collections.singletonList(key));
+ atWatermark.activeStreamsCountIncr();
+ pool.processChannelStateChange(mapped.getId(), ConnectivityState.TRANSIENT_FAILURE);
+
+ assertThat(pool.getChannelRef(key)).isSameInstanceAs(mapped);
+ }
+
+ @Test
+ public void scaleDownUnbindsAffinityAndUpdatesAggregateCount() throws Exception {
+ pool = newPool(2, 1, 2, 1, 3, Duration.ofMillis(20), Duration.ofSeconds(5), builder());
+ ChannelRef victim = pool.channelRefs.get(0);
+ String key = "session";
+ pool.bind(victim, Collections.singletonList(key));
+ pool.channelRefs.get(1).activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 1);
+ assertThat(pool.affinityKeyToChannelRef).doesNotContainKey(key);
+ assertThat(pool.affinityKeyLastUsed).doesNotContainKey(key);
+ assertThat(victim.getAffinityCount()).isEqualTo(0);
+ assertThat(pool.totalAffinityCountForTest()).isEqualTo(0);
+ }
+
+ @Test
+ public void affinityKeyReResolvesAwayFromDrainingChannel() throws Exception {
+ pool = newPool(2, 1, 2, 1, 3, Duration.ofSeconds(30), builder());
+ ChannelRef victim = pool.channelRefs.get(0);
+ String key = "session";
+ pool.bind(victim, Collections.singletonList(key));
+ pool.channelRefs.get(1).activeStreamsCountIncr();
+
+ pool.checkScaleDown();
+
+ assertThat(pool.removedChannelRefs).contains(victim);
+ ChannelRef resolved = pool.getChannelRef(key);
+ assertThat(resolved).isNotSameInstanceAs(victim);
+ assertThat(resolved.isActive()).isTrue();
+ }
+
+ @Test
+ public void readyAccountingRemainsExactWhenDrainingChannelIsReused() throws Exception {
+ pool = newPool(2, 1, 2, 1, 3, Duration.ofSeconds(30), Duration.ofSeconds(5), builder());
+ for (ChannelRef ref : pool.channelRefs) {
+ ((GcpManagedChannelTest.FakeManagedChannel) ref.getChannel())
+ .setState(ConnectivityState.READY);
+ }
+ awaitCondition(() -> pool.readyChannelCountForTest() == 2);
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(1);
+ assertThat(pool.readyChannelCountForTest()).isEqualTo(1);
+
+ ChannelRef active = pool.channelRefs.get(0);
+ for (int i = 0; i < 4; i++) {
+ active.activeStreamsCountIncr();
+ }
+ awaitCondition(() -> pool.getNumberOfChannels() == 2);
+ assertThat(pool.readyChannelCountForTest()).isEqualTo(2);
+ }
+
+ @Test
+ public void dynamicPoolWithZeroInitialSizeCreatesFirstChannelWithoutDivision() {
+ pool = newPool(0, 0, 2, 1, 3, Duration.ofSeconds(30), builder());
+
+ assertThat(pool.getChannelRef(null)).isNotNull();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(1);
+ }
+
+ @Test
+ public void oneScaleDownCheckRemovesAtMostConfiguredLimit() throws Exception {
+ pool = newPool(6, 1, 6, 1, 3, Duration.ofSeconds(30), builder());
+
+ pool.checkScaleDown();
+
+ assertThat(pool.getNumberOfChannels()).isEqualTo(4);
+ assertThat(pool.removedChannelRefs).hasSize(2);
+ }
+
+ @Test
+ public void scaleUpEventAddsAtMostThirtyPercentWithTwoChannelMinimum() throws Exception {
+ pool =
+ newPool(
+ 10,
+ 10,
+ 30,
+ 2,
+ 5,
+ Duration.ofSeconds(30),
+ Duration.ofMinutes(1),
+ Duration.ofMinutes(1),
+ builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+
+ hot.setActiveStreamsForTest(99);
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 13);
+ await()
+ .during(Duration.ofMillis(100))
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> pool.getNumberOfChannels() == 13);
+
+ pool.shutdownNow();
+ pool = newPool(1, 1, 5, 2, 5, Duration.ofSeconds(30), builder());
+ hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+
+ pool.shutdownNow();
+ pool = newPool(1, 1, 2, 2, 5, Duration.ofSeconds(30), builder());
+ hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> pool.getNumberOfChannels() == 2);
+ }
+
+ @Test
+ public void scaleDownRequiresConfiguredConsecutiveLowLoadChecks() throws Exception {
+ pool =
+ newPool(
+ 6,
+ 1,
+ 6,
+ 1,
+ 3,
+ Duration.ofSeconds(30),
+ Duration.ofMinutes(1),
+ Duration.ZERO,
+ 3,
+ builder());
+
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(6);
+
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(4);
+ }
+
+ @Test
+ public void drainingChannelsAreSkippedByPowerOfTwoAndRoundRobin() throws Exception {
+ pool = newPool(4, 2, 4, 1, 3, Duration.ofSeconds(30), builder());
+ pool.checkScaleDown();
+ assertThat(pool.removedChannelRefs).hasSize(2);
+
+ for (int i = 0; i < 200; i++) {
+ assertThat(pool.removedChannelRefs).doesNotContain(pool.getChannelRef(null));
+ assertThat(pool.removedChannelRefs).doesNotContain(pool.getChannelRefRoundRobin());
+ }
+ }
+
+ @Test
+ public void idleDrainWaitsForGraceBeforeClosing() {
+ AtomicLong clock = new AtomicLong(System.nanoTime());
+ pool = newPool(0, 0, 2, 1, 3, Duration.ofSeconds(30), Duration.ofMinutes(1), builder());
+ pool.setNanoClock(clock::get);
+ pool.createNewChannel();
+ pool.createNewChannel();
+ pool.checkScaleDown();
+ ChannelRef draining = pool.removedChannelRefs.iterator().next();
+
+ pool.finishDrain(draining);
+ assertThat(draining.getChannel().isShutdown()).isFalse();
+
+ clock.addAndGet(Duration.ofMinutes(1).plusNanos(1).toNanos());
+ pool.finishDrain(draining);
+ assertThat(draining.getChannel().isShutdown()).isTrue();
+ }
+
+ @Test
+ public void affinityReferenceStaysOnDrainingChannelUntilShutdown() throws Exception {
+ AtomicLong clock = new AtomicLong(System.nanoTime());
+ pool = newPool(4, 2, 4, 1, 3, Duration.ofSeconds(30), Duration.ofMinutes(1), builder());
+ pool.setNanoClock(clock::get);
+ ChannelRef victim = pool.channelRefs.get(0);
+ ChannelAffinityRef handle = new ChannelAffinityRef();
+ handle.setChannelIdForTest(victim.getId());
+
+ pool.checkScaleDown();
+ assertThat(pool.removedChannelRefs).contains(victim);
+
+ ChannelRef firstCall = pool.getChannelRefByAffinityRef(handle);
+ firstCall.activeStreamsCountIncr();
+ assertThat(firstCall).isSameInstanceAs(victim);
+ assertThat(firstCall.getId()).isEqualTo(victim.getId());
+ pool.finishDrain(victim);
+ assertThat(victim.getChannel().isShutdown()).isFalse();
+
+ ChannelRef secondCall = pool.getChannelRefByAffinityRef(handle);
+ assertThat(secondCall).isSameInstanceAs(victim);
+ assertThat(secondCall.getId()).isEqualTo(firstCall.getId());
+ secondCall.activeStreamsCountDecr(clock.get(), Status.OK, false);
+
+ clock.addAndGet(Duration.ofMinutes(1).plusNanos(1).toNanos());
+ pool.finishDrain(victim);
+ assertThat(victim.getChannel().isShutdown()).isTrue();
+
+ ChannelRef afterShutdown = pool.getChannelRefByAffinityRef(handle);
+ assertThat(afterShutdown).isNotSameInstanceAs(victim);
+ assertThat(afterShutdown.isActive()).isTrue();
+ }
+
+ @Test
+ public void shutdownHandlesRebindWithoutHerdingOntoLateCreatedChannel() throws Exception {
+ pool = newPool(8, 8, 9, 2, 5, Duration.ofSeconds(30), builder());
+ AtomicLong clock = new AtomicLong(System.nanoTime() + Duration.ofMinutes(1).toNanos());
+ pool.setNanoClock(clock::get);
+ ChannelRef lateCreated = pool.createNewChannel();
+ ChannelRef removed = pool.channelRefs.get(0);
+ pool.channelRefs.remove(removed);
+ removed.deactivateForTest();
+ pool.removedChannelRefs.add(removed);
+ removed.getChannel().shutdownNow();
+
+ int[] picksById = new int[9];
+ for (int i = 0; i < 10_000; i++) {
+ ChannelAffinityRef handle = new ChannelAffinityRef();
+ handle.setChannelIdForTest(removed.getId());
+ picksById[pool.getChannelRefByAffinityRef(handle).getId()]++;
+ }
+
+ // Old warmth-biased P2 made the newest channel win about 25% of these eight-way rebinds.
+ assertThat(picksById[lateCreated.getId()]).isLessThan(2_000);
+ for (ChannelRef active : pool.channelRefs) {
+ assertThat(picksById[active.getId()]).isGreaterThan(0);
+ }
+ }
+
+ private GcpManagedChannelTest.FakeManagedChannelBuilder builder() {
+ return new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(stateExecutor));
+ }
+
+ private GcpManagedChannel newPool(
+ int initial,
+ int minimum,
+ int maximum,
+ int minRpc,
+ int maxRpc,
+ Duration scaleDownInterval,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ return newPool(
+ initial,
+ minimum,
+ maximum,
+ minRpc,
+ maxRpc,
+ scaleDownInterval,
+ Duration.ofMinutes(1),
+ Duration.ofNanos(1),
+ delegate);
+ }
+
+ private GcpManagedChannel newPool(
+ int initial,
+ int minimum,
+ int maximum,
+ int minRpc,
+ int maxRpc,
+ Duration scaleDownInterval,
+ Duration drainIdleGrace,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ return newPool(
+ initial,
+ minimum,
+ maximum,
+ minRpc,
+ maxRpc,
+ scaleDownInterval,
+ drainIdleGrace,
+ Duration.ofNanos(1),
+ delegate);
+ }
+
+ private GcpManagedChannel newPool(
+ int initial,
+ int minimum,
+ int maximum,
+ int minRpc,
+ int maxRpc,
+ Duration scaleDownInterval,
+ Duration drainIdleGrace,
+ Duration scaleUpCooldown,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ return newPool(
+ initial,
+ minimum,
+ maximum,
+ minRpc,
+ maxRpc,
+ scaleDownInterval,
+ drainIdleGrace,
+ scaleUpCooldown,
+ 1,
+ delegate);
+ }
+
+ private GcpManagedChannel newPool(
+ int initial,
+ int minimum,
+ int maximum,
+ int minRpc,
+ int maxRpc,
+ Duration scaleDownInterval,
+ Duration drainIdleGrace,
+ Duration scaleUpCooldown,
+ int consecutiveLowLoadChecks,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(initial)
+ .setMinSize(minimum)
+ .setMaxSize(maximum)
+ .setDynamicScaling(minRpc, maxRpc, scaleDownInterval)
+ .setScaleUpCooldown(scaleUpCooldown)
+ .setScaleDownConsecutiveLowLoadChecks(consecutiveLowLoadChecks)
+ .setMaxScaleUpPercent(30)
+ .setMaxScaleDownChannels(2)
+ .setDrainIdleGrace(drainIdleGrace)
+ .setErrorPenaltyDuration(Duration.ofSeconds(5))
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(poolOptions).build())
+ .build();
+ }
+
+ private GcpManagedChannel fallbackPool(int size, int watermark) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(size)
+ .setMinSize(size)
+ .setMaxSize(size)
+ .setConcurrentStreamsLowWatermark(watermark)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(builder())
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(poolOptions)
+ .withResiliencyOptions(
+ GcpResiliencyOptions.newBuilder().setNotReadyFallback(true).build())
+ .build())
+ .build();
+ }
+
+ private GcpManagedChannel affinityPool(
+ Duration affinityKeyLifetime, GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(2)
+ .setMaxSize(2)
+ .setAffinityKeyLifetime(affinityKeyLifetime)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(poolOptions).build())
+ .build();
+ }
+
+ private GcpManagedChannel newPrimedPool(
+ int initial,
+ int maximum,
+ GcpChannelPrimer primer,
+ Duration primeTimeout,
+ int primeMaxAttempts,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(initial)
+ .setMinSize(1)
+ .setMaxSize(maximum)
+ .setDynamicScaling(1, 3, Duration.ofSeconds(30))
+ .setScaleUpCooldown(Duration.ofNanos(1))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setMaxScaleUpPercent(30)
+ .setMaxScaleDownChannels(2)
+ .setDrainIdleGrace(Duration.ofMinutes(1))
+ .setChannelPrimer(primer)
+ .setChannelPrimeTimeout(primeTimeout)
+ .setChannelPrimeMaxAttempts(primeMaxAttempts)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(poolOptions).build())
+ .build();
+ }
+
+ private GcpManagedChannel newPrimedPool(
+ GcpChannelPrimer primer,
+ Duration primeTimeout,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ return newPrimedPool(primer, primeTimeout, 3, delegate);
+ }
+
+ private GcpManagedChannel newPrimedPool(
+ GcpChannelPrimer primer,
+ Duration primeTimeout,
+ int primeMaxAttempts,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(1)
+ .setMaxSize(3)
+ .setDynamicScaling(1, 3, Duration.ofSeconds(30))
+ .setScaleUpCooldown(Duration.ofNanos(1))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setMaxScaleUpPercent(30)
+ .setMaxScaleDownChannels(2)
+ .setDrainIdleGrace(Duration.ofMinutes(1))
+ .setChannelPrimer(primer)
+ .setChannelPrimeTimeout(primeTimeout)
+ .setChannelPrimeMaxAttempts(primeMaxAttempts)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(poolOptions).build())
+ .build();
+ }
+
+ private static void awaitCondition(java.util.concurrent.Callable condition) {
+ await().atMost(Duration.ofSeconds(5)).until(condition);
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelHotChannelReproducerTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelHotChannelReproducerTest.java
new file mode 100644
index 000000000000..2b4ca42a7509
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelHotChannelReproducerTest.java
@@ -0,0 +1,1142 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assume.assumeTrue;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelAffinityRef;
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.cloud.grpc.proto.AffinityConfig;
+import com.google.cloud.grpc.proto.ApiConfig;
+import com.google.cloud.grpc.proto.MethodConfig;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.spanner.v1.CreateSessionRequest;
+import com.google.spanner.v1.ExecuteSqlRequest;
+import com.google.spanner.v1.ResultSet;
+import com.google.spanner.v1.Session;
+import io.grpc.CallOptions;
+import io.grpc.ClientCall;
+import io.grpc.ClientInterceptor;
+import io.grpc.CompressorRegistry;
+import io.grpc.ConnectivityState;
+import io.grpc.DecompressorRegistry;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+import io.grpc.NameResolver.Factory;
+import io.grpc.Status;
+import io.grpc.protobuf.ProtoUtils;
+import io.grpc.stub.ClientCalls;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Dynamic-pool per-channel request-skew tests.
+ *
+ * Deterministic regressions always run. The load-shaped reproducer is opt-in with {@code
+ * -Dhotchannel.load=true}; configure its seeds with {@code -Dhotchannel.seeds=1103,7,42,99,123}.
+ */
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelHotChannelReproducerTest {
+ private static final int MIN_SIZE = 2;
+ private static final int INITIAL_SIZE = 4;
+ private static final int MAX_SIZE = 48;
+ private static final int MIN_RPC_PER_CHANNEL = 15;
+ private static final int MAX_RPC_PER_CHANNEL = 25;
+ private static final int DEFAULT_CALLERS = 300;
+ private static final int SESSION_COUNT = 100;
+ private static final Duration TEST_SCALE_DOWN_INTERVAL = Duration.ofMillis(120);
+ private static final CallOptions.Key TEST_LATENCY_MILLIS =
+ CallOptions.Key.create("hotchannel-test-latency-millis");
+ private static final MethodDescriptor CREATE_SESSION_METHOD =
+ MethodDescriptor.newBuilder()
+ .setType(MethodDescriptor.MethodType.UNARY)
+ .setFullMethodName("google.spanner.v1.Spanner/CreateSession")
+ .setRequestMarshaller(ProtoUtils.marshaller(CreateSessionRequest.getDefaultInstance()))
+ .setResponseMarshaller(ProtoUtils.marshaller(Session.getDefaultInstance()))
+ .build();
+ private static final MethodDescriptor EXECUTE_SQL_METHOD =
+ MethodDescriptor.newBuilder()
+ .setType(MethodDescriptor.MethodType.UNARY)
+ .setFullMethodName("google.spanner.v1.Spanner/ExecuteSql")
+ .setRequestMarshaller(ProtoUtils.marshaller(ExecuteSqlRequest.getDefaultInstance()))
+ .setResponseMarshaller(ProtoUtils.marshaller(ResultSet.getDefaultInstance()))
+ .build();
+ private static final CallOptions.Key CHANNEL_AFFINITY_REF_KEY =
+ GcpManagedChannel.CHANNEL_AFFINITY_REF_KEY;
+
+ @Test
+ public void dynamicPoolDoesNotDevelopOneHotChannel() throws Exception {
+ assumeTrue("load-shaped reproducer disabled", Boolean.getBoolean("hotchannel.load"));
+ Variant variant =
+ Variant.valueOf(
+ System.getProperty("hotchannel.variant", "REF_6120").toUpperCase(Locale.ROOT));
+ LoadShape loadShape =
+ LoadShape.valueOf(
+ System.getProperty("hotchannel.loadShape", "BURSTY").toUpperCase(Locale.ROOT));
+ String[] seeds = System.getProperty("hotchannel.seeds", "1103").split(",");
+ boolean observeOnly = Boolean.getBoolean("hotchannel.observeOnly");
+
+ for (String seedValue : seeds) {
+ long seed = Long.parseLong(seedValue.trim());
+ RunResult result = runScenario(seed, variant, loadShape);
+ System.out.println(result.format());
+ if (!observeOnly) {
+ assertWithMessage("one-hot-channel skew: %s", result.format())
+ .that(result.hotToMedian)
+ .isLessThan(3.0);
+ }
+ }
+ }
+
+ @Test
+ public void warmChannelDoesNotBiasPowerOfTwoTies() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool = fixedPool(new RecordingChannelBuilder(responses, 1), 2, 2, false);
+ ChannelRef warm = pool.channelRefs.get(1);
+ warm.messageReceived();
+
+ int warmPicks = 0;
+ for (int i = 0; i < 200; i++) {
+ if (pool.getChannelRef(null) == warm) {
+ warmPicks++;
+ }
+ }
+
+ assertThat(warmPicks).isGreaterThan(60);
+ assertThat(warmPicks).isLessThan(140);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ responses.shutdownNow();
+ }
+ }
+
+ @Test
+ public void selectedCallsReserveLoadBeforeStart() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ RecordingChannelBuilder delegateBuilder = new RecordingChannelBuilder(responses, 2);
+ pool = fixedPool(delegateBuilder, 2, 2, false);
+ pool.channelRefs.get(1).messageReceived();
+
+ for (int i = 0; i < 200; i++) {
+ ChannelAffinityRef affinityRef = new ChannelAffinityRef();
+ pool.newCall(
+ EXECUTE_SQL_METHOD,
+ CallOptions.DEFAULT.withOption(CHANNEL_AFFINITY_REF_KEY, affinityRef));
+ }
+
+ long first = delegateBuilder.stats.get(0).selected.get();
+ long second = delegateBuilder.stats.get(1).selected.get();
+ assertThat(first).isGreaterThan(60);
+ assertThat(second).isGreaterThan(60);
+ assertThat(pool.channelRefs.get(0).getActiveStreamsCount()).isEqualTo((int) first);
+ assertThat(pool.channelRefs.get(1).getActiveStreamsCount()).isEqualTo((int) second);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ responses.shutdownNow();
+ }
+ }
+
+ @Test
+ public void affinityReferencesRedistributeAfterDrainingChannelsShutdown() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ ExecutorService executor = Executors.newFixedThreadPool(100);
+ GcpManagedChannel pool = null;
+ try {
+ pool = fixedPool(new RecordingChannelBuilder(responses, 3), 2, 4, true);
+ List affinityRefs = new ArrayList<>();
+ List originalIds = new ArrayList<>();
+ for (int i = 0; i < 200; i++) {
+ int channelId = pool.channelRefs.get(i % 4).getId();
+ ChannelAffinityRef affinityRef = new ChannelAffinityRef();
+ affinityRef.setChannelIdForTest(channelId);
+ affinityRefs.add(affinityRef);
+ originalIds.add(channelId);
+ }
+
+ invokeScaleDownCheck(pool, 3);
+ assertThat(pool.channelRefs).hasSize(2);
+ Set removedIds = new HashSet<>();
+ for (ChannelRef removed : pool.removedChannelRefs) {
+ removedIds.add(removed.getId());
+ }
+ List orphaned = new ArrayList<>();
+ for (int i = 0; i < affinityRefs.size(); i++) {
+ if (removedIds.contains(originalIds.get(i))) {
+ orphaned.add(affinityRefs.get(i));
+ }
+ }
+ assertThat(orphaned).hasSize(100);
+ for (int i = 0; i < affinityRefs.size(); i++) {
+ if (removedIds.contains(originalIds.get(i))) {
+ assertThat(pool.getChannelRefByAffinityRef(affinityRefs.get(i)).getId())
+ .isEqualTo(originalIds.get(i));
+ }
+ }
+ for (ChannelRef removed : pool.removedChannelRefs) {
+ removed.getChannel().shutdownNow();
+ }
+
+ ChannelRef first = pool.channelRefs.get(0);
+ ChannelRef second = pool.channelRefs.get(1);
+ second.messageReceived();
+ CountDownLatch start = new CountDownLatch(1);
+ List> resolutions = new ArrayList<>();
+ GcpManagedChannel resolvingPool = pool;
+ for (ChannelAffinityRef affinityRef : orphaned) {
+ resolutions.add(
+ executor.submit(
+ () -> {
+ start.await();
+ return resolvingPool.getChannelRefByAffinityRef(affinityRef);
+ }));
+ }
+ start.countDown();
+ int firstPicks = 0;
+ int secondPicks = 0;
+ for (Future resolution : resolutions) {
+ ChannelRef resolved = resolution.get(5, TimeUnit.SECONDS);
+ if (resolved == first) {
+ firstPicks++;
+ } else if (resolved == second) {
+ secondPicks++;
+ }
+ }
+ assertThat(firstPicks).isGreaterThan(25);
+ assertThat(secondPicks).isGreaterThan(25);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ executor.shutdownNow();
+ responses.shutdownNow();
+ }
+ }
+
+ @Test
+ public void scaleDownMarksAtMostTwoOfFortyEightLiveReferencesPerCheck() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool = fixedPool(new RecordingChannelBuilder(responses, 4), 2, 48, true);
+ List handles = new ArrayList<>();
+ for (ChannelRef channelRef : pool.channelRefs) {
+ channelRef.activeStreamsCountIncr();
+ ChannelAffinityRef handle = new ChannelAffinityRef();
+ handle.setChannelIdForTest(channelRef.getId());
+ handles.add(handle);
+ }
+ for (ChannelRef channelRef : pool.channelRefs) {
+ channelRef.activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
+ }
+
+ invokeScaleDownCheck(pool, 2);
+ assertThat(pool.channelRefs).hasSize(48);
+ invokeScaleDownCheck(pool, 1);
+ assertThat(pool.channelRefs).hasSize(46);
+ assertThat(pool.removedChannelRefs).hasSize(2);
+ for (ChannelRef removed : pool.removedChannelRefs) {
+ ChannelRef resolved = pool.getChannelRefByAffinityRef(handles.get(removed.getId()));
+ assertThat(resolved).isSameInstanceAs(removed);
+ assertThat(resolved.isActive()).isFalse();
+ assertThat(resolved.getChannel().isShutdown()).isFalse();
+ }
+
+ invokeScaleDownCheck(pool, 3);
+ assertThat(pool.channelRefs).hasSize(44);
+ assertThat(pool.removedChannelRefs).hasSize(4);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ responses.shutdownNow();
+ }
+ }
+
+ private static GcpManagedChannel fixedPool(
+ RecordingChannelBuilder delegateBuilder, int minimum, int initial, boolean dynamic) {
+ GcpChannelPoolOptions.Builder options =
+ GcpChannelPoolOptions.newBuilder()
+ .setMinSize(minimum)
+ .setInitSize(initial)
+ .setMaxSize(initial)
+ .setDrainIdleGrace(Duration.ofMinutes(1));
+ if (dynamic) {
+ options
+ .setDynamicScaling(10, 20, Duration.ofMinutes(1))
+ .setScaleDownConsecutiveLowLoadChecks(3)
+ .setMaxScaleDownChannels(2);
+ }
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegateBuilder)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(options.build())
+ .build())
+ .build();
+ }
+
+ private static void invokeScaleDownCheck(GcpManagedChannel pool, int times) {
+ for (int i = 0; i < times; i++) {
+ pool.checkScaleDown();
+ }
+ }
+
+ private RunResult runScenario(long seed, Variant variant, LoadShape loadShape) throws Exception {
+ int callers = Integer.getInteger("hotchannel.callers", DEFAULT_CALLERS);
+ if (callers < 20) {
+ throw new IllegalArgumentException("hotchannel.callers must be at least 20");
+ }
+ boolean staticPool = Boolean.getBoolean("hotchannel.staticPool");
+ boolean noScaleDown = Boolean.getBoolean("hotchannel.noScaleDown");
+ Duration phaseInterval =
+ Duration.ofMillis(
+ Long.getLong("hotchannel.scaleDownMillis", TEST_SCALE_DOWN_INTERVAL.toMillis()));
+ Duration scaleDownInterval = noScaleDown ? Duration.ofHours(1) : phaseInterval;
+
+ ScheduledExecutorService responses = Executors.newScheduledThreadPool(32);
+ ExecutorService callersExecutor = Executors.newFixedThreadPool(callers);
+ GcpManagedChannel pool = null;
+ try {
+ RecordingChannelBuilder delegateBuilder = new RecordingChannelBuilder(responses, seed);
+ GcpChannelPoolOptions.Builder poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(MAX_SIZE)
+ .setMinSize(staticPool ? MAX_SIZE : MIN_SIZE)
+ .setInitSize(staticPool ? MAX_SIZE : INITIAL_SIZE)
+ .setAffinityKeyLifetime(Duration.ofMinutes(10))
+ .setCleanupInterval(Duration.ofMinutes(1));
+ if (staticPool) {
+ poolOptions.disableDynamicScaling();
+ } else {
+ poolOptions.setDynamicScaling(MIN_RPC_PER_CHANNEL, MAX_RPC_PER_CHANNEL, scaleDownInterval);
+ compressBranchSpecificScaleUpTimer(poolOptions);
+ }
+
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegateBuilder)
+ .withApiConfig(spannerAffinityConfig())
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(poolOptions.build())
+ .build())
+ .build();
+
+ // Reach the customer-sized pool before creating the session pool. Each caller owns one
+ // transaction and fans out five RPCs on its transaction affinity handle.
+ runWave(pool, callersExecutor, variant, Collections.emptyList(), seed, callers, 5, 0);
+
+ List sessionNames = createSessionsInBatches(pool);
+
+ if (!staticPool && loadShape == LoadShape.BURSTY) {
+ runScaleCycles(pool, callersExecutor, variant, sessionNames, seed, callers, phaseInterval);
+ } else if (loadShape == LoadShape.CONSTANT) {
+ runConstantLoad(pool, callersExecutor, variant, sessionNames, seed, callers);
+ }
+
+ // Final ramp restores a full active pool. Reset delegate counters so removed/reused channel
+ // history does not manufacture skew in the measured hold period.
+ runWave(pool, callersExecutor, variant, sessionNames, seed + 10_000, callers, 5, 10_000);
+ delegateBuilder.resetMeasurements();
+
+ if (variant == Variant.REF_6120 && loadShape == LoadShape.BURSTY) {
+ runTransactionsAcrossScaleDown(
+ pool,
+ callersExecutor,
+ delegateBuilder,
+ sessionNames,
+ seed + 15_000,
+ callers,
+ phaseInterval);
+ }
+
+ int measureWaves = Integer.getInteger("hotchannel.measureWaves", 8);
+ for (int wave = 0; wave < measureWaves; wave++) {
+ runWave(
+ pool,
+ callersExecutor,
+ variant,
+ sessionNames,
+ seed + 20_000,
+ callers,
+ 5,
+ 20_000 + wave * callers);
+ }
+
+ Set activeIds = new HashSet<>();
+ for (ChannelRef channelRef : pool.channelRefs) {
+ activeIds.add(channelRef.getId());
+ }
+ return delegateBuilder.snapshot(
+ seed, variant, loadShape, pool.getNumberOfChannels(), activeIds);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ callersExecutor.shutdownNow();
+ responses.shutdownNow();
+ callersExecutor.awaitTermination(10, TimeUnit.SECONDS);
+ responses.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ }
+
+ private static void compressBranchSpecificScaleUpTimer(GcpChannelPoolOptions.Builder builder) {
+ builder.setScaleUpCooldown(Duration.ofNanos(1));
+ }
+
+ private static void runTransactionsAcrossScaleDown(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ RecordingChannelBuilder delegateBuilder,
+ List sessions,
+ long seed,
+ int callers,
+ Duration scaleDownInterval)
+ throws Exception {
+ List transactions = new ArrayList<>(callers);
+ for (int transaction = 0; transaction < callers; transaction++) {
+ String session = sessions.get(Math.floorMod(mix(seed ^ transaction), sessions.size()));
+ transactions.add(new TransactionContext(transaction, session, new ChannelAffinityRef()));
+ }
+
+ CountDownLatch firstStart = new CountDownLatch(1);
+ List>> firstRpcStarts = new ArrayList<>(callers);
+ for (TransactionContext transaction : transactions) {
+ firstRpcStarts.add(
+ executor.submit(
+ () -> {
+ firstStart.await();
+ CallOptions options =
+ CallOptions.DEFAULT.withOption(
+ CHANNEL_AFFINITY_REF_KEY, transaction.affinityRef);
+ if (transaction.id < 20) {
+ options = options.withOption(TEST_LATENCY_MILLIS, 900L);
+ }
+ return executeSql(pool, options, transaction.session);
+ }));
+ }
+ firstStart.countDown();
+ List> firstRpcs = new ArrayList<>(callers);
+ for (Future> start : firstRpcStarts) {
+ firstRpcs.add(start.get(10, TimeUnit.SECONDS));
+ }
+ for (int transaction = 20; transaction < callers; transaction++) {
+ firstRpcs.get(transaction).get(10, TimeUnit.SECONDS);
+ }
+
+ // Twenty slow streams remain while a scaled 120 ms interval stands in for Spanner's three
+ // minutes. Mainline removes almost the entire pool in one check. Caller-owned refs still point
+ // at those now-inactive channels.
+ await()
+ .atMost(scaleDownInterval.multipliedBy(5).plusSeconds(1))
+ .until(() -> pool.getNumberOfChannels() < MAX_SIZE);
+
+ // Select every remaining call before starting any of them. This exposes the real newCall/start
+ // accounting gap: channel selection happens before SimpleGcpClientCall publishes its stream.
+ Set candidateIds = new HashSet<>();
+ for (ChannelRef channelRef : pool.channelRefs) {
+ candidateIds.add(channelRef.getId());
+ }
+ delegateBuilder.resetSelections();
+ List>>> prepared =
+ new ArrayList<>(callers);
+ for (TransactionContext transaction : transactions) {
+ prepared.add(
+ executor.submit(
+ () -> {
+ int remainingRpcCount = 1 + Math.floorMod(mix(seed + transaction.id * 17L), 4);
+ List> calls =
+ new ArrayList<>(remainingRpcCount);
+ CallOptions options =
+ CallOptions.DEFAULT.withOption(
+ CHANNEL_AFFINITY_REF_KEY, transaction.affinityRef);
+ for (int rpc = 0; rpc < remainingRpcCount; rpc++) {
+ calls.add(pool.newCall(EXECUTE_SQL_METHOD, options));
+ }
+ return calls;
+ }));
+ }
+
+ List>> preparedCalls = new ArrayList<>(callers);
+ for (Future>> calls : prepared) {
+ preparedCalls.add(calls.get(10, TimeUnit.SECONDS));
+ }
+ System.out.println(delegateBuilder.formatSelections(seed, candidateIds));
+ CountDownLatch remainingStart = new CountDownLatch(1);
+ List> remainingTransactions = new ArrayList<>(callers);
+ for (int transaction = 0; transaction < callers; transaction++) {
+ TransactionContext context = transactions.get(transaction);
+ List> calls = preparedCalls.get(transaction);
+ remainingTransactions.add(
+ executor.submit(
+ () -> {
+ remainingStart.await();
+ List> results = new ArrayList<>(calls.size());
+ for (ClientCall call : calls) {
+ results.add(
+ ClientCalls.futureUnaryCall(
+ call,
+ ExecuteSqlRequest.newBuilder()
+ .setSession(context.session)
+ .setSql("SELECT 1")
+ .build()));
+ }
+ for (ListenableFuture result : results) {
+ result.get(10, TimeUnit.SECONDS);
+ }
+ return null;
+ }));
+ }
+ remainingStart.countDown();
+ for (Future transaction : remainingTransactions) {
+ transaction.get(30, TimeUnit.SECONDS);
+ }
+ for (int transaction = 0; transaction < 20; transaction++) {
+ firstRpcs.get(transaction).get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ private static ListenableFuture executeSql(
+ GcpManagedChannel pool, CallOptions callOptions, String session) {
+ return ClientCalls.futureUnaryCall(
+ pool.newCall(EXECUTE_SQL_METHOD, callOptions),
+ ExecuteSqlRequest.newBuilder().setSession(session).setSql("SELECT 1").build());
+ }
+
+ private static void runScaleCycles(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ Variant variant,
+ List sessions,
+ long seed,
+ int callers,
+ Duration interval)
+ throws Exception {
+ int[] ramp = {callers / 8, callers / 3, (callers * 2) / 3, callers};
+ for (int cycle = 0; cycle < 3; cycle++) {
+ for (int waveSize : ramp) {
+ runWave(
+ pool,
+ executor,
+ variant,
+ sessions,
+ seed + cycle * 1_000,
+ waveSize,
+ 0,
+ cycle * 10_000 + waveSize);
+ }
+ for (int hold = 0; hold < 3; hold++) {
+ runWave(
+ pool,
+ executor,
+ variant,
+ sessions,
+ seed + cycle * 1_000,
+ callers,
+ 0,
+ cycle * 10_000 + 1_000 + hold * callers);
+ }
+ int sizeBeforeScaleDown = pool.getNumberOfChannels();
+ await()
+ .atMost(interval.multipliedBy(5).plusSeconds(1))
+ .until(
+ () ->
+ pool.getNumberOfChannels() < sizeBeforeScaleDown
+ || pool.getNumberOfChannels() == MIN_SIZE);
+ }
+ }
+
+ private static void runConstantLoad(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ Variant variant,
+ List sessions,
+ long seed,
+ int callers)
+ throws Exception {
+ for (int wave = 0; wave < 20; wave++) {
+ runWave(pool, executor, variant, sessions, seed + 5_000, callers, 0, 5_000 + wave * callers);
+ }
+ }
+
+ private static List createSessionsInBatches(GcpManagedChannel pool) throws Exception {
+ List sessions = new ArrayList<>();
+ for (int batch = 0; batch < SESSION_COUNT / 10; batch++) {
+ List> futures = new ArrayList<>();
+ for (int item = 0; item < 10; item++) {
+ futures.add(
+ ClientCalls.futureUnaryCall(
+ pool.newCall(CREATE_SESSION_METHOD, CallOptions.DEFAULT),
+ CreateSessionRequest.newBuilder()
+ .setDatabase("projects/p/instances/i/databases/d")
+ .build()));
+ }
+ for (ListenableFuture future : futures) {
+ sessions.add(future.get(10, TimeUnit.SECONDS).getName());
+ }
+ }
+ return sessions;
+ }
+
+ private static void runWave(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ Variant variant,
+ List sessions,
+ long seed,
+ int transactionCount,
+ int fixedRpcCount,
+ int transactionOffset)
+ throws Exception {
+ CountDownLatch start = new CountDownLatch(1);
+ List> transactions = new ArrayList<>(transactionCount);
+ for (int transaction = 0; transaction < transactionCount; transaction++) {
+ final int transactionId = transactionOffset + transaction;
+ transactions.add(
+ executor.submit(
+ () -> {
+ start.await();
+ int rpcCount =
+ fixedRpcCount > 0
+ ? fixedRpcCount
+ : 1 + Math.floorMod(mix(seed + transactionId), 5);
+ ChannelAffinityRef affinityRef = new ChannelAffinityRef();
+ String session =
+ sessions.isEmpty()
+ ? "projects/p/instances/i/databases/d/sessions/bootstrap"
+ : sessions.get(Math.floorMod(mix(seed ^ transactionId), sessions.size()));
+ List> calls = new ArrayList<>(rpcCount);
+ for (int rpc = 0; rpc < rpcCount; rpc++) {
+ CallOptions callOptions = CallOptions.DEFAULT;
+ if (variant == Variant.REF_6120) {
+ callOptions = callOptions.withOption(CHANNEL_AFFINITY_REF_KEY, affinityRef);
+ } else {
+ int route = Math.floorMod(mix(seed + transactionId * 31L), 100);
+ if (route < 9) {
+ callOptions =
+ callOptions.withOption(
+ GcpManagedChannel.AFFINITY_KEY,
+ "projects/p/instances/i/databases/d/sessions/multiplexed");
+ } else if (route >= 29) {
+ callOptions =
+ callOptions.withOption(GcpManagedChannel.DISABLE_AFFINITY_KEY, true);
+ }
+ }
+ calls.add(
+ ClientCalls.futureUnaryCall(
+ pool.newCall(EXECUTE_SQL_METHOD, callOptions),
+ ExecuteSqlRequest.newBuilder()
+ .setSession(session)
+ .setSql("SELECT 1")
+ .build()));
+ }
+ for (ListenableFuture call : calls) {
+ call.get(10, TimeUnit.SECONDS);
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (Future transaction : transactions) {
+ transaction.get(30, TimeUnit.SECONDS);
+ }
+ }
+
+ private static ApiConfig spannerAffinityConfig() {
+ AffinityConfig bind =
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BIND)
+ .setAffinityKey("name")
+ .build();
+ AffinityConfig bound =
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BOUND)
+ .setAffinityKey("session")
+ .build();
+ return ApiConfig.newBuilder()
+ .addMethod(
+ MethodConfig.newBuilder()
+ .addName(CREATE_SESSION_METHOD.getFullMethodName())
+ .setAffinity(bind))
+ .addMethod(
+ MethodConfig.newBuilder()
+ .addName(EXECUTE_SQL_METHOD.getFullMethodName())
+ .setAffinity(bound))
+ .build();
+ }
+
+ private static int mix(long value) {
+ value = (value ^ (value >>> 33)) * 0xff51afd7ed558ccdL;
+ value = (value ^ (value >>> 33)) * 0xc4ceb9fe1a85ec53L;
+ return (int) (value ^ (value >>> 33));
+ }
+
+ private enum Variant {
+ REF_6120,
+ KEY_6117
+ }
+
+ private enum LoadShape {
+ BURSTY,
+ CONSTANT
+ }
+
+ private static final class RecordingChannelBuilder
+ extends ManagedChannelBuilder {
+ private final ScheduledExecutorService responses;
+ private final long seed;
+ private final AtomicInteger nextId = new AtomicInteger();
+ private final AtomicInteger nextSession = new AtomicInteger();
+ private final AtomicLong nextCall = new AtomicLong();
+ private final Map stats = new ConcurrentHashMap<>();
+
+ private RecordingChannelBuilder(ScheduledExecutorService responses, long seed) {
+ this.responses = responses;
+ this.seed = seed;
+ }
+
+ @Override
+ public ManagedChannel build() {
+ int id = nextId.getAndIncrement();
+ ChannelStats channelStats = new ChannelStats(id);
+ stats.put(id, channelStats);
+ return new RecordingManagedChannel(id, channelStats, responses, nextSession, nextCall, seed);
+ }
+
+ private void resetMeasurements() {
+ for (ChannelStats channelStats : stats.values()) {
+ channelStats.completed.set(0);
+ channelStats.maxActive.set(channelStats.active.get());
+ }
+ }
+
+ private void resetSelections() {
+ for (ChannelStats channelStats : stats.values()) {
+ channelStats.selected.set(0);
+ }
+ }
+
+ private String formatSelections(long runSeed, Set candidateIds) {
+ List candidates = new ArrayList<>();
+ for (int id : candidateIds) {
+ candidates.add(stats.get(id));
+ }
+ candidates.sort(Comparator.comparingInt(candidate -> candidate.id));
+ ChannelStats hot =
+ Collections.max(
+ candidates, Comparator.comparingLong(candidate -> candidate.selected.get()));
+ long[] selected =
+ candidates.stream().mapToLong(candidate -> candidate.selected.get()).sorted().toArray();
+ double median =
+ selected.length % 2 == 0
+ ? (selected[selected.length / 2 - 1] + selected[selected.length / 2]) / 2.0
+ : selected[selected.length / 2];
+ StringBuilder distribution = new StringBuilder();
+ for (ChannelStats candidate : candidates) {
+ if (distribution.length() > 0) {
+ distribution.append(',');
+ }
+ distribution.append(candidate.id).append('=').append(candidate.selected.get());
+ }
+ return String.format(
+ Locale.ROOT,
+ "HOTCHANNEL_EARLIEST seed=%d poolAfterDip=%d hotId=%d hotSelected=%d median=%.1f "
+ + "hotToMedian=%.3f preparedDistribution=[%s]",
+ runSeed,
+ candidateIds.size(),
+ hot.id,
+ hot.selected.get(),
+ median,
+ hot.selected.get() / median,
+ distribution);
+ }
+
+ private RunResult snapshot(
+ long runSeed,
+ Variant variant,
+ LoadShape loadShape,
+ int activePoolSize,
+ Set activeIds) {
+ List snapshots = new ArrayList<>();
+ for (int id : activeIds) {
+ ChannelStats channelStats = stats.get(id);
+ snapshots.add(
+ new ChannelStatsSnapshot(
+ id, channelStats.completed.get(), channelStats.maxActive.get()));
+ }
+ snapshots.sort(Comparator.comparingInt(snapshot -> snapshot.id));
+ return new RunResult(runSeed, variant, loadShape, activePoolSize, nextId.get(), snapshots);
+ }
+
+ @Override
+ public RecordingChannelBuilder directExecutor() {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder executor(Executor executor) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder intercept(List interceptors) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder intercept(ClientInterceptor... interceptors) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder userAgent(String userAgent) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder overrideAuthority(String authority) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder nameResolverFactory(Factory resolverFactory) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder decompressorRegistry(DecompressorRegistry registry) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder compressorRegistry(CompressorRegistry registry) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder idleTimeout(long value, TimeUnit unit) {
+ return this;
+ }
+ }
+
+ private static final class RecordingManagedChannel extends ManagedChannel {
+ private final int id;
+ private final ChannelStats stats;
+ private final ScheduledExecutorService responses;
+ private final AtomicInteger nextSession;
+ private final AtomicLong nextCall;
+ private final long seed;
+ private final AtomicBoolean shutdown = new AtomicBoolean();
+
+ private RecordingManagedChannel(
+ int id,
+ ChannelStats stats,
+ ScheduledExecutorService responses,
+ AtomicInteger nextSession,
+ AtomicLong nextCall,
+ long seed) {
+ this.id = id;
+ this.stats = stats;
+ this.responses = responses;
+ this.nextSession = nextSession;
+ this.nextCall = nextCall;
+ this.seed = seed;
+ }
+
+ @Override
+ public ClientCall newCall(
+ MethodDescriptor method, CallOptions callOptions) {
+ stats.selected.incrementAndGet();
+ return new RecordingClientCall<>(
+ id, stats, responses, nextSession, nextCall, seed, method, callOptions);
+ }
+
+ @Override
+ public ConnectivityState getState(boolean requestConnection) {
+ return shutdown.get() ? ConnectivityState.SHUTDOWN : ConnectivityState.READY;
+ }
+
+ @Override
+ public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) {}
+
+ @Override
+ public ManagedChannel shutdown() {
+ shutdown.set(true);
+ return this;
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return shutdown.get();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return shutdown.get();
+ }
+
+ @Override
+ public ManagedChannel shutdownNow() {
+ return shutdown();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return shutdown.get();
+ }
+
+ @Override
+ public String authority() {
+ return "in-process-channel-" + id;
+ }
+
+ @Override
+ public void enterIdle() {}
+ }
+
+ private static final class RecordingClientCall extends ClientCall {
+ private final int channelId;
+ private final ChannelStats stats;
+ private final ScheduledExecutorService responses;
+ private final AtomicInteger nextSession;
+ private final AtomicLong nextCall;
+ private final long seed;
+ private final MethodDescriptor method;
+ private final Long latencyMillisOverride;
+ private final AtomicBoolean completed = new AtomicBoolean();
+ private Listener listener;
+
+ private RecordingClientCall(
+ int channelId,
+ ChannelStats stats,
+ ScheduledExecutorService responses,
+ AtomicInteger nextSession,
+ AtomicLong nextCall,
+ long seed,
+ MethodDescriptor method,
+ CallOptions callOptions) {
+ this.channelId = channelId;
+ this.stats = stats;
+ this.responses = responses;
+ this.nextSession = nextSession;
+ this.nextCall = nextCall;
+ this.seed = seed;
+ this.method = method;
+ this.latencyMillisOverride = callOptions.getOption(TEST_LATENCY_MILLIS);
+ }
+
+ @Override
+ public void start(Listener listener, Metadata headers) {
+ this.listener = listener;
+ int active = stats.active.incrementAndGet();
+ stats.maxActive.accumulateAndGet(active, Math::max);
+ }
+
+ @Override
+ public void request(int numMessages) {}
+
+ @Override
+ public void cancel(String message, Throwable cause) {
+ finish(Status.CANCELLED, null);
+ }
+
+ @Override
+ public void halfClose() {
+ long callId = nextCall.getAndIncrement();
+ long latencyMillis =
+ latencyMillisOverride == null
+ ? 20 + Math.floorMod(mix(seed ^ callId), 11)
+ : latencyMillisOverride;
+ responses.schedule(
+ () -> finish(Status.OK, responseForMethod()), latencyMillis, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void sendMessage(ReqT message) {}
+
+ @Override
+ public boolean isReady() {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ private RespT responseForMethod() {
+ if (method.getFullMethodName().equals(CREATE_SESSION_METHOD.getFullMethodName())) {
+ return (RespT)
+ Session.newBuilder()
+ .setName(
+ "projects/p/instances/i/databases/d/sessions/session-"
+ + nextSession.incrementAndGet())
+ .build();
+ }
+ return (RespT) ResultSet.getDefaultInstance();
+ }
+
+ private void finish(Status status, RespT response) {
+ if (!completed.compareAndSet(false, true)) {
+ return;
+ }
+ if (response != null) {
+ listener.onMessage(response);
+ }
+ listener.onClose(status, new Metadata());
+ if (status.isOk()) {
+ stats.completed.incrementAndGet();
+ }
+ stats.active.decrementAndGet();
+ }
+ }
+
+ private static final class ChannelStats {
+ private final int id;
+ private final AtomicLong completed = new AtomicLong();
+ private final AtomicLong selected = new AtomicLong();
+ private final AtomicInteger active = new AtomicInteger();
+ private final AtomicInteger maxActive = new AtomicInteger();
+
+ private ChannelStats(int id) {
+ this.id = id;
+ }
+ }
+
+ private static final class TransactionContext {
+ private final int id;
+ private final String session;
+ private final ChannelAffinityRef affinityRef;
+
+ private TransactionContext(int id, String session, ChannelAffinityRef affinityRef) {
+ this.id = id;
+ this.session = session;
+ this.affinityRef = affinityRef;
+ }
+ }
+
+ private static final class ChannelStatsSnapshot {
+ private final int id;
+ private final long completed;
+ private final int maxActive;
+
+ private ChannelStatsSnapshot(int id, long completed, int maxActive) {
+ this.id = id;
+ this.completed = completed;
+ this.maxActive = maxActive;
+ }
+ }
+
+ private static final class RunResult {
+ private final long seed;
+ private final Variant variant;
+ private final LoadShape loadShape;
+ private final int activePoolSize;
+ private final int createdChannels;
+ private final List channels;
+ private final ChannelStatsSnapshot hot;
+ private final double median;
+ private final double hotToMedian;
+
+ private RunResult(
+ long seed,
+ Variant variant,
+ LoadShape loadShape,
+ int activePoolSize,
+ int createdChannels,
+ List channels) {
+ this.seed = seed;
+ this.variant = variant;
+ this.loadShape = loadShape;
+ this.activePoolSize = activePoolSize;
+ this.createdChannels = createdChannels;
+ this.channels = channels;
+ this.hot =
+ Collections.max(channels, Comparator.comparingLong(snapshot -> snapshot.completed));
+ long[] counts =
+ channels.stream().mapToLong(snapshot -> snapshot.completed).sorted().toArray();
+ this.median =
+ counts.length % 2 == 0
+ ? (counts[counts.length / 2 - 1] + counts[counts.length / 2]) / 2.0
+ : counts[counts.length / 2];
+ this.hotToMedian = hot.completed / median;
+ }
+
+ private String format() {
+ StringBuilder distribution = new StringBuilder();
+ for (ChannelStatsSnapshot channel : channels) {
+ if (distribution.length() > 0) {
+ distribution.append(',');
+ }
+ distribution
+ .append(channel.id)
+ .append('=')
+ .append(channel.completed)
+ .append('/')
+ .append(channel.maxActive);
+ }
+ return String.format(
+ Locale.ROOT,
+ "HOTCHANNEL_RESULT seed=%d variant=%s load=%s pool=%d created=%d hotId=%d "
+ + "hotOrder=%d hotCalls=%d hotMaxActive=%d median=%.1f hotToMedian=%.3f "
+ + "distribution[id=calls/maxActive]=[%s]",
+ seed,
+ variant,
+ loadShape,
+ activePoolSize,
+ createdChannels,
+ hot.id,
+ hot.id + 1,
+ hot.completed,
+ hot.maxActive,
+ median,
+ hotToMedian,
+ distribution);
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
index 7dc407ff1381..f38c32283727 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
@@ -20,11 +20,14 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions;
+import com.google.common.util.concurrent.Futures;
+import io.grpc.ManagedChannel;
import io.opencensus.metrics.LabelKey;
import io.opencensus.metrics.LabelValue;
import io.opencensus.metrics.MetricRegistry;
@@ -208,6 +211,134 @@ public void testAffinityKeysCleanupZeroByDefault() {
assertThat(channelPoolOptions.getCleanupInterval()).isEqualTo(Duration.ZERO);
}
+ @Test
+ public void testDynamicScalingKnobsHaveDefaultsAndSurviveCopy() {
+ GcpChannelPoolOptions defaults = GcpChannelPoolOptions.newBuilder().build();
+
+ assertThat(defaults.getScaleUpCooldown()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(defaults.getScaleDownConsecutiveLowLoadChecks()).isEqualTo(3);
+ assertThat(defaults.getMaxScaleUpPercent()).isEqualTo(30);
+ assertThat(defaults.getMaxScaleDownChannels()).isEqualTo(2);
+ assertThat(defaults.getDrainIdleGrace()).isEqualTo(Duration.ofMinutes(1));
+ assertThat(defaults.getErrorPenaltyStep()).isEqualTo(5);
+ assertThat(defaults.getErrorPenaltyDuration()).isEqualTo(Duration.ofSeconds(5));
+ assertThat(defaults.getChannelPrimer()).isNull();
+ assertThat(defaults.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(defaults.getChannelPrimeMaxAttempts()).isEqualTo(3);
+
+ GcpChannelPoolOptions zeroValues =
+ GcpChannelPoolOptions.newBuilder()
+ .setScaleUpCooldown(Duration.ZERO)
+ .setErrorPenaltyStep(0)
+ .setChannelPrimeTimeout(Duration.ZERO)
+ .setChannelPrimeMaxAttempts(0)
+ .build();
+ assertThat(zeroValues.getScaleUpCooldown()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(zeroValues.getErrorPenaltyStep()).isEqualTo(5);
+ assertThat(zeroValues.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(zeroValues.getChannelPrimeMaxAttempts()).isEqualTo(3);
+ GcpChannelPoolOptions copiedZeroValues = GcpChannelPoolOptions.newBuilder(zeroValues).build();
+ assertThat(copiedZeroValues.getScaleUpCooldown()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(copiedZeroValues.getErrorPenaltyStep()).isEqualTo(5);
+ assertThat(copiedZeroValues.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(copiedZeroValues.getChannelPrimeMaxAttempts()).isEqualTo(3);
+ assertThat(copiedZeroValues.toString()).contains("scaleUpCooldown: PT10S");
+ assertThat(copiedZeroValues.toString()).contains("errorPenaltyStep: 5");
+ assertThat(copiedZeroValues.toString()).contains("channelPrimeTimeout: PT10S");
+ assertThat(copiedZeroValues.toString()).contains("channelPrimeMaxAttempts: 3");
+
+ GcpChannelPoolOptions configured =
+ GcpChannelPoolOptions.newBuilder(defaults)
+ .setScaleUpCooldown(Duration.ofSeconds(1))
+ .setScaleDownConsecutiveLowLoadChecks(4)
+ .setMaxScaleUpPercent(40)
+ .setMaxScaleDownChannels(3)
+ .setDrainIdleGrace(Duration.ofSeconds(2))
+ .setErrorPenaltyStep(2)
+ .setErrorPenaltyDuration(Duration.ofSeconds(3))
+ .build();
+ GcpChannelPoolOptions copied = GcpChannelPoolOptions.newBuilder(configured).build();
+
+ assertThat(copied.getScaleUpCooldown()).isEqualTo(Duration.ofSeconds(1));
+ assertThat(copied.getScaleDownConsecutiveLowLoadChecks()).isEqualTo(4);
+ assertThat(copied.getMaxScaleUpPercent()).isEqualTo(40);
+ assertThat(copied.getMaxScaleDownChannels()).isEqualTo(3);
+ assertThat(copied.getDrainIdleGrace()).isEqualTo(Duration.ofSeconds(2));
+ assertThat(copied.getErrorPenaltyStep()).isEqualTo(2);
+ assertThat(copied.getErrorPenaltyDuration()).isEqualTo(Duration.ofSeconds(3));
+ }
+
+ @Test
+ public void dynamicScalingDefaultKnobsRejectNegativeValues() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setScaleUpCooldown(Duration.ofNanos(-1)));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setErrorPenaltyStep(-1));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setChannelPrimeTimeout(Duration.ofNanos(-1)));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setChannelPrimeMaxAttempts(-1));
+ }
+
+ @Test
+ public void dynamicScalingAllowsEqualLoadBounds() {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder().setDynamicScaling(10, 10, Duration.ofSeconds(1)).build();
+
+ assertThat(options.getMinRpcPerChannel()).isEqualTo(10);
+ assertThat(options.getMaxRpcPerChannel()).isEqualTo(10);
+ }
+
+ @Test
+ public void channelPoolOptionsToStringIncludesEveryKnob() {
+ String options = GcpChannelPoolOptions.newBuilder().build().toString();
+
+ assertThat(options).contains("maxSize:");
+ assertThat(options).contains("minSize:");
+ assertThat(options).contains("initSize:");
+ assertThat(options).contains("minRpcPerChannel:");
+ assertThat(options).contains("maxRpcPerChannel:");
+ assertThat(options).contains("scaleDownInterval:");
+ assertThat(options).contains("scaleUpCooldown:");
+ assertThat(options).contains("scaleDownConsecutiveLowLoadChecks:");
+ assertThat(options).contains("maxScaleUpPercent:");
+ assertThat(options).contains("maxScaleDownChannels:");
+ assertThat(options).contains("drainIdleGrace:");
+ assertThat(options).contains("errorPenaltyStep:");
+ assertThat(options).contains("errorPenaltyDuration:");
+ assertThat(options).contains("concurrentStreamsLowWatermark:");
+ assertThat(options).contains("useRoundRobinOnBind:");
+ assertThat(options).contains("affinityKeyLifetime:");
+ assertThat(options).contains("cleanupInterval:");
+ assertThat(options).contains("channelPickStrategy:");
+ assertThat(options).contains("channelPrimer:");
+ assertThat(options).contains("channelPrimeTimeout:");
+ assertThat(options).contains("channelPrimeMaxAttempts:");
+ }
+
+ @Test
+ public void channelPrimerOptionsSurviveCopy() {
+ GcpChannelPrimer primer = (ManagedChannel channel) -> Futures.immediateVoidFuture();
+ GcpChannelPoolOptions configured =
+ GcpChannelPoolOptions.newBuilder()
+ .setChannelPrimer(primer)
+ .setChannelPrimeTimeout(Duration.ofSeconds(7))
+ .setChannelPrimeMaxAttempts(2)
+ .build();
+
+ GcpChannelPoolOptions copied = GcpChannelPoolOptions.newBuilder(configured).build();
+
+ assertThat(copied.getChannelPrimer()).isSameInstanceAs(primer);
+ assertThat(copied.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(7));
+ assertThat(copied.getChannelPrimeMaxAttempts()).isEqualTo(2);
+ assertThat(copied.toString()).contains("channelPrimeTimeout: PT7S");
+ assertThat(copied.toString()).contains("channelPrimeMaxAttempts: 2");
+ }
+
@Test
public void testCleanupDefault() {
GcpManagedChannelOptions opts =
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java
index c7eb432ce281..87b8cd33f7f3 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java
@@ -98,5 +98,11 @@ public void emitsOtelMetricsWhenMeterProvided() {
names.stream()
.anyMatch(
n -> n.equals("test/grpc-gcp/" + GcpMetricsConstants.METRIC_MAX_READY_CHANNELS)));
+ assertTrue(
+ names.stream()
+ .anyMatch(
+ n ->
+ n.equals(
+ "test/grpc-gcp/" + GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES)));
}
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
index c765efac7675..7a5dd4ef020a 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
@@ -19,6 +19,7 @@
import static com.google.cloud.grpc.GcpManagedChannel.getKeysFromMessage;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
+import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -54,18 +55,13 @@
import io.opencensus.metrics.LabelValue;
import java.io.File;
import java.io.InputStream;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -79,7 +75,6 @@
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
-import java.util.stream.Collectors;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -280,7 +275,9 @@ public void testGetChannelRefInitializationWithMinSize() throws InterruptedExcep
GcpManagedChannelBuilder.forDelegateBuilder(builder).withOptions(options).build();
// Should have 2 channels since the beginning.
assertThat(gcpChannel.channelRefs.size()).isEqualTo(2);
- TimeUnit.MILLISECONDS.sleep(50);
+ await()
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> gcpChannel.getState(false) != ConnectivityState.IDLE);
// The connection establishment must have been started on these two channels.
assertThat(gcpChannel.getState(false))
.isAnyOf(
@@ -350,7 +347,7 @@ public void testChannelAffinityRefInvalidChannelIdPicksAvailableChannel() throws
gcpChannel = createPoolWithFakeReadyChannels(executorService, 2);
ChannelAffinityRef affinityRef = new ChannelAffinityRef();
- setChannelAffinityRefState(affinityRef, 1000);
+ affinityRef.setChannelIdForTest(999);
ChannelRef selected = gcpChannel.getChannelRefByAffinityRef(affinityRef);
ChannelRef next = gcpChannel.getChannelRefByAffinityRef(affinityRef);
@@ -365,7 +362,7 @@ public void testChannelAffinityRefInvalidChannelIdPicksAvailableChannel() throws
}
@Test
- public void testChannelAffinityRefRemovedChannelPicksAvailableChannel() throws Exception {
+ public void testChannelAffinityRefRemovedOpenChannelStaysStickyUntilShutdown() throws Exception {
resetGcpChannel();
ExecutorService executorService = Executors.newSingleThreadExecutor();
try {
@@ -374,15 +371,19 @@ public void testChannelAffinityRefRemovedChannelPicksAvailableChannel() throws E
ChannelRef removed = gcpChannel.getChannelRefByAffinityRef(affinityRef);
gcpChannel.channelRefs.remove(removed);
- deactivateChannelRef(removed);
+ removed.deactivateForTest();
ChannelRef selected = gcpChannel.getChannelRefByAffinityRef(affinityRef);
- ChannelRef next = gcpChannel.getChannelRefByAffinityRef(affinityRef);
+ assertThat(selected).isSameInstanceAs(removed);
+ assertThat(selected.isActive()).isFalse();
- assertThat(selected).isNotSameInstanceAs(removed);
- assertThat(selected).isIn(gcpChannel.channelRefs);
- assertThat(selected.isActive()).isTrue();
- assertThat(next).isSameInstanceAs(selected);
+ removed.getChannel().shutdownNow();
+ ChannelRef afterShutdown = gcpChannel.getChannelRefByAffinityRef(affinityRef);
+ ChannelRef next = gcpChannel.getChannelRefByAffinityRef(affinityRef);
+ assertThat(afterShutdown).isNotSameInstanceAs(removed);
+ assertThat(afterShutdown).isIn(gcpChannel.channelRefs);
+ assertThat(afterShutdown.isActive()).isTrue();
+ assertThat(next).isSameInstanceAs(afterShutdown);
} finally {
gcpChannel.shutdownNow();
executorService.shutdownNow();
@@ -410,19 +411,6 @@ private GcpManagedChannel createPoolWithFakeReadyChannels(
.build();
}
- private void setChannelAffinityRefState(ChannelAffinityRef affinityRef, int state)
- throws Exception {
- Field stateField = ChannelAffinityRef.class.getDeclaredField("state");
- stateField.setAccessible(true);
- ((AtomicInteger) stateField.get(affinityRef)).set(state);
- }
-
- private void deactivateChannelRef(ChannelRef channelRef) throws Exception {
- Method deactivate = ChannelRef.class.getDeclaredMethod("deactivate");
- deactivate.setAccessible(true);
- deactivate.invoke(channelRef);
- }
-
@Test
public void testGetChannelRefPickUpSmallest() {
// This test verifies deterministic smallest-stream selection (LINEAR_SCAN behavior).
@@ -536,9 +524,8 @@ public void testPickLeastBusyStillPrefersLessBusyChannels() {
busyPicks++;
}
}
- // Power-of-two guarantees distinct indices, so channel 0 (50 streams) is always
- // paired with an idle channel (0 streams) and can never win.
- assertEquals(0, busyPicks);
+ // Sampling is with replacement, so the busy channel wins only when sampled twice.
+ assertThat(busyPicks).isLessThan(10);
}
/**
@@ -549,8 +536,8 @@ public void testPickLeastBusyStillPrefersLessBusyChannels() {
public void testPickLeastBusyWithDynamicScaleUp() throws InterruptedException {
final int minSize = 2;
final int maxSize = 6;
- final int minRpcPerChannel = 2;
- final int maxRpcPerChannel = 5;
+ final int minRpcPerChannel = 5;
+ final int maxRpcPerChannel = 7;
final Duration scaleDownInterval = Duration.ofMillis(50);
final ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -587,6 +574,7 @@ public void testPickLeastBusyWithDynamicScaleUp() throws InterruptedException {
// One more call triggers scale-up.
pool.getChannelRef(null).activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() > minSize);
assertThat(pool.getNumberOfChannels()).isEqualTo(minSize + 1);
// Mark the new channel as READY.
@@ -639,10 +627,7 @@ public void testPickLeastBusySingleChannel() {
}
}
- /**
- * With only 2 channels, power-of-two degenerates to comparing both — should always pick the less
- * busy one.
- */
+ /** With only 2 channels, sampling with replacement strongly prefers the less busy one. */
@Test
public void testPickLeastBusyTwoChannels() {
resetGcpChannel();
@@ -651,11 +636,14 @@ public void testPickLeastBusyTwoChannels() {
gcpChannel.channelRefs.add(gcpChannel.new ChannelRef(ch0, 0, 10));
gcpChannel.channelRefs.add(gcpChannel.new ChannelRef(ch1, 1, 3));
- // With 2 channels, both are always selected, so the one with fewer streams always wins.
+ int lessBusyPicks = 0;
for (int i = 0; i < 100; i++) {
ChannelRef picked = gcpChannel.getChannelRef(null);
- assertThat(picked).isEqualTo(gcpChannel.channelRefs.get(1));
+ if (picked == gcpChannel.channelRefs.get(1)) {
+ lessBusyPicks++;
+ }
}
+ assertThat(lessBusyPicks).isGreaterThan(60);
}
/**
@@ -692,13 +680,9 @@ public void testLinearScanStrategyAlwaysPicksFirstOnTie() {
}
}
- /**
- * Verifies that under low traffic with POWER_OF_TWO, the warm channel (most recently active) is
- * preferred when stream counts are tied. This preserves connection warmth without the thundering
- * herd problem.
- */
+ /** Verifies that POWER_OF_TWO does not add a warmth bias when stream counts are tied. */
@Test
- public void testPowerOfTwoPrefersWarmChannelOnTie() throws Exception {
+ public void testPowerOfTwoDoesNotPreferWarmChannelOnTie() throws Exception {
resetGcpChannel();
// Use a fake clock to deterministically control lastResponseNanos.
final AtomicLong fakeNanos = new AtomicLong(1_000_000_000L);
@@ -716,8 +700,7 @@ public void testPowerOfTwoPrefersWarmChannelOnTie() throws Exception {
ChannelRef warmChannel = gcpChannel.channelRefs.get(5);
warmChannel.messageReceived();
- // Pick many times. The warm channel should be picked more often than average because
- // whenever it appears in a random pair with another 0-stream channel, it wins the tie.
+ // Repeated picks keep the first sample on ties, so warmth does not bias selection.
int warmPicks = 0;
final int numPicks = 1000;
for (int i = 0; i < numPicks; i++) {
@@ -727,11 +710,8 @@ public void testPowerOfTwoPrefersWarmChannelOnTie() throws Exception {
}
}
- // Without warmth bias, channel 5 would get ~10% (100/1000) picks.
- // With warmth bias, it should get significantly more because it wins every tie.
- // P(channel 5 in sample of 2) = 1 - (9/10)*(8/9) -- wait, it's 1-(9/10)^2 = 19%.
- // It wins tie with any other cold channel, so ~19% of picks. Allow some variance.
- assertThat(warmPicks).isGreaterThan(numPicks * 14 / 100);
+ assertThat(warmPicks).isAtLeast(numPicks * 5 / 100);
+ assertThat(warmPicks).isLessThan(numPicks * 15 / 100);
}
private void assertFallbacksMetric(
@@ -833,26 +813,23 @@ public void testGetChannelRefWithFallback() {
assertEquals(2, chRef.getId());
assertEquals(3, pool.getNumberOfChannels());
- // Now we reached max pool size. Let's bring channel 2 to the low watermark and channel 1 to the
- // low watermark + 1 streams.
- for (int i = 0; i < lowWatermark; i++) {
+ // Now we reached max pool size. Bring channel 2 just below the configured watermark and
+ // channel 1 above it.
+ for (int i = 0; i < lowWatermark - 1; i++) {
pool.channelRefs.get(2).activeStreamsCountIncr();
}
pool.channelRefs.get(1).activeStreamsCountIncr();
- // As we reached max size and cannot create new channels and having ready channels with low
- // watermark and low watermark + 1 streams, the best channel for the next channel request with
- // the fallback enabled is the channel 2 with low watermark streams because it's the least busy
- // ready channel.
+ // Channel 2 remains eligible because its load is below the configured watermark.
assertEquals(lowWatermark + 1, pool.channelRefs.get(1).getActiveStreamsCount());
- assertEquals(lowWatermark, pool.channelRefs.get(2).getActiveStreamsCount());
+ assertEquals(lowWatermark - 1, pool.channelRefs.get(2).getActiveStreamsCount());
chRef = pool.getChannelRef(null);
assertEquals(2, chRef.getId());
assertEquals(3, pool.getNumberOfChannels());
- // This was the third fallback from non-ready channel 0 to the channel 2.
- assertFallbacksMetric(fakeRegistry, 3, 0);
+ // Both creating and subsequently selecting channel 2 are successful fallbacks.
+ assertFallbacksMetric(fakeRegistry, 4, 0);
// Let's bring channel 1 to max streams and mark channel 2 as not ready.
- for (int i = 0; i < MAX_STREAM - lowWatermark; i++) {
+ for (int i = 0; i < MAX_STREAM - (lowWatermark - 1); i++) {
pool.channelRefs.get(2).activeStreamsCountIncr();
}
pool.processChannelStateChange(1, ConnectivityState.CONNECTING);
@@ -871,7 +848,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Failed to find fallback for channel 0");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 3, 1);
+ assertFallbacksMetric(fakeRegistry, 4, 1);
// Let's have an affinity key and bind it to channel 0.
final String key = "ABC";
@@ -886,9 +863,12 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Failed to find fallback for channel 0");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 3, 2);
+ assertFallbacksMetric(fakeRegistry, 4, 2);
- // Let's return channel 1 to a ready state.
+ // Return channel 1 below the configured watermark and to a ready state.
+ while (pool.channelRefs.get(1).getActiveStreamsCount() >= lowWatermark) {
+ pool.channelRefs.get(1).activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
+ }
pool.processChannelStateChange(1, ConnectivityState.READY);
logCount = logRecords.size();
// Now we have a fallback candidate.
@@ -898,7 +878,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Setting fallback channel: 0 -> 1");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 4, 2);
+ assertFallbacksMetric(fakeRegistry, 5, 2);
// Let's briefly bring channel 2 to ready state.
pool.processChannelStateChange(2, ConnectivityState.READY);
@@ -912,7 +892,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Using fallback channel: 0 -> 1");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 5, 2);
+ assertFallbacksMetric(fakeRegistry, 6, 2);
pool.processChannelStateChange(2, ConnectivityState.CONNECTING);
// Let's bring channel 1 back to connecting state.
@@ -926,7 +906,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Failed to find fallback for channel 0");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 5, 3);
+ assertFallbacksMetric(fakeRegistry, 6, 3);
// Finally, we bring both channel 1 and channel 0 to the ready state and we should get the
// original channel 0 for the key without any fallbacks happening.
@@ -936,7 +916,7 @@ public void testGetChannelRefWithFallback() {
chRef = pool.getChannelRef(key);
assertEquals(0, chRef.getId());
assertThat(logRecords.size()).isEqualTo(logCount);
- assertFallbacksMetric(fakeRegistry, 5, 3);
+ assertFallbacksMetric(fakeRegistry, 6, 3);
}
@Test
@@ -1027,15 +1007,15 @@ public void testUsingKeyWithoutBinding() {
final String key = "non-binded-key";
ChannelRef channelRef = gcpChannel.getChannelRef(key);
- // Should bind on the fly to the least busy channel, which is 2.
- assertThat(channelRef.getId()).isEqualTo(2);
+ // Power-of-two binds on the fly to its sampled winner.
+ int boundChannelId = channelRef.getId();
+ assertThat(gcpChannel.affinityKeyToChannelRef.get(key)).isSameInstanceAs(channelRef);
cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true);
cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true);
channelRef = gcpChannel.getChannelRef(key);
- // Even after channel 1 now has less active streams (3) the channel 2 is still mapped for the
- // same key.
- assertThat(channelRef.getId()).isEqualTo(2);
+ // A load change does not move the existing binding.
+ assertThat(channelRef.getId()).isEqualTo(boundChannelId);
}
@Test
@@ -1210,7 +1190,9 @@ public void testMetrics() {
}
MetricsRecord record = fakeRegistry.pollRecord();
- assertThat(record.getMetrics().size()).isEqualTo(28);
+ assertThat(record.getMetrics().size()).isEqualTo(29);
+ assertThat(record.getMetrics())
+ .containsKey(prefix + GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES);
// Initial log messages count.
int logCount = logRecords.size();
@@ -1341,6 +1323,8 @@ public void testLogMetrics() throws Exception {
.build())
.build())
.build();
+ AtomicLong nanoClock = new AtomicLong(System.nanoTime());
+ pool.setNanoClock(nanoClock::get);
try {
final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
@@ -1352,7 +1336,7 @@ public void testLogMetrics() throws Exception {
// Simulate channel connecting.
channels.get(i).setState(ConnectivityState.CONNECTING);
waitForStateCallbacks(executorService);
- TimeUnit.MILLISECONDS.sleep(10);
+ nanoClock.addAndGet(Duration.ofMillis(10).toNanos());
// For the last one...
if (i == streams.length - 1) {
@@ -1364,12 +1348,12 @@ public void testLogMetrics() throws Exception {
channels.get(j).setState(ConnectivityState.CONNECTING);
}
waitForStateCallbacks(executorService);
- TimeUnit.MILLISECONDS.sleep(100);
+ nanoClock.addAndGet(Duration.ofMillis(110).toNanos());
// And this will be a failed fallback (no ready channels).
pool.getChannelRef(null);
// Simulate unresponsive connection.
- long startNanos = System.nanoTime();
+ long startNanos = nanoClock.get();
final Status deStatus = Status.fromCode(Code.DEADLINE_EXCEEDED);
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
@@ -1377,12 +1361,12 @@ public void testLogMetrics() throws Exception {
ref.activeStreamsCountDecr(startNanos, deStatus, false);
// Simulate unresponsive connection with more dropped calls.
- startNanos = System.nanoTime();
+ startNanos = nanoClock.get();
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
- TimeUnit.MILLISECONDS.sleep(110);
+ nanoClock.addAndGet(Duration.ofMillis(110).toNanos());
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
}
@@ -1494,8 +1478,9 @@ public void testLogMetrics() throws Exception {
assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_dropped_calls = 3");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_up = 0");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_down = 0");
+ assertThat(messages).contains(poolIndex + ": stat: scale_up_prime_failures = 0");
- assertThat(logRecords.size()).isEqualTo(39);
+ assertThat(logRecords.size()).isEqualTo(40);
logRecords.forEach(
logRecord ->
assertWithMessage(logRecord.getMessage())
@@ -1549,8 +1534,9 @@ public void testLogMetrics() throws Exception {
assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_dropped_calls = 0");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_up = 0");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_down = 0");
+ assertThat(messages).contains(poolIndex + ": stat: scale_up_prime_failures = 0");
- assertThat(logRecords.size()).isEqualTo(39);
+ assertThat(logRecords.size()).isEqualTo(40);
} finally {
pool.shutdownNow();
@@ -1581,6 +1567,8 @@ public void testUnresponsiveDetection() throws InterruptedException {
GcpMetricsOptions.newBuilder().withMetricRegistry(fakeRegistry).build())
.build())
.build();
+ AtomicLong nanoClock = new AtomicLong(System.nanoTime());
+ pool.setNanoClock(nanoClock::get);
int currentIndex = GcpManagedChannel.channelPoolIndex.get();
String poolIndex = String.format("pool-%d", currentIndex);
final AtomicInteger idleCounter = new AtomicInteger();
@@ -1588,10 +1576,10 @@ public void testUnresponsiveDetection() throws InterruptedException {
ChannelRef chRef = pool.new ChannelRef(channel);
assertEquals(0, idleCounter.get());
- TimeUnit.MILLISECONDS.sleep(105);
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
// Report 3 deadline exceeded errors after 100 ms.
- long startNanos = System.nanoTime();
+ long startNanos = nanoClock.get();
final Status deStatus = Status.fromCode(Code.DEADLINE_EXCEEDED);
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(0, idleCounter.get());
@@ -1667,34 +1655,36 @@ public void testUnresponsiveDetection() throws InterruptedException {
+ " = 1\\d\\d");
// Any message from the server must reset the dropped requests count and timestamp.
- TimeUnit.MILLISECONDS.sleep(105);
- startNanos = System.nanoTime();
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
+ startNanos = nanoClock.get();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
// A message received from the server.
+ nanoClock.incrementAndGet();
chRef.messageReceived();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
// No idle increment expected because dropped requests count and timestamp were reset.
assertEquals(1, idleCounter.get());
// Any non-deadline exceeded response must reset the dropped requests count and timestamp.
- TimeUnit.MILLISECONDS.sleep(105);
- startNanos = System.nanoTime();
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
+ startNanos = nanoClock.get();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
// Response with UNAVAILABLE status received from the server.
final Status unavailableStatus = Status.fromCode(Code.UNAVAILABLE);
+ nanoClock.incrementAndGet();
chRef.activeStreamsCountDecr(startNanos, unavailableStatus, false);
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
// No idle increment expected because dropped requests count and timestamp were reset.
assertEquals(1, idleCounter.get());
// Even if dropped requests count is reached, it must also respect 100 ms configured.
- startNanos = System.nanoTime();
+ startNanos = nanoClock.get();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
@@ -1703,7 +1693,7 @@ public void testUnresponsiveDetection() throws InterruptedException {
// Even it's third deadline exceeded no idle increment is expected because 100ms has not pass.
assertEquals(1, idleCounter.get());
- TimeUnit.MILLISECONDS.sleep(105);
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
// Any subsequent deadline exceeded after 100ms must trigger the reconnection.
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(2, idleCounter.get());
@@ -1751,9 +1741,7 @@ public void testStateNotifications() throws InterruptedException {
gcpChannel.notifyWhenStateChanged(
ConnectivityState.SHUTDOWN, () -> immediateCallbackCalled.set(true));
- TimeUnit.MILLISECONDS.sleep(2);
-
- assertThat(immediateCallbackCalled.get()).isTrue();
+ await().atMost(Duration.ofSeconds(1)).untilTrue(immediateCallbackCalled);
// Subscribe for notification when leaving IDLE state.
final AtomicReference newState = new AtomicReference<>();
@@ -1777,10 +1765,12 @@ public void run() {
// Make sure it was IDLE;
assertThat(currentState).isEqualTo(ConnectivityState.IDLE);
- TimeUnit.MILLISECONDS.sleep(25);
-
- assertThat(newState.get())
- .isAnyOf(ConnectivityState.CONNECTING, ConnectivityState.TRANSIENT_FAILURE);
+ await()
+ .atMost(Duration.ofSeconds(1))
+ .untilAsserted(
+ () ->
+ assertThat(newState.get())
+ .isAnyOf(ConnectivityState.CONNECTING, ConnectivityState.TRANSIENT_FAILURE));
}
@Test
@@ -1935,6 +1925,8 @@ public void testAffinityKeysCleanup() throws InterruptedException {
.build())
.build())
.build();
+ AtomicLong nanoClock = new AtomicLong(System.nanoTime());
+ pool.setNanoClock(nanoClock::get);
final String liveKey = "live-key";
ChannelRef ch0 = pool.getChannelRef(liveKey);
@@ -1962,15 +1954,18 @@ public void testAffinityKeysCleanup() throws InterruptedException {
assertThat(pool.getChannelRef(expKey).getId()).isEqualTo(2);
// Halfway through affinity lifetime we use the live key again.
- TimeUnit.MILLISECONDS.sleep(100);
+ nanoClock.addAndGet(Duration.ofMillis(100).toNanos());
ch0 = pool.getChannelRef(liveKey);
// Make sure affinity still works.
assertThat(ch0.getId()).isEqualTo(0);
// Wait the remaining time and check that there is still affinity for the live key
// but no affinity for the expired key.
+ nanoClock.addAndGet(Duration.ofMillis(150).toNanos());
- TimeUnit.MILLISECONDS.sleep(150);
+ await()
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> !pool.affinityKeyToChannelRef.containsKey(expKey));
assertThat(pool.affinityKeyToChannelRef.keySet().size()).isEqualTo(1);
assertThat(pool.affinityKeyToChannelRef.get(liveKey)).isEqualTo(ch0);
@@ -1987,443 +1982,88 @@ public void testAffinityKeysCleanup() throws InterruptedException {
@Test
public void testDynamicChannelPool() throws InterruptedException {
-
- final int minSize = 2;
- final int maxSize = 4;
- final int minRpcPerChannel = 2;
- final int maxRpcPerChannel = 5;
- final Duration scaleDownInterval = Duration.ofMillis(50);
- // Must catch 2 check scale down invocations + some time to avoid race with channel movement.
- final long intervalWaitMs = 2 * scaleDownInterval.toMillis() + 10;
- final ExecutorService executorService = Executors.newSingleThreadExecutor();
-
- FakeManagedChannelBuilder fmcb =
- new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService));
-
- // Creating a pool with dynamic sizing and LINEAR_SCAN for deterministic assertions.
- final GcpManagedChannel pool =
- (GcpManagedChannel)
- GcpManagedChannelBuilder.forDelegateBuilder(fmcb)
- .withOptions(
- GcpManagedChannelOptions.newBuilder()
- .withChannelPoolOptions(
- GcpChannelPoolOptions.newBuilder()
- .setMinSize(minSize)
- .setMaxSize(maxSize)
- .setDynamicScaling(
- minRpcPerChannel, maxRpcPerChannel, scaleDownInterval)
- .setChannelPickStrategy(
- GcpManagedChannelOptions.ChannelPickStrategy.LINEAR_SCAN)
- .build())
- .build())
- .build();
-
- // Starts with minSize.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Mark connected in random order.
- List shuffled = new ArrayList<>(pool.channelRefs);
- Collections.shuffle(shuffled);
- for (ChannelRef channelRef : shuffled) {
- ((FakeManagedChannel) channelRef.getChannel()).setState(ConnectivityState.READY);
- }
-
- long startTime = System.nanoTime();
-
- // Simulate starting 10 calls which should be within the limit (2 channels x 5
- // maxRpcPerChannel).
- for (int i = 0; i < minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
-
- // As we are still within threshold of maxRpcPerChannel the pool must not scale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Adding 11th call should trigger scaling up immediately.
- pool.getChannelRef(null).activeStreamsCountIncr();
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize + 1);
-
- // Mark newly created channel connected.
- ((FakeManagedChannel) pool.channelRefs.get(minSize).getChannel())
- .setState(ConnectivityState.READY);
-
- // Continue adding calls to verify the pool respects the maxSize value.
- for (int i = 0; i < maxSize * maxRpcPerChannel - minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
-
- // Now we have 21 calls in-flight which should bring us to 5 channels because
- // of maxRpcPerChannel is 5, but the max size of the pool is 4, so there should be 4 channels.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Threshold for scaling down is minRpcPerChannel * number of channels. 2 * 3 in our case.
- // Going down 21 -> 7.
- for (ChannelRef channelRef : pool.channelRefs) {
- for (int i = 0; i < maxRpcPerChannel - minRpcPerChannel; i++) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- }
- }
- for (int i = 0; i < minRpcPerChannel; i++) {
- pool.channelRefs.get(i).activeStreamsCountDecr(startTime, Status.OK, false);
- }
-
- // Should not downscale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Should not downscale even after scale down check is passed.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Set all except last channel ready.
- for (int i = 0; i < pool.getNumberOfChannels(); i++) {
- if (i == pool.getNumberOfChannels() - 1) {
- continue;
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(2)
+ .setMaxSize(4)
+ .setDynamicScaling(2, 5, Duration.ofMillis(20))
+ .setScaleUpCooldown(Duration.ofNanos(1))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setDrainIdleGrace(Duration.ZERO)
+ .build())
+ .build())
+ .build();
+ GcpManagedChannel monitoredPool = pool;
+
+ ChannelRef hot = pool.channelRefs.get(0);
+ for (int i = 0; i < 7; i++) {
+ hot.activeStreamsCountIncr();
}
- ((FakeManagedChannel) pool.channelRefs.get(i).getChannel()).setState(ConnectivityState.READY);
- }
-
- // Remember not connected channel or oldest connected channel. In our case the last one (not
- // connected yet).
- final ChannelRef disconnectedRef =
- pool.channelRefs.stream()
- .min(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .get();
-
- // Removing one more stream should trigger scale down after the interval.
- pool.channelRefs.get(0).activeStreamsCountDecr(startTime, Status.OK, false);
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure the oldest connected channel is removed.
- assertThat(pool.channelRefs.stream().anyMatch((chRef) -> (chRef == disconnectedRef))).isFalse();
-
- Set prevChannels = new HashSet<>(pool.channelRefs);
-
- // Scale up again to make sure not connected channels are not reused.
- for (int i = 0; i < 2 * maxRpcPerChannel + disconnectedRef.getActiveStreamsCount(); i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
-
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Find newly created channel.
- ChannelRef newChannel =
- pool.channelRefs.stream().filter(chRef -> !prevChannels.contains(chRef)).findFirst().get();
- // Mark ready.
- ((FakeManagedChannel) newChannel.getChannel()).setState(ConnectivityState.READY);
-
- // Make sure disconnectedRef is not reused.
- assertThat(newChannel == disconnectedRef).isFalse();
+ await().atMost(Duration.ofSeconds(5)).until(() -> monitoredPool.getNumberOfChannels() == 3);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(3);
- // Make sure previously removed channel is not shutted down as it still has a couple of calls.
- assertThat(disconnectedRef.getState()).isNotEqualTo(ConnectivityState.SHUTDOWN);
-
- // Cancel the calls and make sure the channel shutdown.
- while (disconnectedRef.getActiveStreamsCount() > 0) {
- disconnectedRef.activeStreamsCountDecr(startTime, Status.CANCELLED, true);
- }
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(disconnectedRef.getChannel().getState(false)).isEqualTo(ConnectivityState.SHUTDOWN);
-
- // Find the oldest connected channel.
- ChannelRef oldestConnected =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .findFirst()
- .get();
-
- // Scale down. Desired state: minRpcPerChannel on every channel, then closing minRpcPerChannel
- // streams cycling through channels.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() != minRpcPerChannel) {
- if (channelRef.getActiveStreamsCount() > minRpcPerChannel) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- } else {
- channelRef.activeStreamsCountIncr();
- }
+ while (hot.getActiveStreamsCount() > 0) {
+ hot.activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
}
- }
- for (int i = 0; i < minRpcPerChannel; i++) {
- pool.channelRefs.get(i).activeStreamsCountDecr(startTime, Status.OK, false);
- }
- // Remember its streams count.
- int oldestStreamsCount = oldestConnected.getActiveStreamsCount();
-
- // Should scale down after interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure it is removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isFalse();
-
- // The active streams should still be there.
- assertThat(oldestConnected.getActiveStreamsCount()).isEqualTo(oldestStreamsCount);
-
- // The removed oldest connected channel must still be ready.
- assertThat(oldestConnected.getState()).isEqualTo(ConnectivityState.READY);
-
- // Scale up.
- for (int i = 0; i < 2 * maxRpcPerChannel + oldestConnected.getActiveStreamsCount(); i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Make sure it is reused.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isTrue();
-
- // Remember maxSize-minSize oldest connected channels.
- List oldestConnectedChannels =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .collect(Collectors.toList())
- .subList(0, maxSize - minSize);
-
- // Remove all streams so that channel pool downscales to minSize.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() > 0) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
+ await().atMost(Duration.ofSeconds(5)).until(() -> monitoredPool.getNumberOfChannels() == 2);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
}
+ executorService.shutdownNow();
}
-
- // Make sure channel pool scaled down to minSize after the interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Make sure the oldest connected channels were removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> oldestConnectedChannels.contains(chRef)))
- .isFalse();
-
- // Make sure the removed channels are shutted down.
- assertThat(
- oldestConnectedChannels.stream()
- .allMatch(chRef -> chRef.getState() == ConnectivityState.SHUTDOWN))
- .isTrue();
-
- pool.shutdown();
}
@Test
public void testDynamicChannelPoolWithAffinity() throws InterruptedException {
- final String keyFormat = "abc-%d";
- final int minSize = 2;
- final int maxSize = 4;
- final int minRpcPerChannel = 2;
- final int maxRpcPerChannel = 5;
- final Duration scaleDownInterval = Duration.ofMillis(50);
- // Must catch 2 check scale down invocations + some time to avoid race with channel movement.
- final long intervalWaitMs = 2 * scaleDownInterval.toMillis() + 10;
- final ExecutorService executorService = Executors.newSingleThreadExecutor();
-
- FakeManagedChannelBuilder fmcb =
- new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService));
-
- // Creating a pool with dynamic sizing and LINEAR_SCAN for deterministic assertions.
- final GcpManagedChannel pool =
- (GcpManagedChannel)
- GcpManagedChannelBuilder.forDelegateBuilder(fmcb)
- .withOptions(
- GcpManagedChannelOptions.newBuilder()
- .withChannelPoolOptions(
- GcpChannelPoolOptions.newBuilder()
- .setMinSize(minSize)
- .setMaxSize(maxSize)
- .setDynamicScaling(
- minRpcPerChannel, maxRpcPerChannel, scaleDownInterval)
- .setChannelPickStrategy(
- GcpManagedChannelOptions.ChannelPickStrategy.LINEAR_SCAN)
- .build())
- .build())
- .build();
-
- // Starts with minSize.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Mark connected in random order.
- List shuffled = new ArrayList<>(pool.channelRefs);
- Collections.shuffle(shuffled);
- for (ChannelRef channelRef : shuffled) {
- ((FakeManagedChannel) channelRef.getChannel()).setState(ConnectivityState.READY);
- }
-
- long startTime = System.nanoTime();
- int keyIndex = 0;
-
- // Simulate starting 10 calls which should be within the limit (2 channels x 5
- // maxRpcPerChannel).
- for (int i = 0; i < minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
-
- // As we are still within threshold of maxRpcPerChannel the pool must not scale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Adding 11th call should trigger scaling up immediately.
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize + 1);
-
- // Mark newly created channel connected.
- ((FakeManagedChannel) pool.channelRefs.get(minSize).getChannel())
- .setState(ConnectivityState.READY);
-
- // Continue adding calls to verify the pool respects the maxSize value.
- for (int i = 0; i < maxSize * maxRpcPerChannel - minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
-
- // Now we have 21 calls in-flight which should bring us to 5 channels because
- // of maxRpcPerChannel is 5, but the max size of the pool is 4, so there should be 4 channels.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Threshold for scaling down is minRpcPerChannel * number of channels. 2 * 3 in our case.
- // Going down 21 -> 7.
- int totalStreamCount =
- pool.channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).sum();
- while (totalStreamCount > 7) {
- for (ChannelRef channelRef : pool.channelRefs) {
- if (channelRef.getActiveStreamsCount() > 0 && totalStreamCount > 7) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- totalStreamCount--;
- }
- }
- }
-
- // Should not downscale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Should not downscale even after scale down check is passed.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Remember not connected channel or oldest connected channel. In our case the last one (not
- // connected yet).
- final ChannelRef disconnectedRef =
- pool.channelRefs.stream()
- .min(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .get();
-
- // Removing one more stream should trigger scale down after the interval.
- pool.channelRefs.get(0).activeStreamsCountDecr(startTime, Status.OK, false);
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure the oldest connected channel is removed.
- assertThat(pool.channelRefs.stream().anyMatch((chRef) -> (chRef == disconnectedRef))).isFalse();
-
- Set prevChannels = new HashSet<>(pool.channelRefs);
-
- // Scale up again to make sure not connected channels are not reused.
- for (int i = 0; i < 2 * maxRpcPerChannel + disconnectedRef.getActiveStreamsCount(); i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
-
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Find newly created channel.
- ChannelRef newChannel =
- pool.channelRefs.stream().filter(chRef -> !prevChannels.contains(chRef)).findFirst().get();
- // Mark ready.
- ((FakeManagedChannel) newChannel.getChannel()).setState(ConnectivityState.READY);
-
- // Make sure disconnectedRef is not reused.
- assertThat(newChannel == disconnectedRef).isFalse();
-
- // Make sure previously removed channel is not shutted down as it still has a couple of calls.
- assertThat(disconnectedRef.getState()).isNotEqualTo(ConnectivityState.SHUTDOWN);
-
- // Cancel the calls and make sure the channel shutdown.
- while (disconnectedRef.getActiveStreamsCount() > 0) {
- disconnectedRef.activeStreamsCountDecr(startTime, Status.CANCELLED, true);
- }
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(disconnectedRef.getChannel().getState(false)).isEqualTo(ConnectivityState.SHUTDOWN);
-
- // Find the oldest connected channel.
- ChannelRef oldestConnected =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .findFirst()
- .get();
-
- // Scale down. Desired state: minRpcPerChannel on every channel, then closing minRpcPerChannel
- // streams cycling through channels.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() != minRpcPerChannel) {
- if (channelRef.getActiveStreamsCount() > minRpcPerChannel) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- } else {
- channelRef.activeStreamsCountIncr();
- }
- }
- }
- for (int i = 0; i < minRpcPerChannel; i++) {
- pool.channelRefs.get(i).activeStreamsCountDecr(startTime, Status.OK, false);
- }
- // Remember its streams count.
- int oldestStreamsCount = oldestConnected.getActiveStreamsCount();
-
- // Should scale down after interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure it is removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isFalse();
-
- // The active streams should still be there.
- assertThat(oldestConnected.getActiveStreamsCount()).isEqualTo(oldestStreamsCount);
-
- // The removed oldest connected channel must still be ready.
- assertThat(oldestConnected.getState()).isEqualTo(ConnectivityState.READY);
-
- // Scale up.
- for (int i = 0; i < 2 * maxRpcPerChannel + oldestConnected.getActiveStreamsCount(); i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Make sure it is reused.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isTrue();
-
- // Remember maxSize-minSize oldest connected channels.
- List oldestConnectedChannels =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .collect(Collectors.toList())
- .subList(0, maxSize - minSize);
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(1)
+ .setMaxSize(2)
+ .setDynamicScaling(1, 3, Duration.ofMillis(20))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setDrainIdleGrace(Duration.ofSeconds(5))
+ .build())
+ .build())
+ .build();
+ GcpManagedChannel monitoredPool = pool;
+
+ ChannelRef victim = pool.channelRefs.get(0);
+ pool.bind(victim, Collections.singletonList("session"));
+ pool.channelRefs.get(1).activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> monitoredPool.getNumberOfChannels() == 1);
- // Remove all streams so that channel pool downscales to minSize.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() > 0) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(1);
+ assertThat(pool.affinityKeyToChannelRef).doesNotContainKey("session");
+ assertThat(pool.affinityKeyLastUsed).doesNotContainKey("session");
+ assertThat(victim.getAffinityCount()).isEqualTo(0);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
}
+ executorService.shutdownNow();
}
-
- // Make sure channel pool scaled down to minSize after the interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Make sure the oldest connected channels were removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> oldestConnectedChannels.contains(chRef)))
- .isFalse();
-
- // Make sure the removed channels are shutted down.
- assertThat(
- oldestConnectedChannels.stream()
- .allMatch(chRef -> chRef.getState() == ConnectivityState.SHUTDOWN))
- .isTrue();
-
- pool.shutdown();
}
static class FakeManagedChannelBuilder extends ManagedChannelBuilder {
@@ -2563,13 +2203,6 @@ public ManagedChannel shutdownNow() {
@Override
public boolean awaitTermination(long timeout, TimeUnit unit) {
- if (this.state == ConnectivityState.SHUTDOWN) {
- return true;
- }
- try {
- unit.sleep(timeout);
- } catch (InterruptedException e) {
- }
return this.state == ConnectivityState.SHUTDOWN;
}
diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RequestIdMockServerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RequestIdMockServerTest.java
index eac63010915f..dfa1ddc96733 100644
--- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RequestIdMockServerTest.java
+++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RequestIdMockServerTest.java
@@ -680,9 +680,9 @@ public void testOtherClientId() {
ImmutableList.of(
XGoogSpannerRequestId.of(getClientId(), -1, 1, 1),
// The CreateSession RPC from the initialization of the second client is included in
- // the requests that we see. This request does not include a channel hint, hence the
- // zero value for the channel number in the request ID.
- XGoogSpannerRequestId.of(otherClientId, 0, 1, 1),
+ // the requests that we see. Its request ID carries the channel that grpc-gcp selected
+ // for the affinity-key path.
+ XGoogSpannerRequestId.of(otherClientId, -1, 1, 1),
XGoogSpannerRequestId.of(otherClientId, -1, 2, 1),
XGoogSpannerRequestId.of(getClientId(), -1, 2, 1)),
actual);