Skip to content

balancer/pickfirst: fix flaky metrics tests by awaiting async metrics #9020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mbissa wants to merge 6 commits into from
Closed

balancer/pickfirst: fix flaky metrics tests by awaiting async metrics #9020

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7535c06
balancer/pickfirst: fix flaky metrics tests by awaiting async metrics
mbissa Mar 25, 2026
0c93633
balancer/pickfirst: revert pick_first metrics to synchronous checks
mbissa Mar 26, 2026
2848dac
review comments
mbissa Mar 26, 2026
c969a89
review comments
mbissa Apr 29, 2026
7d50135
testutils/stats: store metric slice and support sequence verification…
mbissa Jun 10, 2026
2669e2e
fixing race review comments
mbissa Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 27 additions & 12 deletions balancer/pickfirst/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"google.golang.org/grpc/balancer/pickfirst"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
estats "google.golang.org/grpc/experimental/stats"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/stubserver"
"google.golang.org/grpc/internal/testutils"
Expand Down Expand Up @@ -108,30 +109,22 @@ func (s) TestPickFirstMetrics(t *testing.T) {
}

// Checking for subchannel metrics as well
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 1)
}
waitForInt64Count(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded", 1)
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 0 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 0)
}
if got, _ := tmr.Metric("grpc.subchannel.disconnections"); got != 0 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.disconnections", got, 0)
}
if got, _ := tmr.Metric("grpc.subchannel.open_connections"); got != 1 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.open_connections", got, 1)
}
waitForInt64UpDownCount(ctx, t, tmr, "grpc.subchannel.open_connections", 1)

ss.Stop()
testutils.AwaitState(ctx, t, cc, connectivity.Idle)
if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 1 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 1)
}
if got, _ := tmr.Metric("grpc.subchannel.disconnections"); got != 1 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.disconnections", got, 1)
}
if got, _ := tmr.Metric("grpc.subchannel.open_connections"); got != -1 {
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.open_connections", got, -1)
}
waitForInt64Count(ctx, t, tmr, "grpc.subchannel.disconnections", 1)
waitForInt64UpDownCount(ctx, t, tmr, "grpc.subchannel.open_connections", -1)
}

// TestPickFirstMetricsFailure tests the connection attempts failed metric. It
Expand Down Expand Up @@ -453,3 +446,25 @@ func runDisconnectLabelTest(t *testing.T, wantLabel string, triggerFunc func(*st

t.Fatalf("Error waiting for metrics grpc.subchannel.disconnections: %v", ctx.Err())
}

func waitForInt64Count(ctx context.Context, t *testing.T, tmr *stats.TestMetricsRecorder, name string, want int64) {
t.Helper()
err := tmr.WaitForInt64Count(ctx, stats.MetricsData{
Handle: &estats.MetricDescriptor{Name: name},
IntIncr: want,
})
if err != nil {
t.Fatal(err)
}
}
Comment on lines +450 to +459

@arjan-bal arjan-bal Jun 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style: Assertion helpers are not considered idiomatic in Go, see go/go-style/best-practices#test-functions

Instead, you can should inline the calls, even if it results in a little repetition.


func waitForInt64UpDownCount(ctx context.Context, t *testing.T, tmr *stats.TestMetricsRecorder, name string, want int64) {
t.Helper()
err := tmr.WaitForInt64UpDownCount(ctx, stats.MetricsData{
Handle: &estats.MetricDescriptor{Name: name},
IntIncr: want,
})
if err != nil {
t.Fatal(err)
}
}
Loading
Loading