Skip to content

feat(bigquery-jdbc): add opt-out controls #14199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Neenu1995 wants to merge 4 commits into
base: jdbc-telemetry-feature
Choose a base branch
from
Open

feat(bigquery-jdbc): add opt-out controls #14199

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4cca913
feat(telemetry): implement PR 8 opt-out controls, enum converters and…
Neenu1995 Aug 25, 2026
3d5c91e
add statement types
Neenu1995 Aug 27, 2026
4f45388
address comments
Neenu1995 Aug 27, 2026
3de37e2
fix property precedence
Neenu1995 Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 77 additions & 4 deletions ...uery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/telemetry/v1/TelemetryBatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
import com.google.protobuf.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
Expand Down Expand Up @@ -198,17 +200,88 @@ TransportResult flush() {
}

private void populatePayloadBuilder(TelemetryPayload.Builder builder, List<Message> events) {
Map<String, ConnectionAttempt.Builder> connections = new HashMap<>();

@Neenu1995 Neenu1995 Aug 27, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am changing this Map structure in the next PR. Adding that 1 also here makes this a very big PR. So I am stacking the related changes onto the next PR.

Map<String, StatementExecution.Builder> statements = new HashMap<>();
Map<String, ErrorMetric.Builder> errors = new HashMap<>();
Map<String, FeatureUsage.Builder> features = new HashMap<>();

for (Message event : events) {
if (event instanceof ConnectionAttempt) {
builder.addConnectionAttempts((ConnectionAttempt) event);
ConnectionAttempt attempt = (ConnectionAttempt) event;
String key =
attempt.getAuthType().name()
+ "|"
+ attempt.getStatus().name()
+ "|"
+ attempt.getErrorCode();
ConnectionAttempt.Builder b = connections.get(key);
if (b == null) {
connections.put(key, attempt.toBuilder());
} else {
b.setCount(b.getCount() + attempt.getCount());
}
} else if (event instanceof StatementExecution) {
builder.addStatementExecutions((StatementExecution) event);
StatementExecution exec = (StatementExecution) event;
String key =
exec.getStatementType().name()
+ "|"
+ exec.getQueryApiType().name()
+ "|"
+ exec.getStatus().name()
+ "|"
+ exec.getErrorCode();
StatementExecution.Builder b = statements.get(key);
if (b == null) {
statements.put(key, exec.toBuilder());
} else {
b.setCount(b.getCount() + exec.getCount());
if (exec.hasDuration()) {
DurationHistogram.Builder durB = b.getDurationBuilder();
DurationHistogram dur = exec.getDuration();
durB.setCount(durB.getCount() + dur.getCount());
durB.setSum(durB.getSum() + dur.getSum());
for (int i = 0; i < dur.getBucketCountsCount(); i++) {
if (i < durB.getBucketCountsCount()) {
durB.setBucketCounts(i, durB.getBucketCounts(i) + dur.getBucketCounts(i));
} else {
durB.addBucketCounts(dur.getBucketCounts(i));
}
}
}
Comment thread
Neenu1995 marked this conversation as resolved.
}
} else if (event instanceof ErrorMetric) {
builder.addErrors((ErrorMetric) event);
ErrorMetric err = (ErrorMetric) event;
String key = err.getErrorCode() + "|" + err.getErrorXdbcCode() + "|" + err.getMethodName();
ErrorMetric.Builder b = errors.get(key);
if (b == null) {
errors.put(key, err.toBuilder());
} else {
b.setCount(b.getCount() + err.getCount());
}
} else if (event instanceof FeatureUsage) {
builder.addFeatureUsages((FeatureUsage) event);
FeatureUsage feat = (FeatureUsage) event;
String key = feat.getDriverFeature().name() + "|" + feat.getCustomFeatureName();
FeatureUsage.Builder b = features.get(key);
if (b == null) {
features.put(key, feat.toBuilder());
} else {
b.setCount(b.getCount() + feat.getCount());
}
}
}

for (ConnectionAttempt.Builder b : connections.values()) {
builder.addConnectionAttempts(b);
}
for (StatementExecution.Builder b : statements.values()) {
builder.addStatementExecutions(b);
}
for (ErrorMetric.Builder b : errors.values()) {
builder.addErrors(b);
}
for (FeatureUsage.Builder b : features.values()) {
builder.addFeatureUsages(b);
}
}

@Override
Expand Down
58 changes: 58 additions & 0 deletions ...dbc/src/main/java/com/google/cloud/bigquery/jdbc/telemetry/v1/TelemetryConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.google.cloud.bigquery.jdbc.telemetry.v1;

import java.util.Objects;
import java.util.Properties;

/** Configuration settings for the BigQuery JDBC driver telemetry client. */
final class TelemetryConfiguration {
Expand Down Expand Up @@ -156,6 +157,63 @@ Builder setDriverEnvironment(DriverEnvironment driverEnvironment) {
return this;
}

Builder resolveProperties(Properties connectionProperties) {
if (connectionProperties != null) {
String propValue = connectionProperties.getProperty("EnableDiagnosticTelemetry");
if (propValue == null) {
propValue = connectionProperties.getProperty("enableDiagnosticTelemetry");
}
if (propValue != null) {
if ("0".equals(propValue) || "false".equalsIgnoreCase(propValue)) {
this.enabled = false;
} else if ("1".equals(propValue) || "true".equalsIgnoreCase(propValue)) {
this.enabled = true;
}
}

String uploadIntervalStr = connectionProperties.getProperty("TelemetryUploadInterval");
if (uploadIntervalStr != null) {
try {
this.uploadIntervalMs = Long.parseLong(uploadIntervalStr);
} catch (NumberFormatException ignored) {
}
}

String batchSizeStr = connectionProperties.getProperty("TelemetryBatchSize");
if (batchSizeStr != null) {
try {
this.batchSizeThreshold = Integer.parseInt(batchSizeStr);
} catch (NumberFormatException ignored) {
}
}
}

String envValue = System.getenv("GOOGLE_CLOUD_TELEMETRY_ENABLED");
if (envValue != null) {
if ("0".equals(envValue) || "false".equalsIgnoreCase(envValue)) {
this.enabled = false;
}
}

String envInterval = System.getenv("GOOGLE_CLOUD_TELEMETRY_UPLOAD_INTERVAL");
if (envInterval != null) {
try {
this.uploadIntervalMs = Long.parseLong(envInterval);
} catch (NumberFormatException ignored) {
}
}

String envBatch = System.getenv("GOOGLE_CLOUD_TELEMETRY_BATCH_SIZE");
if (envBatch != null) {
try {
this.batchSizeThreshold = Integer.parseInt(envBatch);
} catch (NumberFormatException ignored) {
}
}

return this;
}

TelemetryConfiguration build() {
return new TelemetryConfiguration(this);
}
Expand Down
141 changes: 140 additions & 1 deletion ...uery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/telemetry/v1/TelemetryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@

package com.google.cloud.bigquery.jdbc.telemetry.v1;

import com.google.cloud.bigquery.JobStatistics.QueryStatistics;
import com.google.cloud.bigquery.jdbc.BigQueryJdbcCustomLogger;
import com.google.protobuf.Descriptors.EnumValueDescriptor;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;

Expand All @@ -32,6 +35,7 @@ final class TelemetryManager implements AutoCloseable {
new BigQueryJdbcCustomLogger(TelemetryManager.class.getName());

private static volatile TelemetryManager instance;
private static volatile boolean globallyDisabled = false;

private final TelemetryBatcher batcher;

Expand All @@ -44,12 +48,36 @@ private TelemetryManager(TelemetryBatcher batcher) {
* and transport.
*/
static TelemetryManager getInstance() {
return getInstance(null);
}

static TelemetryManager getInstance(Properties properties) {
if (globallyDisabled) {
return null;
}

if (properties != null) {
TelemetryConfiguration configCheck =
TelemetryConfiguration.builder().resolveProperties(properties).build();
if (!configCheck.isEnabled()) {
synchronized (TelemetryManager.class) {
globallyDisabled = true;
closeInstance();
}
return null;
}
}

TelemetryManager localRef = instance;
if (localRef == null) {
synchronized (TelemetryManager.class) {
if (globallyDisabled) {
return null;
}
localRef = instance;
if (localRef == null) {
TelemetryConfiguration config = TelemetryConfiguration.builder().build();
TelemetryConfiguration config =
TelemetryConfiguration.builder().resolveProperties(properties).build();
ClearcutTransport transport = new ClearcutTransport(config);
TelemetryBatcher batcher = new TelemetryBatcher(config, transport);
localRef = new TelemetryManager(batcher);
Expand Down Expand Up @@ -116,4 +144,115 @@ public void close() {
batcher.close();
}
}

// Package-private test helper to reset the global kill switch between test runs
static synchronized void resetGlobalDisableForTest() {
globallyDisabled = false;
}

static StatementType toStatementType(QueryStatistics.StatementType bqStatementType) {
if (bqStatementType == null) {
return StatementType.STATEMENT_TYPE_UNSPECIFIED;
}

EnumValueDescriptor desc =
StatementType.getDescriptor().findValueByName("STATEMENT_TYPE_" + bqStatementType.name());

return desc != null ? StatementType.valueOf(desc) : StatementType.STATEMENT_TYPE_OTHER;
}
Comment thread
Neenu1995 marked this conversation as resolved.

static AuthenticationType toAuthenticationType(int oauthType) {
switch (oauthType) {
case 0:
return AuthenticationType.AUTHENTICATION_TYPE_SERVICE_ACCOUNT;
case 1:
return AuthenticationType.AUTHENTICATION_TYPE_USER_AUTHENTICATION;
case 2:
return AuthenticationType.AUTHENTICATION_TYPE_APPLICATION_DEFAULT_CREDENTIALS;
case 3:
return AuthenticationType.AUTHENTICATION_TYPE_EXTERNAL;
case 4:
return AuthenticationType.AUTHENTICATION_TYPE_TOKEN;
default:
return AuthenticationType.AUTHENTICATION_TYPE_CUSTOM;
}
}

static final double[] HISTOGRAM_BOUNDS = {
10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0, 10000.0
};

static DurationHistogram toDurationBucketMs(long durationMs) {
DurationHistogram.Builder builder =
DurationHistogram.newBuilder().setCount(1).setSum(durationMs);

int bucketIndex = HISTOGRAM_BOUNDS.length;
for (int i = 0; i < HISTOGRAM_BOUNDS.length; i++) {
builder.addExplicitBounds(HISTOGRAM_BOUNDS[i]);
if (bucketIndex == HISTOGRAM_BOUNDS.length && durationMs < HISTOGRAM_BOUNDS[i]) {
bucketIndex = i;
}
}
for (int i = 0; i <= HISTOGRAM_BOUNDS.length; i++) {
builder.addBucketCounts(i == bucketIndex ? 1L : 0L);
}
return builder.build();
}

static void recordConnectionAttempt(Status status, int errorCode, AuthenticationType authType) {
runSafely(
() -> {
TelemetryManager mgr = instance;
if (mgr != null && mgr.getBatcher() != null) {
mgr.getBatcher()
.offerConnectionAttempt(
ConnectionAttempt.newBuilder()
.setStatus(status)
.setErrorCode(errorCode)
.setAuthType(authType)
.setCount(1)
.build());
}
});
}

static void recordStatementExecution(
StatementType statementType,
QueryApiType apiType,
Status status,
int errorCode,
long durationMs) {
runSafely(
() -> {
TelemetryManager mgr = instance;
if (mgr != null && mgr.getBatcher() != null) {
mgr.getBatcher()
.offerStatementExecution(
StatementExecution.newBuilder()
.setStatementType(statementType)
.setQueryApiType(apiType)
.setStatus(status)
.setErrorCode(errorCode)
.setCount(1)
.setDuration(toDurationBucketMs(durationMs))
.build());
}
});
}

static void recordFeatureUsage(DriverFeature feature, String customFeatureName) {
runSafely(
() -> {
TelemetryManager mgr = instance;
if (mgr != null && mgr.getBatcher() != null) {
mgr.getBatcher()
.offerFeatureUsage(
FeatureUsage.newBuilder()
.setDriverFeature(feature)
.setCustomFeatureName(customFeatureName == null ? "" : customFeatureName)
.setCount(1)
.build());
}
});
}
}
Loading
Loading