Skip to content

feat(generator): add debug_redact to key fields and enable redact in DebugString #16398

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
colinmoy wants to merge 5 commits into
base: main
Choose a base branch
from
Open

feat(generator): add debug_redact to key fields and enable redact in DebugString #16398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2509790
feat(generator): handle Any fields with Value or Struct
colinmoy Aug 25, 2026
214eb62
feat(generator): add debug_redact to key fields and enable redact in …
colinmoy Aug 27, 2026
c7f71dc
Merge branch 'main' into feat-generator-debug-redact-keys
colinmoy Aug 27, 2026
1e723fc
fix(generator): address review feedback on key word-boundary matching…
colinmoy Aug 27, 2026
fb42587
fix(generator): address maintainer review feedback and fix compilatio…
colinmoy Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions generator/internal/discovery_type_vertex.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
#include "google/cloud/internal/algorithm.h"
#include "google/cloud/internal/make_status.h"
#include "google/cloud/log.h"
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_replace.h"
Expand All @@ -43,6 +45,23 @@ std::optional<std::string> CheckForScalarType(nlohmann::json const& j) {
return std::nullopt;
}

bool IsStringOrBytes(nlohmann::json const& field_json) {
std::string const type = field_json.value("type", "");
if (type == "string" || type == "bytes") return true;
if (type == "array" && field_json.contains("items")) {
std::string const item_type = field_json["items"].value("type", "");
if (item_type == "string" || item_type == "bytes") return true;
}
return false;
}

bool ContainsKeyWord(std::string_view s) {

@scotthart scotthart Aug 27, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prefer using string matching functions from absl (e.g. StrContains) and the absl Suffix/Prefix functions instead of std::string::find and iterators directly.

for (auto const& token : absl::StrSplit(s, '_')) {
if (token == "key") return true;
}
return false;
}

} // namespace

DiscoveryTypeVertex::DiscoveryTypeVertex(
Expand Down Expand Up @@ -495,6 +514,10 @@ std::string DiscoveryTypeVertex::FormatFieldOptions(
absl::StrCat("\"", field_name, "\""));
}

if (IsStringOrBytes(field_json) && ContainsKeyWord(field_name)) {
field_options.emplace_back("debug_redact", "true");
}

// Discovery doc defined field names that are not always in strict
// camelCase, leading to translation issue between json and protobuf. Thus,
// the emitted proto fields need to have their name as it appears in the
Expand All @@ -512,6 +535,8 @@ std::string DiscoveryTypeVertex::FormatFieldOptions(
std::pair<std::string, std::string> const& p) {
if (p.first == "json_name") {
*s += absl::StrFormat("%s=\"%s\"", p.first, p.second);
} else if (p.first == "debug_redact") {
*s += absl::StrFormat("%s = %s", p.first, p.second);
} else {
*s += absl::StrFormat("(%s) = %s", p.first, p.second);
}
Expand Down
150 changes: 150 additions & 0 deletions generator/internal/discovery_type_vertex_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,114 @@ TEST(DiscoveryTypeVertexTest, FormatFieldOptionsRequiredIsResource) {
"REQUIRED,json_name=\"__json_request_body\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactString) {
auto constexpr kFieldJson = R"""(
{
"type": "string"
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactBytes) {
auto constexpr kFieldJson = R"""(
{
"type": "bytes"
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactArrayString) {
auto constexpr kFieldJson = R"""(
{
"type": "array",
"items": {
"type": "string"
}
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactArrayBytes) {
auto constexpr kFieldJson = R"""(
{
"type": "array",
"items": {
"type": "bytes"
}
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactNonMatches) {
auto constexpr kFieldJson = R"""(
{
"type": "string"
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("monkey", "monkey", json),
Eq(" [json_name=\"monkey\"]"));
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("keyboard", "keyboard", json),
Eq(" [json_name=\"keyboard\"]"));
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("hockey", "hockey", json),
Eq(" [json_name=\"hockey\"]"));
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("keypad", "keypad", json),
Eq(" [json_name=\"keypad\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactNonStringNotRedacted) {
auto constexpr kFieldJson = R"""(
{
"type": "integer"
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("key_id", "keyId", json),
Eq(" [json_name=\"keyId\"]"));
}

TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactRequired) {
auto constexpr kFieldJson = R"""(
{
"type": "string",
"required": true
}
)""";
auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
ASSERT_TRUE(json.is_object());
EXPECT_THAT(
DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
Eq(" [(google.api.field_behavior) = "
"REQUIRED,debug_redact = true,json_name=\"rawKey\"]"));
}

struct DetermineTypesSuccess {
std::string name;
std::string json;
Expand Down Expand Up @@ -1312,6 +1420,48 @@ message TestSchema {
"optional string to: optional double")));
}

TEST_F(DiscoveryTypeVertexDescriptorTest,
JsonToProtobufCustomerEncryptionKey) {
auto constexpr kSchemaJson = R"""(
{
"id": "CustomerEncryptionKey",
"properties": {
"kmsKeyName": {
"type": "string"
},
"rawKey": {
"type": "string"
},
"rsaEncryptedKey": {
"type": "string"
},
"sha256": {
"type": "string"
}
}
}
)""";

auto constexpr kExpectedProto = R"""(message CustomerEncryptionKey {
optional string kms_key_name = 1 [debug_redact = true,json_name="kmsKeyName"];

optional string raw_key = 2 [debug_redact = true,json_name="rawKey"];

optional string rsa_encrypted_key = 3 [debug_redact = true,json_name="rsaEncryptedKey"];

optional string sha256 = 4 [json_name="sha256"];
}
)""";

auto json = nlohmann::json::parse(kSchemaJson, nullptr, false);
ASSERT_TRUE(json.is_object());
DiscoveryTypeVertex t("CustomerEncryptionKey", "test.package", json, &pool());
std::map<std::string, DiscoveryTypeVertex> types;
auto result = t.JsonToProtobufMessage(types, "test.package");
ASSERT_THAT(result, ::google::cloud::testing_util::IsOk());
EXPECT_THAT(*result, Eq(kExpectedProto));
}

} // namespace
} // namespace generator_internal
} // namespace cloud
Expand Down
10 changes: 10 additions & 0 deletions google/cloud/internal/debug_string_protobuf.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,22 @@ class TimestampMessagePrinter
}
};

template <typename Printer>
auto SetRedact(Printer& p, int)
-> decltype(p.SetRedactDebugString(true), void()) {
p.SetRedactDebugString(true);
}

template <typename Printer>
void SetRedact(Printer&, ...) {}

} // namespace

std::string DebugString(google::protobuf::Message const& m,
TracingOptions const& options) {
std::string str;
google::protobuf::TextFormat::Printer p;
SetRedact(p, 0);
p.SetSingleLineMode(options.single_line_mode());
if (!options.single_line_mode()) p.SetInitialIndentLevel(1);
p.SetUseShortRepeatedPrimitives(options.use_short_repeated_primitives());
Expand Down
51 changes: 50 additions & 1 deletion google/cloud/internal/debug_string_protobuf_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
#include "google/iam/v1/policy.pb.h"
#include "google/protobuf/duration.pb.h"
#include "google/protobuf/timestamp.pb.h"
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/text_format.h>
#include <gmock/gmock.h>

Expand Down Expand Up @@ -51,6 +54,7 @@ TEST(LogWrapperHelpers, DefaultOptions) {
// clang-format off
std::string const text =
R"pb(google.iam.v1.Policy { )pb"
R"pb(goo.gle/debugonly )pb"
R"pb(bindings { )pb"
R"pb(role: "roles/viewer" )pb"
R"pb(members: "user:user1@example.com" )pb"
Expand All @@ -72,6 +76,7 @@ TEST(LogWrapperHelpers, MultiLine) {
tracing_options.SetOptions("single_line_mode=off");
// clang-format off
std::string const text = R"pb(google.iam.v1.Policy {
goo.gle/debugonly
bindings {
role: "roles/viewer"
members: "user:user1@example.com"
Expand All @@ -94,6 +99,7 @@ TEST(LogWrapperHelpers, Truncate) {
// clang-format off
std::string const text =
R"pb(google.iam.v1.Policy { )pb"
R"pb(goo.gle/debugonly )pb"
R"pb(bindings { )pb"
R"pb(role: "roles/vi...<truncated>..." )pb"
R"pb(members: "user:use...<truncated>..." )pb"
Expand All @@ -115,7 +121,7 @@ TEST(LogWrapperHelpers, Duration) {
duration.set_seconds((11 * 60 + 22) * 60 + 33);
duration.set_nanos(123456789);
std::string const expected =
R"(google.protobuf.Duration { "11h22m33.123456789s" })";
R"(google.protobuf.Duration { goo.gle/debugonly "11h22m33.123456789s" })";
EXPECT_EQ(expected, DebugString(duration, TracingOptions{}.SetOptions(
"single_line_mode=on")));
}
Expand All @@ -125,12 +131,55 @@ TEST(LogWrapperHelpers, Timestamp) {
timestamp.set_seconds(1658470436);
timestamp.set_nanos(123456789);
std::string const expected = R"(google.protobuf.Timestamp {
goo.gle/debugonly
"2022-07-22T06:13:56.123456789Z"
})";
EXPECT_EQ(expected, DebugString(timestamp, TracingOptions{}.SetOptions(
"single_line_mode=off")));
}

TEST(LogWrapperHelpers, RedactedField) {
google::protobuf::FileDescriptorProto file_proto;
file_proto.set_name("test_redact.proto");
file_proto.set_syntax("proto3");
google::protobuf::DescriptorProto* message_proto =
file_proto.add_message_type();
message_proto->set_name("RedactedMessage");

google::protobuf::FieldDescriptorProto* unredacted_field =
message_proto->add_field();
unredacted_field->set_name("public_field");
unredacted_field->set_number(1);
unredacted_field->set_type(google::protobuf::FieldDescriptorProto::TYPE_STRING);

google::protobuf::FieldDescriptorProto* redacted_field =
message_proto->add_field();
redacted_field->set_name("secret_key");
redacted_field->set_number(2);
redacted_field->set_type(google::protobuf::FieldDescriptorProto::TYPE_STRING);
redacted_field->mutable_options()->set_debug_redact(true);

google::protobuf::DescriptorPool pool;
google::protobuf::FileDescriptor const* file_desc =
pool.BuildFile(file_proto);
ASSERT_THAT(file_desc, ::testing::NotNull());
google::protobuf::Descriptor const* msg_desc =
file_desc->FindMessageTypeByName("RedactedMessage");
ASSERT_THAT(msg_desc, ::testing::NotNull());

google::protobuf::DynamicMessageFactory factory;
std::unique_ptr<google::protobuf::Message> message(
factory.GetPrototype(msg_desc)->New());
google::protobuf::Reflection const* reflection = message->GetReflection();
reflection->SetString(message.get(), msg_desc->field(0), "public_value");
reflection->SetString(message.get(), msg_desc->field(1), "secret_value");

TracingOptions tracing_options;
std::string const actual = DebugString(*message, tracing_options);
EXPECT_THAT(actual, ::testing::HasSubstr("public_value"));
EXPECT_THAT(actual, ::testing::Not(::testing::HasSubstr("secret_value")));
}

} // namespace
} // namespace internal
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
Expand Down
Loading