diff --git a/generator/internal/discovery_type_vertex.cc b/generator/internal/discovery_type_vertex.cc index 9568d6d662138..85be70bde7e65 100644 --- a/generator/internal/discovery_type_vertex.cc +++ b/generator/internal/discovery_type_vertex.cc @@ -17,6 +17,8 @@ #include "google/cloud/internal/algorithm.h" #include "google/cloud/internal/make_status.h" #include "google/cloud/log.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/strings/str_replace.h" @@ -43,6 +45,23 @@ std::optional CheckForScalarType(nlohmann::json const& j) { return std::nullopt; } +bool IsStringOrBytes(nlohmann::json const& field_json) { + std::string const type = field_json.value("type", ""); + if (type == "string" || type == "bytes") return true; + if (type == "array" && field_json.contains("items")) { + std::string const item_type = field_json["items"].value("type", ""); + if (item_type == "string" || item_type == "bytes") return true; + } + return false; +} + +bool ContainsKeyWord(std::string_view s) { + for (auto const& token : absl::StrSplit(s, '_')) { + if (token == "key") return true; + } + return false; +} + } // namespace DiscoveryTypeVertex::DiscoveryTypeVertex( @@ -495,6 +514,10 @@ std::string DiscoveryTypeVertex::FormatFieldOptions( absl::StrCat("\"", field_name, "\"")); } + if (IsStringOrBytes(field_json) && ContainsKeyWord(field_name)) { + field_options.emplace_back("debug_redact", "true"); + } + // Discovery doc defined field names that are not always in strict // camelCase, leading to translation issue between json and protobuf. Thus, // the emitted proto fields need to have their name as it appears in the @@ -512,6 +535,8 @@ std::string DiscoveryTypeVertex::FormatFieldOptions( std::pair const& p) { if (p.first == "json_name") { *s += absl::StrFormat("%s=\"%s\"", p.first, p.second); + } else if (p.first == "debug_redact") { + *s += absl::StrFormat("%s = %s", p.first, p.second); } else { *s += absl::StrFormat("(%s) = %s", p.first, p.second); } diff --git a/generator/internal/discovery_type_vertex_test.cc b/generator/internal/discovery_type_vertex_test.cc index eafbc5ade81bc..5f583c9e29152 100644 --- a/generator/internal/discovery_type_vertex_test.cc +++ b/generator/internal/discovery_type_vertex_test.cc @@ -182,6 +182,114 @@ TEST(DiscoveryTypeVertexTest, FormatFieldOptionsRequiredIsResource) { "REQUIRED,json_name=\"__json_request_body\"]")); } +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactString) { + auto constexpr kFieldJson = R"""( +{ + "type": "string" +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json), + Eq(" [debug_redact = true,json_name=\"rawKey\"]")); +} + +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactBytes) { + auto constexpr kFieldJson = R"""( +{ + "type": "bytes" +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json), + Eq(" [debug_redact = true,json_name=\"rawKey\"]")); +} + +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactArrayString) { + auto constexpr kFieldJson = R"""( +{ + "type": "array", + "items": { + "type": "string" + } +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json), + Eq(" [debug_redact = true,json_name=\"rawKey\"]")); +} + +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactArrayBytes) { + auto constexpr kFieldJson = R"""( +{ + "type": "array", + "items": { + "type": "bytes" + } +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json), + Eq(" [debug_redact = true,json_name=\"rawKey\"]")); +} + +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactNonMatches) { + auto constexpr kFieldJson = R"""( +{ + "type": "string" +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("monkey", "monkey", json), + Eq(" [json_name=\"monkey\"]")); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("keyboard", "keyboard", json), + Eq(" [json_name=\"keyboard\"]")); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("hockey", "hockey", json), + Eq(" [json_name=\"hockey\"]")); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("keypad", "keypad", json), + Eq(" [json_name=\"keypad\"]")); +} + +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactNonStringNotRedacted) { + auto constexpr kFieldJson = R"""( +{ + "type": "integer" +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("key_id", "keyId", json), + Eq(" [json_name=\"keyId\"]")); +} + +TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactRequired) { + auto constexpr kFieldJson = R"""( +{ + "type": "string", + "required": true +} +)"""; + auto json = nlohmann::json::parse(kFieldJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + EXPECT_THAT( + DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json), + Eq(" [(google.api.field_behavior) = " + "REQUIRED,debug_redact = true,json_name=\"rawKey\"]")); +} + struct DetermineTypesSuccess { std::string name; std::string json; @@ -1312,6 +1420,48 @@ message TestSchema { "optional string to: optional double"))); } +TEST_F(DiscoveryTypeVertexDescriptorTest, + JsonToProtobufCustomerEncryptionKey) { + auto constexpr kSchemaJson = R"""( +{ + "id": "CustomerEncryptionKey", + "properties": { + "kmsKeyName": { + "type": "string" + }, + "rawKey": { + "type": "string" + }, + "rsaEncryptedKey": { + "type": "string" + }, + "sha256": { + "type": "string" + } + } +} +)"""; + + auto constexpr kExpectedProto = R"""(message CustomerEncryptionKey { + optional string kms_key_name = 1 [debug_redact = true,json_name="kmsKeyName"]; + + optional string raw_key = 2 [debug_redact = true,json_name="rawKey"]; + + optional string rsa_encrypted_key = 3 [debug_redact = true,json_name="rsaEncryptedKey"]; + + optional string sha256 = 4 [json_name="sha256"]; +} +)"""; + + auto json = nlohmann::json::parse(kSchemaJson, nullptr, false); + ASSERT_TRUE(json.is_object()); + DiscoveryTypeVertex t("CustomerEncryptionKey", "test.package", json, &pool()); + std::map types; + auto result = t.JsonToProtobufMessage(types, "test.package"); + ASSERT_THAT(result, ::google::cloud::testing_util::IsOk()); + EXPECT_THAT(*result, Eq(kExpectedProto)); +} + } // namespace } // namespace generator_internal } // namespace cloud diff --git a/google/cloud/internal/debug_string_protobuf.cc b/google/cloud/internal/debug_string_protobuf.cc index e652d2458bbd5..c65c56df9bcb8 100644 --- a/google/cloud/internal/debug_string_protobuf.cc +++ b/google/cloud/internal/debug_string_protobuf.cc @@ -62,12 +62,22 @@ class TimestampMessagePrinter } }; +template +auto SetRedact(Printer& p, int) + -> decltype(p.SetRedactDebugString(true), void()) { + p.SetRedactDebugString(true); +} + +template +void SetRedact(Printer&, ...) {} + } // namespace std::string DebugString(google::protobuf::Message const& m, TracingOptions const& options) { std::string str; google::protobuf::TextFormat::Printer p; + SetRedact(p, 0); p.SetSingleLineMode(options.single_line_mode()); if (!options.single_line_mode()) p.SetInitialIndentLevel(1); p.SetUseShortRepeatedPrimitives(options.use_short_repeated_primitives()); diff --git a/google/cloud/internal/debug_string_protobuf_test.cc b/google/cloud/internal/debug_string_protobuf_test.cc index bb6bfed365c57..7af21c16c0193 100644 --- a/google/cloud/internal/debug_string_protobuf_test.cc +++ b/google/cloud/internal/debug_string_protobuf_test.cc @@ -18,6 +18,9 @@ #include "google/iam/v1/policy.pb.h" #include "google/protobuf/duration.pb.h" #include "google/protobuf/timestamp.pb.h" +#include +#include +#include #include #include @@ -51,6 +54,7 @@ TEST(LogWrapperHelpers, DefaultOptions) { // clang-format off std::string const text = R"pb(google.iam.v1.Policy { )pb" + R"pb(goo.gle/debugonly )pb" R"pb(bindings { )pb" R"pb(role: "roles/viewer" )pb" R"pb(members: "user:user1@example.com" )pb" @@ -72,6 +76,7 @@ TEST(LogWrapperHelpers, MultiLine) { tracing_options.SetOptions("single_line_mode=off"); // clang-format off std::string const text = R"pb(google.iam.v1.Policy { + goo.gle/debugonly bindings { role: "roles/viewer" members: "user:user1@example.com" @@ -94,6 +99,7 @@ TEST(LogWrapperHelpers, Truncate) { // clang-format off std::string const text = R"pb(google.iam.v1.Policy { )pb" + R"pb(goo.gle/debugonly )pb" R"pb(bindings { )pb" R"pb(role: "roles/vi......" )pb" R"pb(members: "user:use......" )pb" @@ -115,7 +121,7 @@ TEST(LogWrapperHelpers, Duration) { duration.set_seconds((11 * 60 + 22) * 60 + 33); duration.set_nanos(123456789); std::string const expected = - R"(google.protobuf.Duration { "11h22m33.123456789s" })"; + R"(google.protobuf.Duration { goo.gle/debugonly "11h22m33.123456789s" })"; EXPECT_EQ(expected, DebugString(duration, TracingOptions{}.SetOptions( "single_line_mode=on"))); } @@ -125,12 +131,55 @@ TEST(LogWrapperHelpers, Timestamp) { timestamp.set_seconds(1658470436); timestamp.set_nanos(123456789); std::string const expected = R"(google.protobuf.Timestamp { + goo.gle/debugonly "2022-07-22T06:13:56.123456789Z" })"; EXPECT_EQ(expected, DebugString(timestamp, TracingOptions{}.SetOptions( "single_line_mode=off"))); } +TEST(LogWrapperHelpers, RedactedField) { + google::protobuf::FileDescriptorProto file_proto; + file_proto.set_name("test_redact.proto"); + file_proto.set_syntax("proto3"); + google::protobuf::DescriptorProto* message_proto = + file_proto.add_message_type(); + message_proto->set_name("RedactedMessage"); + + google::protobuf::FieldDescriptorProto* unredacted_field = + message_proto->add_field(); + unredacted_field->set_name("public_field"); + unredacted_field->set_number(1); + unredacted_field->set_type(google::protobuf::FieldDescriptorProto::TYPE_STRING); + + google::protobuf::FieldDescriptorProto* redacted_field = + message_proto->add_field(); + redacted_field->set_name("secret_key"); + redacted_field->set_number(2); + redacted_field->set_type(google::protobuf::FieldDescriptorProto::TYPE_STRING); + redacted_field->mutable_options()->set_debug_redact(true); + + google::protobuf::DescriptorPool pool; + google::protobuf::FileDescriptor const* file_desc = + pool.BuildFile(file_proto); + ASSERT_THAT(file_desc, ::testing::NotNull()); + google::protobuf::Descriptor const* msg_desc = + file_desc->FindMessageTypeByName("RedactedMessage"); + ASSERT_THAT(msg_desc, ::testing::NotNull()); + + google::protobuf::DynamicMessageFactory factory; + std::unique_ptr message( + factory.GetPrototype(msg_desc)->New()); + google::protobuf::Reflection const* reflection = message->GetReflection(); + reflection->SetString(message.get(), msg_desc->field(0), "public_value"); + reflection->SetString(message.get(), msg_desc->field(1), "secret_value"); + + TracingOptions tracing_options; + std::string const actual = DebugString(*message, tracing_options); + EXPECT_THAT(actual, ::testing::HasSubstr("public_value")); + EXPECT_THAT(actual, ::testing::Not(::testing::HasSubstr("secret_value"))); +} + } // namespace } // namespace internal GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END