feat(generator): add debug_redact to key fields and enable redact in DebugString

generator/internal/discovery_to_proto.cc

@@ -360,7 +360,7 @@ std::set<std::string> FindAllTypesToImport(nlohmann::json const& json) {
       types_to_import.insert((*current)["$ref"]);
     }
 
-    if (current->contains("format")) {
+    if (current->contains("format") && (*current)["format"].is_string()) {
       std::string const format = (*current)["format"];
       if (absl::StartsWith(format, "google.protobuf.")) {
         types_to_import.insert(format);
@@ -383,8 +383,9 @@ std::set<std::string> FindAllTypesToImport(nlohmann::json const& json) {
 
     if (IsDiscoveryArrayType(*current)) {
       auto const& items = (*current)["items"];
-      if (items.contains("type") && items["type"] == "object" &&
-          items.contains("additionalProperties") &&
+      if (items.is_object() && items.contains("type") &&
+          items["type"] == "object" && items.contains("additionalProperties") &&
+          items["additionalProperties"].is_object() &&
           items["additionalProperties"].value("type", "") == "any" &&
           !items.contains("format") &&
           !items["additionalProperties"].contains("format")) {
@@ -396,7 +397,8 @@ std::set<std::string> FindAllTypesToImport(nlohmann::json const& json) {
 
     if (IsDiscoveryMapType(*current)) {
       auto const& additional_properties = (*current)["additionalProperties"];
-      if (additional_properties.contains("type") &&
+      if (additional_properties.is_object() &&
+          additional_properties.contains("type") &&
           additional_properties["type"] == "any" &&
           !additional_properties.contains("format") &&
           !current->contains("format")) {

generator/internal/discovery_type_vertex.cc

@@ -17,6 +17,8 @@
 #include "google/cloud/internal/algorithm.h"
 #include "google/cloud/internal/make_status.h"
 #include "google/cloud/log.h"
+#include "absl/strings/ascii.h"
+#include "absl/strings/match.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/str_replace.h"
@@ -43,6 +45,27 @@ std::optional<std::string> CheckForScalarType(nlohmann::json const& j) {
   return std::nullopt;
 }
 
+bool IsStringOrBytes(nlohmann::json const& field_json) {
+  std::string const type = field_json.value("type", "");
+  if (type == "string" || type == "bytes") return true;
+  if (type == "array" && field_json.contains("items") &&
+      field_json["items"].is_object()) {
+    std::string const item_type = field_json["items"].value("type", "");
+    if (item_type == "string" || item_type == "bytes") return true;
+  }
+  return false;
+}
+
+bool ContainsKeyWord(std::string_view s) {
+  for (std::size_t pos = s.find("key"); pos != std::string_view::npos;
+       pos = s.find("key", pos + 1)) {
+    bool const prefix_ok = (pos == 0 || s[pos - 1] == '_');
+    bool const suffix_ok = (pos + 3 == s.size() || s[pos + 3] == '_');
+    if (prefix_ok && suffix_ok) return true;
+  }
+  return false;
+}
+
 }  // namespace
 
 DiscoveryTypeVertex::DiscoveryTypeVertex(
@@ -110,7 +133,7 @@ DiscoveryTypeVertex::DetermineTypeAndSynthesis(nlohmann::json const& v,
   }
 
   if (type == "any") {
-    if (v.contains("format")) {
+    if (v.contains("format") && v["format"].is_string()) {
       type = v["format"];
     } else {
       type = "google.protobuf.Value";
@@ -151,9 +174,11 @@ DiscoveryTypeVertex::DetermineTypeAndSynthesis(nlohmann::json const& v,
         properties_for_synthesis = &additional_properties;
         is_message = true;
       } else if (map_type == "any") {
-        if (additional_properties.contains("format")) {
+        if (additional_properties.is_object() &&
+            additional_properties.contains("format") &&
+            additional_properties["format"].is_string()) {
           map_type = additional_properties["format"];
-        } else if (v.contains("format")) {
+        } else if (v.contains("format") && v["format"].is_string()) {
           map_type = v["format"];
         } else {
           map_type = "google.protobuf.Struct";
@@ -194,21 +219,26 @@ DiscoveryTypeVertex::DetermineTypeAndSynthesis(nlohmann::json const& v,
       if (scalar_type) {
         type = *scalar_type;
       } else if (type == "any") {
-        if (items.contains("format")) {
+        if (items.is_object() && items.contains("format") &&
+            items["format"].is_string()) {
           type = items["format"];
         } else {
           type = "google.protobuf.Value";
         }
         return TypeInfo{type, compare_package_name, nullptr, false, false};
-      } else if (type == "object" && items.contains("properties")) {
+      } else if (type == "object" && items.is_object() &&
+                 items.contains("properties")) {
         // Synthesize a nested type for this array.
         type = CapitalizeFirstLetter(field_name + "Item");
         return TypeInfo{type, compare_package_name, &items, false, true};
-      } else if (type == "object" && items.contains("additionalProperties") &&
-                 (items["additionalProperties"]).value("type", "") == "any") {
-        if (items.contains("format")) {
+      } else if (type == "object" && items.is_object() &&
+                 items.contains("additionalProperties") &&
+                 items["additionalProperties"].is_object() &&
+                 items["additionalProperties"].value("type", "") == "any") {
+        if (items.contains("format") && items["format"].is_string()) {
           type = items["format"];
-        } else if (items["additionalProperties"].contains("format")) {
+        } else if (items["additionalProperties"].contains("format") &&
+                   items["additionalProperties"]["format"].is_string()) {
           type = items["additionalProperties"]["format"];
         } else {
           type = "google.protobuf.Struct";
@@ -495,6 +525,10 @@ std::string DiscoveryTypeVertex::FormatFieldOptions(
                                absl::StrCat("\"", field_name, "\""));
   }
 
+  if (IsStringOrBytes(field_json) && ContainsKeyWord(field_name)) {
+    field_options.emplace_back("debug_redact", "true");
+  }
+
   // Discovery doc defined field names that are not always in strict
   // camelCase, leading to translation issue between json and protobuf. Thus,
   // the emitted proto fields need to have their name as it appears in the
@@ -512,6 +546,8 @@ std::string DiscoveryTypeVertex::FormatFieldOptions(
                         std::pair<std::string, std::string> const& p) {
       if (p.first == "json_name") {
         *s += absl::StrFormat("%s=\"%s\"", p.first, p.second);
+      } else if (p.first == "debug_redact") {
+        *s += absl::StrFormat("%s = %s", p.first, p.second);
       } else {
         *s += absl::StrFormat("(%s) = %s", p.first, p.second);
       }

generator/internal/discovery_type_vertex_test.cc

@@ -182,6 +182,114 @@ TEST(DiscoveryTypeVertexTest, FormatFieldOptionsRequiredIsResource) {
          "REQUIRED,json_name=\"__json_request_body\"]"));
 }
 
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactString) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "string"
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
+      Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
+}
+
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactBytes) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "bytes"
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
+      Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
+}
+
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactArrayString) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "array",
+  "items": {
+    "type": "string"
+  }
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
+      Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
+}
+
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactArrayBytes) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "array",
+  "items": {
+    "type": "bytes"
+  }
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
+      Eq(" [debug_redact = true,json_name=\"rawKey\"]"));
+}
+
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactNonMatches) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "string"
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("monkey", "monkey", json),
+      Eq(" [json_name=\"monkey\"]"));
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("keyboard", "keyboard", json),
+      Eq(" [json_name=\"keyboard\"]"));
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("hockey", "hockey", json),
+      Eq(" [json_name=\"hockey\"]"));
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("keypad", "keypad", json),
+      Eq(" [json_name=\"keypad\"]"));
+}
+
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactNonStringNotRedacted) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "integer"
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("key_id", "keyId", json),
+      Eq(" [json_name=\"keyId\"]"));
+}
+
+TEST(DiscoveryTypeVertexTest, FormatFieldOptionsDebugRedactRequired) {
+  auto constexpr kFieldJson = R"""(
+{
+  "type": "string",
+  "required": true
+}
+)""";
+  auto json = nlohmann::json::parse(kFieldJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  EXPECT_THAT(
+      DiscoveryTypeVertex::FormatFieldOptions("raw_key", "rawKey", json),
+      Eq(" [(google.api.field_behavior) = "
+         "REQUIRED,debug_redact = true,json_name=\"rawKey\"]"));
+}
+
 struct DetermineTypesSuccess {
   std::string name;
   std::string json;
@@ -1312,6 +1420,48 @@ message TestSchema {
                                  "optional string to: optional double")));
 }
 
+TEST_F(DiscoveryTypeVertexDescriptorTest,
+       JsonToProtobufCustomerEncryptionKey) {
+  auto constexpr kSchemaJson = R"""(
+{
+  "id": "CustomerEncryptionKey",
+  "properties": {
+    "kmsKeyName": {
+      "type": "string"
+    },
+    "rawKey": {
+      "type": "string"
+    },
+    "rsaEncryptedKey": {
+      "type": "string"
+    },
+    "sha256": {
+      "type": "string"
+    }
+  }
+}
+)""";
+
+  auto constexpr kExpectedProto = R"""(message CustomerEncryptionKey {
+  optional string kms_key_name = 1 [debug_redact = true,json_name="kmsKeyName"];
+
+  optional string raw_key = 2 [debug_redact = true,json_name="rawKey"];
+
+  optional string rsa_encrypted_key = 3 [debug_redact = true,json_name="rsaEncryptedKey"];
+
+  optional string sha256 = 4 [json_name="sha256"];
+}
+)""";
+
+  auto json = nlohmann::json::parse(kSchemaJson, nullptr, false);
+  ASSERT_TRUE(json.is_object());
+  DiscoveryTypeVertex t("CustomerEncryptionKey", "test.package", json, &pool());
+  std::map<std::string, DiscoveryTypeVertex> types;
+  auto result = t.JsonToProtobufMessage(types, "test.package");
+  ASSERT_THAT(result, ::google::cloud::testing_util::IsOk());
+  EXPECT_THAT(*result, Eq(kExpectedProto));
+}
+
 }  // namespace
 }  // namespace generator_internal
 }  // namespace cloud

google/cloud/internal/debug_string_protobuf.cc

@@ -68,6 +68,7 @@ std::string DebugString(google::protobuf::Message const& m,
                         TracingOptions const& options) {
   std::string str;
   google::protobuf::TextFormat::Printer p;
+  p.SetRedactDebugString(true);
   p.SetSingleLineMode(options.single_line_mode());
   if (!options.single_line_mode()) p.SetInitialIndentLevel(1);
   p.SetUseShortRepeatedPrimitives(options.use_short_repeated_primitives());