Add sentencepiece@0.2.1

modules/sentencepiece/0.2.1/MODULE.bazel

@@ -0,0 +1,25 @@
+"""SentencePiece is an unsupervised text tokenizer and detokenizer mainly for Neural Network-based text generation systems."""
+
+module(
+    name = "sentencepiece",
+    version = "0.2.1",
+    bazel_compatibility = [">=7.2.1"],
+)
+
+bazel_dep(name = "abseil-cpp", version = "20260526.0")
+
+# Upstream darts-clone v0.32h plus the compatibility helpers (copy_array,
+# validate, ...) that the vendored third_party/darts_clone copy carries.
+bazel_dep(name = "darts-clone", version = "0.32h.bcr.1")
+
+# Upstream esaxx at the commit the vendored third_party/esaxx copy is based
+# on. The .bcr.1 revision exports esa.hxx and carries the index_type(0)
+# template-deduction fix that the int64_t instantiation of esaxx() needs.
+bazel_dep(name = "esaxx", version = "20250106.1.bcr.1")
+bazel_dep(name = "platforms", version = "1.1.0")
+
+# The .pb.cc/.pb.h files are regenerated from src/*.proto at build time, so
+# the Bazel build follows the SPM_PROTOBUF_PROVIDER=package CMake
+# configuration instead of the vendored third_party/protobuf-lite runtime.
+bazel_dep(name = "protobuf", version = "35.1")
+bazel_dep(name = "rules_cc", version = "0.2.18")

modules/sentencepiece/0.2.1/overlay/BUILD.bazel

@@ -0,0 +1,31 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.!
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files([
+    "VERSION.txt",
+    "config.h.in",
+    "LICENSE",
+])
+
+alias(
+    name = "sentencepiece",
+    actual = "//src:sentencepiece",
+)
+
+alias(
+    name = "sentencepiece_train",
+    actual = "//src:sentencepiece_train",
+)

modules/sentencepiece/0.2.1/overlay/bcr_tests/BUILD.bazel

@@ -0,0 +1,10 @@
+load("@rules_cc//cc:defs.bzl", "cc_test")
+
+cc_test(
+    name = "roundtrip_test",
+    srcs = ["roundtrip_test.cc"],
+    deps = [
+        "@sentencepiece//:sentencepiece",
+        "@sentencepiece//:sentencepiece_train",
+    ],
+)

modules/sentencepiece/0.2.1/overlay/bcr_tests/MODULE.bazel

@@ -0,0 +1,9 @@
+module(name = "sentencepiece_bcr_tests")
+
+bazel_dep(name = "sentencepiece")
+local_path_override(
+    module_name = "sentencepiece",
+    path = "..",
+)
+
+bazel_dep(name = "rules_cc", version = "0.2.18")

modules/sentencepiece/0.2.1/overlay/bcr_tests/roundtrip_test.cc

@@ -0,0 +1,68 @@
+// Trains a small model fully in memory, then checks that encoding and
+// decoding roundtrips. This replaces the upstream spm_test, whose test data
+// (data/) is not shipped in the release tarball.
+//
+// SentencePiece 0.2.1 reports errors as sentencepiece::util::Status, so the
+// check macro takes the status by auto instead of naming the type.
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "sentencepiece_processor.h"
+#include "sentencepiece_trainer.h"
+
+#define CHECK_TEST(condition)                                         \
+  if (!(condition)) {                                                 \
+    std::cerr << __FILE__ << ":" << __LINE__ << " check failed: "     \
+              << #condition << std::endl;                             \
+    return 1;                                                         \
+  }
+
+#define CHECK_OK_TEST(expr)                                           \
+  {                                                                   \
+    const auto _status = (expr);                                      \
+    if (!_status.ok()) {                                              \
+      std::cerr << __FILE__ << ":" << __LINE__ << " status not ok: "  \
+                << _status.ToString() << std::endl;                   \
+      return 1;                                                       \
+    }                                                                 \
+  }
+
+int main() {
+  const std::vector<std::string> base = {
+      "The quick brown fox jumps over the lazy dog.",
+      "SentencePiece is an unsupervised text tokenizer and detokenizer.",
+      "Hello world. Hello Bazel.",
+      "I saw a girl with a telescope.",
+  };
+  std::vector<std::string> sentences;
+  for (int i = 0; i < 200; ++i) {
+    sentences.push_back(base[i % base.size()]);
+  }
+
+  std::string serialized_model;
+  CHECK_OK_TEST(sentencepiece::SentencePieceTrainer::Train(
+      "--vocab_size=60 --hard_vocab_limit=false --minloglevel=1", sentences,
+      &serialized_model));
+  CHECK_TEST(!serialized_model.empty());
+
+  sentencepiece::SentencePieceProcessor sp;
+  CHECK_OK_TEST(sp.LoadFromSerializedProto(serialized_model));
+
+  const std::string input = "Hello world.";
+  std::vector<std::string> pieces;
+  CHECK_OK_TEST(sp.Encode(input, &pieces));
+  CHECK_TEST(!pieces.empty());
+
+  std::string detokenized;
+  CHECK_OK_TEST(sp.Decode(pieces, &detokenized));
+  CHECK_TEST(detokenized == input);
+
+  std::vector<int> ids;
+  CHECK_OK_TEST(sp.Encode(input, &ids));
+  CHECK_TEST(!ids.empty());
+
+  std::cout << "PASS" << std::endl;
+  return 0;
+}

modules/sentencepiece/0.2.1/overlay/src/BUILD.bazel

@@ -0,0 +1,240 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.!
+
+load("@protobuf//bazel:cc_proto_library.bzl", "cc_proto_library")
+load("@protobuf//bazel:proto_library.bzl", "proto_library")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
+
+package(default_visibility = ["//visibility:public"])
+
+NOT_WIN_COPTS = [
+    "-pthread",
+    "-Wall",
+]
+
+# Generates config.h from config.h.in, like the configure_file() step in
+# CMakeLists.txt. The version is read from VERSION.txt.
+genrule(
+    name = "config_h",
+    srcs = [
+        "//:VERSION.txt",
+        "//:config.h.in",
+    ],
+    outs = ["config.h"],
+    cmd = "VERSION=$$(tr -d ' \\r\\n' < $(location //:VERSION.txt)); " +
+          "sed -e \"s/@PROJECT_VERSION@/$${VERSION}/g\" " +
+          "-e \"s/@PROJECT_NAME@/sentencepiece/g\" " +
+          "-e \"s|@INSTALL_DATADIR@|share/sentencepiece|g\" " +
+          "$(location //:config.h.in) > $@",
+)
+
+cc_library(
+    name = "config",
+    hdrs = [":config_h"],
+    includes = ["."],
+)
+
+# The protobuf code is regenerated from the .proto files at build time with
+# the protobuf module from the Bazel Central Registry, matching the
+# SPM_PROTOBUF_PROVIDER=package CMake configuration. The pre-generated
+# sources in builtin_pb/ and the vendored third_party/protobuf-lite runtime
+# are only used by CMake (SPM_PROTOBUF_PROVIDER=internal, its default).
+proto_library(
+    name = "sentencepiece_proto",
+    srcs = ["sentencepiece.proto"],
+)
+
+cc_proto_library(
+    name = "sentencepiece_cc_proto",
+    deps = [":sentencepiece_proto"],
+)
+
+proto_library(
+    name = "sentencepiece_model_proto",
+    srcs = ["sentencepiece_model.proto"],
+)
+
+cc_proto_library(
+    name = "sentencepiece_model_cc_proto",
+    deps = [":sentencepiece_model_proto"],
+)
+
+# Wrapper that puts the generated headers on the include path expected by
+# the sources (#include "sentencepiece.pb.h") and selects the external
+# protobuf branch of the #ifdef _USE_EXTERNAL_PROTOBUF includes.
+cc_library(
+    name = "pb",
+    defines = ["_USE_EXTERNAL_PROTOBUF"],
+    includes = ["."],
+    deps = [
+        ":sentencepiece_cc_proto",
+        ":sentencepiece_model_cc_proto",
+    ],
+)
+
+# Runtime library (the `sentencepiece` target in CMake; SPM_SRCS).
+cc_library(
+    name = "sentencepiece",
+    srcs = [
+        "bpe_model.cc",
+        "char_model.cc",
+        "error.cc",
+        "filesystem.cc",
+        "model_factory.cc",
+        "model_interface.cc",
+        "normalizer.cc",
+        "sentencepiece_processor.cc",
+        "unigram_model.cc",
+        "util.cc",
+        "word_model.cc",
+    ],
+    hdrs = [
+        "bpe_model.h",
+        "char_model.h",
+        "common.h",
+        "filesystem.h",
+        "freelist.h",
+        "init.h",
+        "model_factory.h",
+        "model_interface.h",
+        "normalizer.h",
+        "sentencepiece_processor.h",
+        "unigram_model.h",
+        "util.h",
+        "word_model.h",
+    ],
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": NOT_WIN_COPTS,
+    }),
+    # Selects the external-absl branch of the #ifdef _USE_EXTERNAL_ABSL in
+    # error.cc, which defines the --minloglevel flag (the internal branch
+    # gets it from the vendored third_party/absl/flags/flag.cc shim).
+    defines = ["_USE_EXTERNAL_ABSL"],
+    linkopts = select({
+        "@platforms//cpu:riscv64": ["-latomic"],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":config",
+        ":pb",
+        "@abseil-cpp//absl/container:flat_hash_map",
+        "@abseil-cpp//absl/container:flat_hash_set",
+        "@abseil-cpp//absl/flags:flag",
+        "@abseil-cpp//absl/flags:parse",
+        "@abseil-cpp//absl/flags:usage",
+        "@abseil-cpp//absl/strings",
+        "@abseil-cpp//absl/strings:str_format",
+        "@darts-clone//:darts-clone",
+    ],
+)
+
+# Trainer library (the `sentencepiece_train` target in CMake; SPM_TRAIN_SRCS).
+cc_library(
+    name = "sentencepiece_train",
+    srcs = [
+        "bpe_model_trainer.cc",
+        "builder.cc",
+        "char_model_trainer.cc",
+        "pretokenizer_for_training.cc",
+        "sentencepiece_trainer.cc",
+        "trainer_factory.cc",
+        "trainer_interface.cc",
+        "unicode_script.cc",
+        "unigram_model_trainer.cc",
+        "word_model_trainer.cc",
+    ],
+    hdrs = [
+        "bpe_model_trainer.h",
+        "builder.h",
+        "char_model_trainer.h",
+        "normalization_rule.h",
+        "pretokenizer_for_training.h",
+        "sentencepiece_trainer.h",
+        "spec_parser.h",
+        "trainer_factory.h",
+        "trainer_interface.h",
+        "unicode_script.h",
+        "unicode_script_map.h",
+        "unigram_model_trainer.h",
+        "word_model_trainer.h",
+    ],
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": NOT_WIN_COPTS,
+    }),
+    deps = [
+        ":sentencepiece",
+        "@abseil-cpp//absl/container:btree",
+        "@abseil-cpp//absl/container:flat_hash_map",
+        "@abseil-cpp//absl/container:flat_hash_set",
+        "@abseil-cpp//absl/flags:flag",
+        "@abseil-cpp//absl/strings",
+        "@abseil-cpp//absl/strings:str_format",
+        "@esaxx//:esa",
+    ],
+)
+
+cc_binary(
+    name = "spm_encode",
+    srcs = ["spm_encode_main.cc"],
+    deps = [
+        ":sentencepiece",
+        # For trainer_interface.h.
+        ":sentencepiece_train",
+        "@abseil-cpp//absl/container:flat_hash_map",
+        "@abseil-cpp//absl/flags:flag",
+        "@abseil-cpp//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "spm_decode",
+    srcs = ["spm_decode_main.cc"],
+    deps = [
+        ":sentencepiece",
+        "@abseil-cpp//absl/flags:flag",
+        "@abseil-cpp//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "spm_normalize",
+    srcs = ["spm_normalize_main.cc"],
+    deps = [
+        ":sentencepiece",
+        ":sentencepiece_train",
+        "@abseil-cpp//absl/flags:flag",
+    ],
+)
+
+cc_binary(
+    name = "spm_train",
+    srcs = ["spm_train_main.cc"],
+    deps = [
+        ":sentencepiece",
+        ":sentencepiece_train",
+        "@abseil-cpp//absl/flags:flag",
+        "@abseil-cpp//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "spm_export_vocab",
+    srcs = ["spm_export_vocab_main.cc"],
+    deps = [
+        ":sentencepiece",
+        "@abseil-cpp//absl/flags:flag",
+    ],
+)