diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 000000000..7bba65f49 --- /dev/null +++ b/.bazelignore @@ -0,0 +1,8 @@ +# Directories populated by the CMake build (FetchContent checkouts and the +# third_party/absl symlink) contain their own BUILD files and must not be +# treated as packages of this workspace. +third_party/abseil-cpp +third_party/absl +third_party/benchmark +build +python diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000..464444065 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,10 @@ +# SentencePiece requires C++17. +build --enable_platform_specific_config + +build:linux --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 +build:macos --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 +build:freebsd --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 +build:windows --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 +build:windows --cxxopt=/utf-8 --host_cxxopt=/utf-8 + +test --test_output=errors diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 000000000..44931da26 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +9.1.1 diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml new file mode 100644 index 000000000..734873daa --- /dev/null +++ b/.github/workflows/bazel.yml @@ -0,0 +1,77 @@ +name: CI for Bazel build + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-and-test: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest ] + + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + env: + # Stop the MSYS bash on Windows from rewriting Bazel labels like + # //src:spm_test as filesystem paths. + MSYS_NO_PATHCONV: "1" + MSYS2_ARG_CONV_EXCL: "*" + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Mount Bazel caches + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/bazel-disk + ~/.cache/bazel-repo + key: bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('MODULE.bazel', '.bazelrc', '.bazelversion') }} + restore-keys: | + bazel-${{ runner.os }}-${{ runner.arch }}- + + - name: Configure Bazel + shell: bash + run: | + home="$HOME" + if [[ "$RUNNER_OS" == "Windows" ]]; then + home="$(cygpath -m "$HOME")" + # Keep output paths short to stay within MAX_PATH limits. + echo "startup --output_user_root=C:/bzl" >> "$HOME/.bazelrc" + # Materialize the runfiles tree so tests can read their data + # dependencies from the working directory, as on Linux/macOS. + echo "build --enable_runfiles" >> "$HOME/.bazelrc" + if [[ -z "${BAZEL_SH:-}" ]]; then + echo 'BAZEL_SH=C:\Program Files\Git\bin\bash.exe' >> "$GITHUB_ENV" + fi + fi + echo "build --disk_cache=$home/.cache/bazel-disk" >> "$HOME/.bazelrc" + echo "build --repository_cache=$home/.cache/bazel-repo" >> "$HOME/.bazelrc" + + - name: Build + shell: bash + run: bazel build --verbose_failures //... + + - name: Test + shell: bash + run: bazel test //src:spm_test + + - name: Smoke test + shell: bash + run: | + workdir="$PWD" + if [[ "$RUNNER_OS" == "Windows" ]]; then + workdir="$(cygpath -m "$PWD")" + fi + bazel run //src:spm_train -- \ + --input="$workdir/data/botchan.txt" --model_prefix="$workdir/m" --vocab_size=1000 + echo "Hello world." | bazel-bin/src/spm_encode --model="$workdir/m.model" diff --git a/.gitignore b/.gitignore index 9a96c8698..c30390d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,7 @@ python/test/sp_*.pickle third_party/absl test_tmp + +# Bazel +/bazel-* +MODULE.bazel.lock diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 000000000..14826b145 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,31 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.! + +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "VERSION.txt", + "config.h.in", + "LICENSE", +]) + +alias( + name = "sentencepiece", + actual = "//src:sentencepiece", +) + +alias( + name = "sentencepiece_train", + actual = "//src:sentencepiece_train", +) diff --git a/CMakeLists.txt b/CMakeLists.txt index 200e39484..9602f8541 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,21 +193,10 @@ if (SPM_ABSL_PROVIDER STREQUAL "module") GIT_PROGRESS TRUE GIT_TAG 20260526.0) add_subdirectory(third_party/abseil-cpp) - if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl) - file(CREATE_LINK - ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp/absl - ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl - SYMBOLIC) - endif() + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp) elseif (SPM_ABSL_PROVIDER STREQUAL "package") find_package(absl REQUIRED) get_target_property(ABSL_INCLUDE_DIRS absl::base INTERFACE_INCLUDE_DIRECTORIES) - if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl) - file(CREATE_LINK - ${ABSL_INCLUDE_DIRS}/absl - ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl - SYMBOLIC) - endif() include_directories(${ABSL_INCLUDE_DIRS}) endif() diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 000000000..003659fe1 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,37 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.! + +module( + name = "sentencepiece", + version = "0.2.2", +) + +# Keep in sync with the abseil-cpp GIT_TAG in CMakeLists.txt. +bazel_dep(name = "abseil-cpp", version = "20260526.0") + +# Upstream darts-clone v0.32h plus the compatibility helpers (copy_array, +# validate, ...) that the vendored third_party/darts_clone copy carries. +bazel_dep(name = "darts-clone", version = "0.32h.bcr.1") + +# Upstream esaxx at the commit the vendored third_party/esaxx copy is based +# on. The .bcr.1 revision exports esa.hxx and carries the index_type(0) +# template-deduction fix that the int64_t instantiation of esaxx() needs. +bazel_dep(name = "esaxx", version = "20250106.1.bcr.1") +bazel_dep(name = "platforms", version = "1.1.0") + +# The .pb.cc/.pb.h files are regenerated from src/*.proto at build time, so +# the Bazel build follows the SPM_PROTOBUF_PROVIDER=package CMake +# configuration instead of the vendored third_party/protobuf-lite runtime. +bazel_dep(name = "protobuf", version = "35.1") +bazel_dep(name = "rules_cc", version = "0.2.18") diff --git a/README.md b/README.md index 90f71e99e..7446c2b4b 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ To run these benchmarks yourself, see the [reproduction instructions and scripts For detailed guides, API references, and advanced usage, please refer to the following resources: * [Command Line Interface (CLI) & Build Guide](doc/cli.md) +* [Building with Bazel](doc/bazel.md) * [C++ API Reference](doc/cpp.md) * [Python API Reference](python/README.md) & [Python Module Directory](python/) * [Python Tokenizer Comparison Cheat Sheet](python/tokenizer_comparison_cheat_sheet.md) diff --git a/cmake/modify_headers.cmake b/cmake/modify_headers.cmake deleted file mode 100644 index 1ca607795..000000000 --- a/cmake/modify_headers.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# CMake script to modify headers for installation -# Replaces #include "third_party/absl/..." with #include - -if (NOT DEFINED INPUT_FILE OR NOT DEFINED OUTPUT_FILE) - message(FATAL_ERROR "INPUT_FILE and OUTPUT_FILE must be defined") -endif() - -file(READ "${INPUT_FILE}" content) -# Replace #include "third_party/absl/..."" with #include -string(REGEX REPLACE "#include \"third_party/(absl/[^\"]+)\"" "#include <\\1>" content "${content}") -file(WRITE "${OUTPUT_FILE}" "${content}") diff --git a/contrib/nlcodec/bpe_model_trainer_nlcodec.cc b/contrib/nlcodec/bpe_model_trainer_nlcodec.cc index b5546d888..8556e4811 100644 --- a/contrib/nlcodec/bpe_model_trainer_nlcodec.cc +++ b/contrib/nlcodec/bpe_model_trainer_nlcodec.cc @@ -4,8 +4,8 @@ #include "contrib/nlcodec/bpe_model_trainer_nlcodec.h" -#include "third_party/absl/container/flat_hash_set.h" -#include "third_party/absl/flags/flag.h" +#include "absl/container/flat_hash_set.h" +#include "absl/flags/flag.h" #include "util.h" // Flag defined here so it's available to all binaries (spm_train, tests, etc.) diff --git a/contrib/nlcodec/bpe_model_trainer_nlcodec_test.cc b/contrib/nlcodec/bpe_model_trainer_nlcodec_test.cc index 551468cfd..b90ef4ad8 100644 --- a/contrib/nlcodec/bpe_model_trainer_nlcodec_test.cc +++ b/contrib/nlcodec/bpe_model_trainer_nlcodec_test.cc @@ -8,8 +8,8 @@ #include "sentencepiece_processor.h" #include "sentencepiece_trainer.h" #include "testharness.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/strings/str_cat.h" +#include "absl/flags/flag.h" +#include "absl/strings/str_cat.h" #include "util.h" ABSL_DECLARE_FLAG(bool, nlcodec_bpe); diff --git a/data/BUILD.bazel b/data/BUILD.bazel new file mode 100644 index 000000000..3bfde7f3a --- /dev/null +++ b/data/BUILD.bazel @@ -0,0 +1,23 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.! + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "testdata", + srcs = glob( + ["**"], + exclude = ["BUILD.bazel"], + ), +) diff --git a/doc/bazel.md b/doc/bazel.md new file mode 100644 index 000000000..5cbc89e4a --- /dev/null +++ b/doc/bazel.md @@ -0,0 +1,85 @@ +# Building SentencePiece with Bazel + +In addition to the [CMake build](cli.md), SentencePiece can be built with +[Bazel](https://bazel.build). All third-party dependencies are resolved +through the [Bazel Central Registry](https://registry.bazel.build): + +- [abseil-cpp](https://github.com/abseil/abseil-cpp), at the same version as + the `GIT_TAG` pinned in `CMakeLists.txt`; +- [protobuf](https://github.com/protocolbuffers/protobuf): the protobuf code + is regenerated from `src/*.proto` at build time, matching the + `SPM_PROTOBUF_PROVIDER=package` CMake configuration (CMake defaults to the + vendored protobuf-lite runtime instead); +- [darts-clone](https://github.com/s-yata/darts-clone), as the + `0.32h.bcr.1` module that carries the compatibility helpers of the + vendored copy; +- [esaxx](https://github.com/hillbig/esaxx), as the `20250106.1.bcr.1` + module, which exports `esa.hxx` and carries the template-deduction fix + the vendored copy has for the `int64_t` instantiation used by + `--train_extremely_large_corpus`. + +The vendored sources under `third_party/` and the pre-generated protobuf +code in `src/builtin_pb` are only used by CMake. + +## Prerequisites + +- Bazel 7.2.1 or later (bzlmod is used; installing via + [Bazelisk](https://github.com/bazelbuild/bazelisk) is recommended) +- A C++17 compiler + +## Building + +Build the libraries and all command line tools: + +```bash +bazel build //... +``` + +The main targets are: + +| Target | Description | +| --- | --- | +| `//src:sentencepiece` (alias `//:sentencepiece`) | Runtime library (encode/decode) | +| `//src:sentencepiece_train` (alias `//:sentencepiece_train`) | Trainer library | +| `//src:spm_train` | Model trainer CLI | +| `//src:spm_encode` | Encoder CLI | +| `//src:spm_decode` | Decoder CLI | +| `//src:spm_normalize` | Text normalizer CLI | +| `//src:spm_export_vocab` | Vocabulary exporter CLI | + +Run a tool directly: + +```bash +bazel run //src:spm_train -- \ + --input=$(pwd)/data/botchan.txt --model_prefix=$(pwd)/m --vocab_size=8000 +echo "Hello world." | bazel-bin/src/spm_encode --model=m.model +``` + +## Running the unit tests + +The unit tests are a single test binary, like the `spm_test` target that +CMake builds with `-DSPM_BUILD_TEST=ON`: + +```bash +bazel test //src:spm_test +``` + +## Notes + +- The optional CMake features `SPM_ENABLE_NFKC_COMPILE` (ICU), + `SPM_ENABLE_TCMALLOC`, `SPM_ENABLE_BENCHMARK`, and `SPM_NLCODEC_BPE` have no + Bazel equivalent yet. +- The SentencePiece sources include third-party headers with the standard + paths the upstream projects export (`absl/...`, `darts.h`, `esa.hxx`), so + Bazel resolves them directly against the Bazel Central Registry modules. + CMake resolves the same includes against the abseil-cpp checkout and the + vendored copies under `third_party/`. No third-party code is vendored into + the Bazel build. +- To use SentencePiece as a dependency in another Bazel module: + + ```starlark + bazel_dep(name = "sentencepiece", version = "0.2.2") + ``` + + and depend on `@sentencepiece//:sentencepiece` (and + `@sentencepiece//:sentencepiece_train` for training support). diff --git a/src/BUILD.bazel b/src/BUILD.bazel new file mode 100644 index 000000000..fd17b72ce --- /dev/null +++ b/src/BUILD.bazel @@ -0,0 +1,320 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.! + +load("@protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +NOT_WIN_COPTS = [ + "-pthread", + "-Wall", +] + +# Generates config.h from config.h.in, like the configure_file() step in +# CMakeLists.txt. The version is read from VERSION.txt. +genrule( + name = "config_h", + srcs = [ + "//:VERSION.txt", + "//:config.h.in", + ], + outs = ["config.h"], + cmd = "VERSION=$$(tr -d ' \\r\\n' < $(location //:VERSION.txt)); " + + "sed -e \"s/@PROJECT_VERSION@/$${VERSION}/g\" " + + "-e \"s/@PROJECT_NAME@/sentencepiece/g\" " + + "-e \"s|@INSTALL_DATADIR@|share/sentencepiece|g\" " + + "$(location //:config.h.in) > $@", +) + +cc_library( + name = "config", + hdrs = [":config_h"], + includes = ["."], +) + +# The protobuf code is regenerated from the .proto files at build time with +# the protobuf module from the Bazel Central Registry, matching the +# SPM_PROTOBUF_PROVIDER=package CMake configuration. The pre-generated +# sources in builtin_pb/ and the vendored third_party/protobuf-lite runtime +# are only used by CMake (SPM_PROTOBUF_PROVIDER=internal, its default). +proto_library( + name = "sentencepiece_proto", + srcs = ["sentencepiece.proto"], +) + +cc_proto_library( + name = "sentencepiece_cc_proto", + deps = [":sentencepiece_proto"], +) + +proto_library( + name = "sentencepiece_model_proto", + srcs = ["sentencepiece_model.proto"], +) + +cc_proto_library( + name = "sentencepiece_model_cc_proto", + deps = [":sentencepiece_model_proto"], +) + +# Wrapper that puts the generated headers on the include path expected by +# the sources (#include "sentencepiece.pb.h") and selects the external +# protobuf branch of the #ifdef _USE_EXTERNAL_PROTOBUF includes. +cc_library( + name = "pb", + defines = ["_USE_EXTERNAL_PROTOBUF"], + includes = ["."], + deps = [ + ":sentencepiece_cc_proto", + ":sentencepiece_model_cc_proto", + ], +) + +# Runtime library (the `sentencepiece` target in CMake; SPM_SRCS). +cc_library( + name = "sentencepiece", + srcs = [ + "bpe_model.cc", + "char_model.cc", + "filesystem.cc", + "init.cc", + "model_factory.cc", + "model_interface.cc", + "normalizer.cc", + "sentencepiece_processor.cc", + "unigram_model.cc", + "util.cc", + "word_model.cc", + ], + hdrs = [ + "bpe_model.h", + "char_model.h", + "common.h", + "filesystem.h", + "freelist.h", + "init.h", + "model_factory.h", + "model_interface.h", + "normalizer.h", + "sentencepiece_processor.h", + "testharness.h", + "unigram_model.h", + "util.h", + "word_model.h", + ], + copts = select({ + "@platforms//os:windows": [], + "//conditions:default": NOT_WIN_COPTS, + }), + linkopts = select({ + "@platforms//cpu:riscv64": ["-latomic"], + "//conditions:default": [], + }), + # @abseil-cpp//absl/base:endian (for base/internal/endian.h, used by + # util.h) is intentionally absent: it is not publicly visible. Its header + # still reaches dependents through the transitive headers of + # @abseil-cpp//absl/strings. + deps = [ + ":config", + ":pb", + "@abseil-cpp//absl/base:core_headers", + "@abseil-cpp//absl/cleanup", + "@abseil-cpp//absl/container:fixed_array", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/flags:config", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/flags:parse", + "@abseil-cpp//absl/flags:usage", + "@abseil-cpp//absl/functional:any_invocable", + "@abseil-cpp//absl/functional:function_ref", + "@abseil-cpp//absl/log", + "@abseil-cpp//absl/log:check", + "@abseil-cpp//absl/log:globals", + "@abseil-cpp//absl/log:initialize", + "@abseil-cpp//absl/numeric:bits", + "@abseil-cpp//absl/random", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_builder", + "@abseil-cpp//absl/status:status_macros", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/synchronization", + "@abseil-cpp//absl/time", + "@abseil-cpp//absl/types:span", + "@darts-clone//:darts-clone", + ], +) + +# Trainer library (the `sentencepiece_train` target in CMake; SPM_TRAIN_SRCS). +cc_library( + name = "sentencepiece_train", + srcs = [ + "bpe_model_trainer.cc", + "builder.cc", + "char_model_trainer.cc", + "pretokenizer_for_training.cc", + "sentencepiece_trainer.cc", + "trainer_factory.cc", + "trainer_interface.cc", + "unicode_script.cc", + "unigram_model_trainer.cc", + "word_model_trainer.cc", + ], + hdrs = [ + "bpe_model_trainer.h", + "builder.h", + "char_model_trainer.h", + "normalization_rule.h", + "pretokenizer_for_training.h", + "sentencepiece_trainer.h", + "spec_parser.h", + "trainer_factory.h", + "trainer_interface.h", + "unicode_script.h", + "unicode_script_map.h", + "unigram_model_trainer.h", + "word_model_trainer.h", + ], + copts = select({ + "@platforms//os:windows": [], + "//conditions:default": NOT_WIN_COPTS, + }), + deps = [ + ":sentencepiece", + "@abseil-cpp//absl/container:btree", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/hash", + "@abseil-cpp//absl/random", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_builder", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/types:span", + "@esaxx//:esa", + ], +) + +cc_binary( + name = "spm_encode", + srcs = ["spm_encode_main.cc"], + deps = [ + ":sentencepiece", + # For trainer_interface.h. + ":sentencepiece_train", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/strings", + ], +) + +cc_binary( + name = "spm_decode", + srcs = ["spm_decode_main.cc"], + deps = [ + ":sentencepiece", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/strings", + ], +) + +cc_binary( + name = "spm_normalize", + srcs = ["spm_normalize_main.cc"], + deps = [ + ":sentencepiece", + ":sentencepiece_train", + "@abseil-cpp//absl/flags:flag", + ], +) + +cc_binary( + name = "spm_train", + srcs = ["spm_train_main.cc"], + deps = [ + ":sentencepiece", + ":sentencepiece_train", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/strings", + ], +) + +cc_binary( + name = "spm_export_vocab", + srcs = ["spm_export_vocab_main.cc"], + deps = [ + ":sentencepiece", + "@abseil-cpp//absl/flags:flag", + ], +) + +# Unit tests (SPM_TEST_SRCS). A single test binary that runs all tests, like +# the spm_test target in CMake. The custom test harness reads the data +# directory from --test_srcdir, relative to the runfiles working directory. +cc_test( + name = "spm_test", + size = "large", + srcs = [ + "bpe_model_test.cc", + "bpe_model_trainer_test.cc", + "builder_test.cc", + "char_model_test.cc", + "char_model_trainer_test.cc", + "filesystem_test.cc", + "invalid_model_test.cc", + "model_factory_test.cc", + "model_interface_test.cc", + "normalizer_test.cc", + "pretokenizer_for_training_test.cc", + "sentencepiece_processor_parallel_test.cc", + "sentencepiece_processor_test.cc", + "sentencepiece_trainer_test.cc", + "test_main.cc", + "testharness.cc", + "trainer_factory_test.cc", + "trainer_interface_test.cc", + "unicode_script_test.cc", + "unigram_model_test.cc", + "unigram_model_trainer_test.cc", + "util_test.cc", + "word_model_test.cc", + "word_model_trainer_test.cc", + ], + args = [ + "--test_srcdir=data", + "--test_tmpdir=spm_test_tmp", + ], + copts = select({ + "@platforms//os:windows": [], + "//conditions:default": NOT_WIN_COPTS + [ + "-Wno-sign-compare", + ], + }), + data = ["//data:testdata"], + deps = [ + ":sentencepiece", + ":sentencepiece_train", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/log", + "@abseil-cpp//absl/log:check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/time", + ], +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4b0e1f32a..967081d39 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -78,6 +78,8 @@ endif() include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../third_party) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/darts_clone) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/esaxx) set(SPM_SRCS ${PROTOBUF_LITE_SRCS} @@ -304,34 +306,10 @@ install(TARGETS ${SPM_INSTALLTARGETS} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() -# Generate modified headers for installation -set(GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/installed_headers") -set(MODIFIER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modify_headers.cmake") - -add_custom_command( - OUTPUT "${GENERATED_HEADERS_DIR}/sentencepiece_processor.h" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/sentencepiece_processor.h" "${MODIFIER_SCRIPT}" - COMMAND ${CMAKE_COMMAND} -DINPUT_FILE="${CMAKE_CURRENT_SOURCE_DIR}/sentencepiece_processor.h" - -DOUTPUT_FILE="${GENERATED_HEADERS_DIR}/sentencepiece_processor.h" - -P "${MODIFIER_SCRIPT}" - COMMENT "Generating modified sentencepiece_processor.h for installation" -) - -add_custom_command( - OUTPUT "${GENERATED_HEADERS_DIR}/sentencepiece_trainer.h" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/sentencepiece_trainer.h" "${MODIFIER_SCRIPT}" - COMMAND ${CMAKE_COMMAND} -DINPUT_FILE="${CMAKE_CURRENT_SOURCE_DIR}/sentencepiece_trainer.h" - -DOUTPUT_FILE="${GENERATED_HEADERS_DIR}/sentencepiece_trainer.h" - -P "${MODIFIER_SCRIPT}" - COMMENT "Generating modified sentencepiece_trainer.h for installation" -) - -add_custom_target(generate_installed_headers ALL - DEPENDS "${GENERATED_HEADERS_DIR}/sentencepiece_processor.h" - "${GENERATED_HEADERS_DIR}/sentencepiece_trainer.h" -) - -install(FILES "${GENERATED_HEADERS_DIR}/sentencepiece_trainer.h" "${GENERATED_HEADERS_DIR}/sentencepiece_processor.h" +# The public headers include Abseil with plain "absl/..." paths, so they can +# be installed as-is; consumers point their include path at an Abseil +# installation as usual. +install(FILES sentencepiece_trainer.h sentencepiece_processor.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) if (NOT SPM_PROTOBUF_PROVIDER STREQUAL "internal") install(FILES ${SPM_PROTO_HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/src/bpe_model.cc b/src/bpe_model.cc index e0c40041f..f28894d7d 100644 --- a/src/bpe_model.cc +++ b/src/bpe_model.cc @@ -25,10 +25,10 @@ #include "freelist.h" #include "model_interface.h" #include "sentencepiece_model.pb.h" -#include "third_party/absl/base/attributes.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/random/random.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/base/attributes.h" +#include "absl/container/flat_hash_map.h" +#include "absl/random/random.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/bpe_model.h b/src/bpe_model.h index 11df76c3f..238554831 100644 --- a/src/bpe_model.h +++ b/src/bpe_model.h @@ -17,7 +17,7 @@ #include "model_interface.h" #include "sentencepiece_model.pb.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/string_view.h" namespace sentencepiece { namespace bpe { diff --git a/src/bpe_model_trainer.cc b/src/bpe_model_trainer.cc index 57715d0fd..3594a66b0 100644 --- a/src/bpe_model_trainer.cc +++ b/src/bpe_model_trainer.cc @@ -20,13 +20,13 @@ #include #include "pretokenizer_for_training.h" -#include "third_party/absl/container/flat_hash_set.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/hash/hash.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/str_replace.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/container/flat_hash_set.h" +#include "absl/flags/flag.h" +#include "absl/hash/hash.h" +#include "absl/status/status.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" #include "util.h" #ifdef SPM_NLCODEC_BPE diff --git a/src/bpe_model_trainer.h b/src/bpe_model_trainer.h index f01edbb54..b5ca82dd3 100644 --- a/src/bpe_model_trainer.h +++ b/src/bpe_model_trainer.h @@ -23,9 +23,9 @@ #include #include "sentencepiece_model.pb.h" -#include "third_party/absl/container/btree_set.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/status/status.h" +#include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" #include "trainer_interface.h" namespace sentencepiece::bpe { diff --git a/src/bpe_model_trainer_test.cc b/src/bpe_model_trainer_test.cc index b811adf5a..9dea38eab 100644 --- a/src/bpe_model_trainer_test.cc +++ b/src/bpe_model_trainer_test.cc @@ -21,8 +21,8 @@ #include "sentencepiece_processor.h" #include "sentencepiece_trainer.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "util.h" namespace sentencepiece { diff --git a/src/builder.cc b/src/builder.cc index e47bd9ff4..7f37aa462 100644 --- a/src/builder.cc +++ b/src/builder.cc @@ -19,15 +19,15 @@ #include #include "filesystem.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/container/flat_hash_set.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/str_replace.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/strings/strip.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" #ifdef ENABLE_NFKC_COMPILE #include @@ -41,7 +41,7 @@ #include #include "normalizer.h" -#include "third_party/darts_clone/darts.h" +#include "darts.h" #include "util.h" #ifndef DISABLE_EMBEDDED_DATA diff --git a/src/builder.h b/src/builder.h index 9206f0920..8959e3e5f 100644 --- a/src/builder.h +++ b/src/builder.h @@ -22,8 +22,8 @@ #include "common.h" #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" namespace sentencepiece::normalizer { diff --git a/src/builder_test.cc b/src/builder_test.cc index 8b785727e..776eb211e 100644 --- a/src/builder_test.cc +++ b/src/builder_test.cc @@ -19,8 +19,8 @@ #include "normalizer.h" #include "sentencepiece_trainer.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/char_model.cc b/src/char_model.cc index e06fde359..3cf986072 100644 --- a/src/char_model.cc +++ b/src/char_model.cc @@ -14,7 +14,7 @@ #include "char_model.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/char_model.h b/src/char_model.h index 0af17d8c3..5fe22b5fd 100644 --- a/src/char_model.h +++ b/src/char_model.h @@ -17,7 +17,7 @@ #include "model_interface.h" #include "sentencepiece_model.pb.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/string_view.h" namespace sentencepiece { namespace character { diff --git a/src/char_model_trainer.cc b/src/char_model_trainer.cc index c35c05980..f11be3645 100644 --- a/src/char_model_trainer.cc +++ b/src/char_model_trainer.cc @@ -17,7 +17,7 @@ #include #include "char_model.h" -#include "third_party/absl/status/status.h" +#include "absl/status/status.h" #include "util.h" namespace sentencepiece { diff --git a/src/char_model_trainer.h b/src/char_model_trainer.h index 01c59fc0e..119aba822 100644 --- a/src/char_model_trainer.h +++ b/src/char_model_trainer.h @@ -16,7 +16,7 @@ #define CHAR_MODEL_TRAINER_H_ #include "sentencepiece_model.pb.h" -#include "third_party/absl/status/status.h" +#include "absl/status/status.h" #include "trainer_interface.h" namespace sentencepiece { diff --git a/src/char_model_trainer_test.cc b/src/char_model_trainer_test.cc index 2cc949e1c..21b9d6b2a 100644 --- a/src/char_model_trainer_test.cc +++ b/src/char_model_trainer_test.cc @@ -20,8 +20,8 @@ #include "filesystem.h" #include "sentencepiece_processor.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "util.h" namespace sentencepiece { diff --git a/src/common.h b/src/common.h index ba8286043..8a251ab8f 100644 --- a/src/common.h +++ b/src/common.h @@ -18,12 +18,12 @@ #include #include -#include "third_party/absl/log/check.h" -#include "third_party/absl/log/globals.h" -#include "third_party/absl/log/log.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/status/status_macros.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/log/check.h" +#include "absl/log/globals.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/status_macros.h" +#include "absl/strings/string_view.h" #if defined(_WIN32) && !defined(__CYGWIN__) #define OS_WIN diff --git a/src/compile_charsmap_main.cc b/src/compile_charsmap_main.cc index a79bc6d2f..cc744ea8f 100644 --- a/src/compile_charsmap_main.cc +++ b/src/compile_charsmap_main.cc @@ -22,10 +22,10 @@ #include "filesystem.h" #include "init.h" #include "sentencepiece_processor.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/flags/flag.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" using sentencepiece::normalizer::Builder; diff --git a/src/filesystem.cc b/src/filesystem.cc index 08c265626..174b6bfa1 100644 --- a/src/filesystem.cc +++ b/src/filesystem.cc @@ -18,8 +18,8 @@ #include #include -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "util.h" #if defined(OS_WIN) && defined(UNICODE) && defined(_UNICODE) diff --git a/src/filesystem.h b/src/filesystem.h index 35f463969..54949c279 100644 --- a/src/filesystem.h +++ b/src/filesystem.h @@ -23,8 +23,8 @@ #include "common.h" #include "sentencepiece_processor.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" namespace sentencepiece { namespace filesystem { diff --git a/src/filesystem_test.cc b/src/filesystem_test.cc index 423a2019f..0825cc380 100644 --- a/src/filesystem_test.cc +++ b/src/filesystem_test.cc @@ -15,7 +15,7 @@ #include "filesystem.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" +#include "absl/strings/str_cat.h" #include "util.h" namespace sentencepiece { diff --git a/src/init.cc b/src/init.cc index c2c651e20..641d9710d 100644 --- a/src/init.cc +++ b/src/init.cc @@ -16,12 +16,12 @@ #include "common.h" #include "config.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/flags/parse.h" -#include "third_party/absl/flags/usage.h" -#include "third_party/absl/flags/usage_config.h" -#include "third_party/absl/log/initialize.h" -#include "third_party/absl/strings/str_cat.h" +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/flags/usage.h" +#include "absl/flags/usage_config.h" +#include "absl/log/initialize.h" +#include "absl/strings/str_cat.h" #include "util.h" #ifdef _USE_EXTERNAL_PROTOBUF diff --git a/src/invalid_model_test.cc b/src/invalid_model_test.cc index c8dc78e9b..f4de5b8b4 100644 --- a/src/invalid_model_test.cc +++ b/src/invalid_model_test.cc @@ -21,10 +21,10 @@ #include #include "testharness.h" -#include "third_party/absl/base/internal/endian.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/darts_clone/darts.h" +#include "absl/base/internal/endian.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "darts.h" #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" #include "util.h" diff --git a/src/model_interface.cc b/src/model_interface.cc index 63eccee1f..f68320617 100644 --- a/src/model_interface.cc +++ b/src/model_interface.cc @@ -17,9 +17,9 @@ #include #include "sentencepiece_model.pb.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_format.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/model_interface.h b/src/model_interface.h index 39e0ef634..25b5dbd78 100644 --- a/src/model_interface.h +++ b/src/model_interface.h @@ -25,10 +25,10 @@ #include "normalizer.h" #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/darts_clone/darts.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "darts.h" #include "util.h" namespace sentencepiece { diff --git a/src/model_interface_test.cc b/src/model_interface_test.cc index c3ca26f52..c4baa1fdc 100644 --- a/src/model_interface_test.cc +++ b/src/model_interface_test.cc @@ -16,7 +16,7 @@ #include "model_factory.h" #include "testharness.h" -#include "third_party/absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_map.h" #include "util.h" namespace sentencepiece { diff --git a/src/normalizer.cc b/src/normalizer.cc index 05a493110..c48778b23 100644 --- a/src/normalizer.cc +++ b/src/normalizer.cc @@ -19,11 +19,11 @@ #include #include "common.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/match.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/strings/strip.h" -#include "third_party/darts_clone/darts.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "darts.h" #include "util.h" namespace sentencepiece { diff --git a/src/normalizer.h b/src/normalizer.h index 67a430b66..196bf8d44 100644 --- a/src/normalizer.h +++ b/src/normalizer.h @@ -24,9 +24,9 @@ #include "common.h" #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/darts_clone/darts.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "darts.h" namespace sentencepiece::normalizer { diff --git a/src/normalizer_test.cc b/src/normalizer_test.cc index 94f6b3488..3eba42805 100644 --- a/src/normalizer_test.cc +++ b/src/normalizer_test.cc @@ -21,9 +21,9 @@ #include "builder.h" #include "sentencepiece_trainer.h" #include "testharness.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/darts_clone/darts.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "darts.h" #include "util.h" namespace sentencepiece { diff --git a/src/pretokenizer_for_training.cc b/src/pretokenizer_for_training.cc index 06debb228..f1da565c2 100644 --- a/src/pretokenizer_for_training.cc +++ b/src/pretokenizer_for_training.cc @@ -15,8 +15,8 @@ #include -#include "third_party/absl/strings/str_replace.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" namespace sentencepiece { namespace pretokenizer { diff --git a/src/pretokenizer_for_training.h b/src/pretokenizer_for_training.h index caa8d3bfd..20d692451 100644 --- a/src/pretokenizer_for_training.h +++ b/src/pretokenizer_for_training.h @@ -21,8 +21,8 @@ #include "common.h" #include "sentencepiece.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" namespace sentencepiece::pretokenizer { diff --git a/src/pretokenizer_for_training_test.cc b/src/pretokenizer_for_training_test.cc index 57c392593..a5b8e3019 100644 --- a/src/pretokenizer_for_training_test.cc +++ b/src/pretokenizer_for_training_test.cc @@ -14,11 +14,11 @@ #include "pretokenizer_for_training.h" #include "testharness.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "trainer_interface.h" namespace sentencepiece { diff --git a/src/sentencepiece_processor.cc b/src/sentencepiece_processor.cc index 1421c1872..6c398ee9a 100644 --- a/src/sentencepiece_processor.cc +++ b/src/sentencepiece_processor.cc @@ -30,20 +30,20 @@ #include "model_interface.h" #include "normalizer.h" #include "sentencepiece.pb.h" -#include "third_party/absl/cleanup/cleanup.h" -#include "third_party/absl/container/fixed_array.h" -#include "third_party/absl/container/flat_hash_set.h" -#include "third_party/absl/functional/function_ref.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/numbers.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/str_replace.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/strings/strip.h" -#include "third_party/absl/synchronization/blocking_counter.h" -#include "third_party/absl/synchronization/mutex.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/fixed_array.h" +#include "absl/container/flat_hash_set.h" +#include "absl/functional/function_ref.h" +#include "absl/status/status.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/mutex.h" #include "unigram_model.h" #include "util.h" diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h index 2e7338963..5b6718838 100644 --- a/src/sentencepiece_processor.h +++ b/src/sentencepiece_processor.h @@ -24,13 +24,13 @@ #include #include -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/types/span.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" namespace absl { using std::string_view; } // namespace absl -#include "third_party/absl/status/status.h" +#include "absl/status/status.h" namespace sentencepiece { diff --git a/src/sentencepiece_processor_benchmarks_test.cc b/src/sentencepiece_processor_benchmarks_test.cc index d11d6e84e..ebfdbdf1f 100644 --- a/src/sentencepiece_processor_benchmarks_test.cc +++ b/src/sentencepiece_processor_benchmarks_test.cc @@ -23,11 +23,11 @@ #include "sentencepiece.pb.h" #include "sentencepiece_processor.h" #include "testharness.h" -#include "third_party/absl/log/check.h" -#include "third_party/absl/log/flags.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/log/check.h" +#include "absl/log/flags.h" +#include "absl/status/status.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "third_party/benchmark/include/benchmark/benchmark.h" #include "util.h" diff --git a/src/sentencepiece_processor_parallel_test.cc b/src/sentencepiece_processor_parallel_test.cc index c0afcce3f..685388264 100644 --- a/src/sentencepiece_processor_parallel_test.cc +++ b/src/sentencepiece_processor_parallel_test.cc @@ -20,8 +20,8 @@ #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" #include "testharness.h" -#include "third_party/absl/log/check.h" -#include "third_party/absl/log/log.h" +#include "absl/log/check.h" +#include "absl/log/log.h" #include "util.h" namespace sentencepiece { diff --git a/src/sentencepiece_processor_test.cc b/src/sentencepiece_processor_test.cc index f07f07619..d8854537f 100644 --- a/src/sentencepiece_processor_test.cc +++ b/src/sentencepiece_processor_test.cc @@ -24,10 +24,10 @@ #include "sentencepiece_model.pb.h" #include "sentencepiece_trainer.h" #include "testharness.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/strings/match.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/container/flat_hash_map.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/sentencepiece_trainer.cc b/src/sentencepiece_trainer.cc index 989801763..65ff2428a 100644 --- a/src/sentencepiece_trainer.cc +++ b/src/sentencepiece_trainer.cc @@ -23,13 +23,13 @@ #include "sentencepiece.pb.h" #include "sentencepiece_model.pb.h" #include "spec_parser.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/numbers.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/strings/strip.h" +#include "absl/flags/flag.h" +#include "absl/status/status.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" #include "trainer_factory.h" #include "util.h" diff --git a/src/sentencepiece_trainer.h b/src/sentencepiece_trainer.h index 5e8d6f731..166c68b11 100644 --- a/src/sentencepiece_trainer.h +++ b/src/sentencepiece_trainer.h @@ -21,9 +21,9 @@ #include #include "sentencepiece_processor.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/types/span.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" namespace sentencepiece { diff --git a/src/sentencepiece_trainer_test.cc b/src/sentencepiece_trainer_test.cc index 04c9f92e8..c7d0b192f 100644 --- a/src/sentencepiece_trainer_test.cc +++ b/src/sentencepiece_trainer_test.cc @@ -17,9 +17,9 @@ #include "filesystem.h" #include "sentencepiece_model.pb.h" #include "testharness.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/spec_parser.h b/src/spec_parser.h index 41fb5733a..f5e48b0ad 100644 --- a/src/spec_parser.h +++ b/src/spec_parser.h @@ -19,12 +19,12 @@ #include #include "sentencepiece_processor.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/status/status_builder.h" -#include "third_party/absl/strings/ascii.h" -#include "third_party/absl/strings/numbers.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/status/status_builder.h" +#include "absl/strings/ascii.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/spm_decode_main.cc b/src/spm_decode_main.cc index dc5ca5cff..97cde9928 100644 --- a/src/spm_decode_main.cc +++ b/src/spm_decode_main.cc @@ -21,8 +21,8 @@ #include "init.h" #include "sentencepiece.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/strings/str_split.h" +#include "absl/flags/flag.h" +#include "absl/strings/str_split.h" #include "util.h" ABSL_FLAG(std::string, model, "", "model file name"); diff --git a/src/spm_encode_main.cc b/src/spm_encode_main.cc index b05724af1..44328f756 100644 --- a/src/spm_encode_main.cc +++ b/src/spm_encode_main.cc @@ -22,11 +22,11 @@ #include "init.h" #include "sentencepiece.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/container/flat_hash_map.h" +#include "absl/flags/flag.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" #include "trainer_interface.h" ABSL_FLAG(std::string, model, "", "model file name"); diff --git a/src/spm_export_vocab_main.cc b/src/spm_export_vocab_main.cc index f68baa467..ed3c57706 100644 --- a/src/spm_export_vocab_main.cc +++ b/src/spm_export_vocab_main.cc @@ -19,7 +19,7 @@ #include "init.h" #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" -#include "third_party/absl/flags/flag.h" +#include "absl/flags/flag.h" ABSL_FLAG(std::string, output, "", "Output filename"); ABSL_FLAG(std::string, model, "", "input model file name"); diff --git a/src/spm_normalize_main.cc b/src/spm_normalize_main.cc index c004189b8..e15fa324b 100644 --- a/src/spm_normalize_main.cc +++ b/src/spm_normalize_main.cc @@ -21,7 +21,7 @@ #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" #include "sentencepiece_trainer.h" -#include "third_party/absl/flags/flag.h" +#include "absl/flags/flag.h" ABSL_FLAG(std::string, model, "", "Model file name"); ABSL_FLAG(bool, use_internal_normalization, false, diff --git a/src/spm_train_main.cc b/src/spm_train_main.cc index f42218ef6..7218bbc8b 100644 --- a/src/spm_train_main.cc +++ b/src/spm_train_main.cc @@ -18,11 +18,11 @@ #include "init.h" #include "sentencepiece_model.pb.h" #include "sentencepiece_trainer.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/strings/ascii.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/flags/flag.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "util.h" using sentencepiece::NormalizerSpec; diff --git a/src/testharness.cc b/src/testharness.cc index c8cd4c2c2..0ad01dc0e 100644 --- a/src/testharness.cc +++ b/src/testharness.cc @@ -26,7 +26,7 @@ #include #include "common.h" -#include "third_party/absl/strings/str_cat.h" +#include "absl/strings/str_cat.h" #include "util.h" namespace sentencepiece { diff --git a/src/testharness.h b/src/testharness.h index c573db30c..2c53f5e47 100644 --- a/src/testharness.h +++ b/src/testharness.h @@ -21,9 +21,9 @@ #include #include "common.h" -#include "third_party/absl/flags/flag.h" -#include "third_party/absl/flags/parse.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/strings/string_view.h" ABSL_DECLARE_FLAG(std::string, test_tmpdir); ABSL_DECLARE_FLAG(std::string, test_srcdir); diff --git a/src/trainer_interface.cc b/src/trainer_interface.cc index cf89c0ac7..6570dabbf 100644 --- a/src/trainer_interface.cc +++ b/src/trainer_interface.cc @@ -29,17 +29,17 @@ #include "normalizer.h" #include "sentencepiece_processor.h" #include "sentencepiece_trainer.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/container/flat_hash_set.h" -#include "third_party/absl/random/random.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/match.h" -#include "third_party/absl/strings/numbers.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_format.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/random/random.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "unicode_script.h" #include "util.h" diff --git a/src/trainer_interface.h b/src/trainer_interface.h index 9b84a8c4e..78115aa2a 100644 --- a/src/trainer_interface.h +++ b/src/trainer_interface.h @@ -27,9 +27,9 @@ #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" #include "sentencepiece_trainer.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/trainer_interface_test.cc b/src/trainer_interface_test.cc index 029290579..51103fd8a 100644 --- a/src/trainer_interface_test.cc +++ b/src/trainer_interface_test.cc @@ -18,9 +18,9 @@ #include "filesystem.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_format.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/unicode_script.cc b/src/unicode_script.cc index c013b4ae7..5e2b645dc 100644 --- a/src/unicode_script.cc +++ b/src/unicode_script.cc @@ -14,7 +14,7 @@ #include "unicode_script.h" -#include "third_party/absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_map.h" #include "unicode_script_map.h" #include "util.h" diff --git a/src/unicode_script_map.h b/src/unicode_script_map.h index a9cf471f3..2abb706e7 100644 --- a/src/unicode_script_map.h +++ b/src/unicode_script_map.h @@ -14,7 +14,7 @@ #ifndef UNICODE_SCRIPT_DATA_H_ #define UNICODE_SCRIPT_DATA_H_ -#include "third_party/absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_map.h" namespace sentencepiece { namespace unicode_script { namespace { diff --git a/src/unicode_script_test.cc b/src/unicode_script_test.cc index e193a7249..74b0e0228 100644 --- a/src/unicode_script_test.cc +++ b/src/unicode_script_test.cc @@ -16,7 +16,7 @@ #include "common.h" #include "testharness.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/unigram_model.cc b/src/unigram_model.cc index e6d81c51c..2cd4829a3 100644 --- a/src/unigram_model.cc +++ b/src/unigram_model.cc @@ -24,12 +24,12 @@ #include #include -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/random/random.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/time/clock.h" -#include "third_party/absl/time/time.h" +#include "absl/container/flat_hash_map.h" +#include "absl/random/random.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "util.h" namespace sentencepiece { diff --git a/src/unigram_model.h b/src/unigram_model.h index 027b46881..5a9c903dd 100644 --- a/src/unigram_model.h +++ b/src/unigram_model.h @@ -24,8 +24,8 @@ #include "freelist.h" #include "model_interface.h" #include "sentencepiece_model.pb.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/darts_clone/darts.h" +#include "absl/strings/string_view.h" +#include "darts.h" namespace sentencepiece { namespace unigram { diff --git a/src/unigram_model_test.cc b/src/unigram_model_test.cc index f64f3dd7a..19201a396 100644 --- a/src/unigram_model_test.cc +++ b/src/unigram_model_test.cc @@ -23,9 +23,9 @@ #include "sentencepiece_model.pb.h" #include "sentencepiece_processor.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/unigram_model_trainer.cc b/src/unigram_model_trainer.cc index 13a637e75..d2e8b6065 100644 --- a/src/unigram_model_trainer.cc +++ b/src/unigram_model_trainer.cc @@ -28,13 +28,13 @@ #include "normalizer.h" #include "pretokenizer_for_training.h" #include "sentencepiece_trainer.h" -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/numbers.h" -#include "third_party/absl/strings/str_replace.h" -#include "third_party/absl/strings/str_split.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/esaxx/esa.hxx" // Suffix array library. +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "esa.hxx" // Suffix array library. #include "trainer_interface.h" #include "unicode_script.h" #include "util.h" diff --git a/src/unigram_model_trainer.h b/src/unigram_model_trainer.h index 9c3b589ca..cee64f98b 100644 --- a/src/unigram_model_trainer.h +++ b/src/unigram_model_trainer.h @@ -21,8 +21,8 @@ #include #include "sentencepiece_model.pb.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "trainer_interface.h" #include "unigram_model.h" #include "util.h" diff --git a/src/unigram_model_trainer_test.cc b/src/unigram_model_trainer_test.cc index 4705c623d..9093d55d2 100644 --- a/src/unigram_model_trainer_test.cc +++ b/src/unigram_model_trainer_test.cc @@ -22,8 +22,8 @@ #include "sentencepiece_processor.h" #include "sentencepiece_trainer.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "util.h" namespace sentencepiece { diff --git a/src/util.cc b/src/util.cc index bd5ea1a16..d5f5532a3 100644 --- a/src/util.cc +++ b/src/util.cc @@ -18,11 +18,11 @@ #include #include -#include "third_party/absl/random/random.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/synchronization/blocking_counter.h" -#include "third_party/absl/synchronization/mutex.h" +#include "absl/random/random.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/mutex.h" namespace sentencepiece { diff --git a/src/util.h b/src/util.h index 07d9f37e3..63b487381 100644 --- a/src/util.h +++ b/src/util.h @@ -32,21 +32,21 @@ #include "common.h" #include "config.h" #include "sentencepiece_processor.h" -#include "third_party/absl/base/internal/endian.h" -#include "third_party/absl/base/thread_annotations.h" -#include "third_party/absl/functional/any_invocable.h" -#include "third_party/absl/numeric/bits.h" -#include "third_party/absl/random/random.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/status/status_builder.h" -#include "third_party/absl/strings/ascii.h" -#include "third_party/absl/strings/numbers.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_format.h" -#include "third_party/absl/strings/str_join.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/strings/strip.h" -#include "third_party/absl/synchronization/mutex.h" +#include "absl/base/internal/endian.h" +#include "absl/base/thread_annotations.h" +#include "absl/functional/any_invocable.h" +#include "absl/numeric/bits.h" +#include "absl/random/random.h" +#include "absl/status/status.h" +#include "absl/status/status_builder.h" +#include "absl/strings/ascii.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/synchronization/mutex.h" static constexpr uint32_t kUnicodeError = 0xFFFD; diff --git a/src/util_test.cc b/src/util_test.cc index 48695f566..b9014055c 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -18,11 +18,11 @@ #include "filesystem.h" #include "testharness.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/string_view.h" -#include "third_party/absl/time/clock.h" -#include "third_party/absl/time/time.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" namespace sentencepiece { namespace { diff --git a/src/word_model.cc b/src/word_model.cc index aa69f4050..786893177 100644 --- a/src/word_model.cc +++ b/src/word_model.cc @@ -14,7 +14,7 @@ #include "word_model.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/string_view.h" #include "util.h" namespace sentencepiece { diff --git a/src/word_model.h b/src/word_model.h index be1d0348b..7ea554cb9 100644 --- a/src/word_model.h +++ b/src/word_model.h @@ -17,7 +17,7 @@ #include "model_interface.h" #include "sentencepiece_model.pb.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/strings/string_view.h" namespace sentencepiece { namespace word { diff --git a/src/word_model_trainer.cc b/src/word_model_trainer.cc index 4bea35e48..005a16200 100644 --- a/src/word_model_trainer.cc +++ b/src/word_model_trainer.cc @@ -17,9 +17,9 @@ #include #include -#include "third_party/absl/container/flat_hash_map.h" -#include "third_party/absl/status/status.h" -#include "third_party/absl/strings/string_view.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "util.h" #include "word_model.h" diff --git a/src/word_model_trainer.h b/src/word_model_trainer.h index 77f2457cf..19ff9f533 100644 --- a/src/word_model_trainer.h +++ b/src/word_model_trainer.h @@ -16,7 +16,7 @@ #define WORD_MODEL_TRAINER_H_ #include "sentencepiece_model.pb.h" -#include "third_party/absl/status/status.h" +#include "absl/status/status.h" #include "trainer_interface.h" namespace sentencepiece { diff --git a/src/word_model_trainer_test.cc b/src/word_model_trainer_test.cc index 649129921..e8abf32f5 100644 --- a/src/word_model_trainer_test.cc +++ b/src/word_model_trainer_test.cc @@ -20,8 +20,8 @@ #include "filesystem.h" #include "sentencepiece_processor.h" #include "testharness.h" -#include "third_party/absl/strings/str_cat.h" -#include "third_party/absl/strings/str_join.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "util.h" namespace sentencepiece {