diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..338323e --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: lint + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + lintrunner: + name: lintrunner + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install lintrunner dependencies + run: pip install -r requirements-lintrunner.txt + + - name: Initialize lintrunner + run: lintrunner init + + - name: Run lintrunner + run: lintrunner --force-color --all-files diff --git a/.gitignore b/.gitignore index cdb9276..e9ec176 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,8 @@ pip-out/ *.so *.dylib *.pyd + +# Swift Package Manager +.build/ +.swiftpm/ +Package.resolved diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..e724eba --- /dev/null +++ b/Package.swift @@ -0,0 +1,374 @@ +// swift-tools-version: 5.9 +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +import PackageDescription + +let package = Package( + name: "tokenizers", + platforms: [.iOS(.v16), .macOS(.v13)], + products: [ + .library(name: "tokenizers", targets: ["tokenizers"]), + ], + targets: [ + .target( + name: "tokenizers", + path: ".", + exclude: [ + // Exclude non-source directories + ".ci", + ".github", + "cmake", + "examples", + "pytorch_tokenizers", + "test", + + // Exclude build/config files + "BUCK", + "CMakeLists.txt", + "CODE_OF_CONDUCT", + "CONTRIBUTING.md", + "LICENSE", + "MANIFEST.in", + "README.md", + "Utils.cmake", + "pyproject.toml", + "pytest.ini", + "setup.py", + ".clang-format", + ".cmakelintrc", + ".flake8", + ".gitmodules", + ".lintrunner.toml", + ".mypy.ini", + "requirements-lintrunner.txt", + + // Exclude unused source files + "src/pcre2_regex.cpp", + "src/python_bindings.cpp", + "src/regex_lookahead.cpp", + "src/std_regex.cpp", + + // Exclude third-party non-source content + "third-party/BUCK", + "third-party/targets.bzl", + "third-party/pcre2", + + // abseil-cpp: only headers used, sources listed explicitly + "third-party/abseil-cpp/CMakeLists.txt", + + // json: header-only, no sources needed + "third-party/json/CMakeLists.txt", + + // sentencepiece: exclude build files and training code + "third-party/sentencepiece/CMakeLists.txt", + "third-party/sentencepiece/src/CMakeLists.txt", + "third-party/sentencepiece/src/builder.cc", + "third-party/sentencepiece/src/compile_charsmap_main.cc", + "third-party/sentencepiece/src/spm_decode_main.cc", + "third-party/sentencepiece/src/spm_encode_main.cc", + "third-party/sentencepiece/src/spm_export_vocab_main.cc", + "third-party/sentencepiece/src/spm_normalize_main.cc", + "third-party/sentencepiece/src/spm_train_main.cc", + "third-party/sentencepiece/src/unicode_script.cc", + "third-party/sentencepiece/src/trainer_factory.cc", + "third-party/sentencepiece/src/trainer_interface.cc", + "third-party/sentencepiece/src/unigram_model_trainer.cc", + "third-party/sentencepiece/src/word_model_trainer.cc", + "third-party/sentencepiece/src/char_model_trainer.cc", + "third-party/sentencepiece/src/bpe_model_trainer.cc", + "third-party/sentencepiece/src/sentencepiece_trainer.cc", + "third-party/sentencepiece/src/pretokenizer_for_training.cc", + "third-party/sentencepiece/third_party/absl", + + // re2: exclude build files and tools + "third-party/re2/CMakeLists.txt", + "third-party/re2/re2/fuzzing", + "third-party/re2/re2/testing", + "third-party/re2/util/pcre.cc", + + // llama.cpp-unicode: exclude build files + "third-party/llama.cpp-unicode/CMakeLists.txt", + ], + sources: [ + // Tokenizers core + "src/bpe_tokenizer_base.cpp", + "src/hf_tokenizer.cpp", + "src/llama2c_tokenizer.cpp", + "src/normalizer.cpp", + "src/post_processor.cpp", + "src/pre_tokenizer.cpp", + "src/re2_regex.cpp", + "src/regex.cpp", + "src/sentencepiece.cpp", + "src/tekken.cpp", + "src/tiktoken.cpp", + "src/token_decoder.cpp", + + // llama.cpp-unicode + "third-party/llama.cpp-unicode/src", + + // RE2 + "third-party/re2/re2/bitmap256.cc", + "third-party/re2/re2/bitstate.cc", + "third-party/re2/re2/compile.cc", + "third-party/re2/re2/dfa.cc", + "third-party/re2/re2/filtered_re2.cc", + "third-party/re2/re2/mimics_pcre.cc", + "third-party/re2/re2/nfa.cc", + "third-party/re2/re2/onepass.cc", + "third-party/re2/re2/parse.cc", + "third-party/re2/re2/perl_groups.cc", + "third-party/re2/re2/prefilter.cc", + "third-party/re2/re2/prefilter_tree.cc", + "third-party/re2/re2/prog.cc", + "third-party/re2/re2/re2.cc", + "third-party/re2/re2/regexp.cc", + "third-party/re2/re2/set.cc", + "third-party/re2/re2/simplify.cc", + "third-party/re2/re2/tostring.cc", + "third-party/re2/re2/unicode_casefold.cc", + "third-party/re2/re2/unicode_groups.cc", + "third-party/re2/util/rune.cc", + "third-party/re2/util/strutil.cc", + + // Abseil - base + "third-party/abseil-cpp/absl/base/internal/cycleclock.cc", + "third-party/abseil-cpp/absl/base/internal/low_level_alloc.cc", + "third-party/abseil-cpp/absl/base/internal/poison.cc", + "third-party/abseil-cpp/absl/base/internal/raw_logging.cc", + "third-party/abseil-cpp/absl/base/internal/spinlock.cc", + "third-party/abseil-cpp/absl/base/internal/spinlock_wait.cc", + "third-party/abseil-cpp/absl/base/internal/strerror.cc", + "third-party/abseil-cpp/absl/base/internal/sysinfo.cc", + "third-party/abseil-cpp/absl/base/internal/thread_identity.cc", + "third-party/abseil-cpp/absl/base/internal/throw_delegate.cc", + "third-party/abseil-cpp/absl/base/internal/tracing.cc", + "third-party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc", + "third-party/abseil-cpp/absl/base/log_severity.cc", + // Abseil - container + "third-party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc", + "third-party/abseil-cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc", + "third-party/abseil-cpp/absl/container/internal/raw_hash_set.cc", + // Abseil - crc + "third-party/abseil-cpp/absl/crc/crc32c.cc", + "third-party/abseil-cpp/absl/crc/internal/cpu_detect.cc", + "third-party/abseil-cpp/absl/crc/internal/crc.cc", + "third-party/abseil-cpp/absl/crc/internal/crc_cord_state.cc", + "third-party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc", + "third-party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc", + "third-party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc", + "third-party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc", + // Abseil - debugging + "third-party/abseil-cpp/absl/debugging/failure_signal_handler.cc", + "third-party/abseil-cpp/absl/debugging/internal/address_is_readable.cc", + "third-party/abseil-cpp/absl/debugging/internal/decode_rust_punycode.cc", + "third-party/abseil-cpp/absl/debugging/internal/demangle.cc", + "third-party/abseil-cpp/absl/debugging/internal/demangle_rust.cc", + "third-party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc", + "third-party/abseil-cpp/absl/debugging/internal/examine_stack.cc", + "third-party/abseil-cpp/absl/debugging/internal/stack_consumption.cc", + "third-party/abseil-cpp/absl/debugging/internal/utf8_for_code_point.cc", + "third-party/abseil-cpp/absl/debugging/internal/vdso_support.cc", + "third-party/abseil-cpp/absl/debugging/leak_check.cc", + "third-party/abseil-cpp/absl/debugging/stacktrace.cc", + "third-party/abseil-cpp/absl/debugging/symbolize.cc", + // Abseil - flags + "third-party/abseil-cpp/absl/flags/commandlineflag.cc", + "third-party/abseil-cpp/absl/flags/internal/commandlineflag.cc", + "third-party/abseil-cpp/absl/flags/internal/flag.cc", + "third-party/abseil-cpp/absl/flags/internal/private_handle_accessor.cc", + "third-party/abseil-cpp/absl/flags/internal/program_name.cc", + "third-party/abseil-cpp/absl/flags/internal/usage.cc", + "third-party/abseil-cpp/absl/flags/marshalling.cc", + "third-party/abseil-cpp/absl/flags/parse.cc", + "third-party/abseil-cpp/absl/flags/reflection.cc", + "third-party/abseil-cpp/absl/flags/usage.cc", + "third-party/abseil-cpp/absl/flags/usage_config.cc", + // Abseil - hash + "third-party/abseil-cpp/absl/hash/internal/city.cc", + "third-party/abseil-cpp/absl/hash/internal/hash.cc", + "third-party/abseil-cpp/absl/hash/internal/low_level_hash.cc", + // Abseil - log + "third-party/abseil-cpp/absl/log/die_if_null.cc", + "third-party/abseil-cpp/absl/log/flags.cc", + "third-party/abseil-cpp/absl/log/globals.cc", + "third-party/abseil-cpp/absl/log/initialize.cc", + "third-party/abseil-cpp/absl/log/internal/check_op.cc", + "third-party/abseil-cpp/absl/log/internal/conditions.cc", + "third-party/abseil-cpp/absl/log/internal/fnmatch.cc", + "third-party/abseil-cpp/absl/log/internal/globals.cc", + "third-party/abseil-cpp/absl/log/internal/log_format.cc", + "third-party/abseil-cpp/absl/log/internal/log_message.cc", + "third-party/abseil-cpp/absl/log/internal/log_sink_set.cc", + "third-party/abseil-cpp/absl/log/internal/nullguard.cc", + "third-party/abseil-cpp/absl/log/internal/proto.cc", + "third-party/abseil-cpp/absl/log/internal/vlog_config.cc", + "third-party/abseil-cpp/absl/log/log_entry.cc", + "third-party/abseil-cpp/absl/log/log_sink.cc", + // Abseil - numeric + "third-party/abseil-cpp/absl/numeric/int128.cc", + // Abseil - profiling + "third-party/abseil-cpp/absl/profiling/internal/exponential_biased.cc", + "third-party/abseil-cpp/absl/profiling/internal/periodic_sampler.cc", + // Abseil - random + "third-party/abseil-cpp/absl/random/discrete_distribution.cc", + "third-party/abseil-cpp/absl/random/gaussian_distribution.cc", + "third-party/abseil-cpp/absl/random/internal/chi_square.cc", + "third-party/abseil-cpp/absl/random/internal/pool_urbg.cc", + "third-party/abseil-cpp/absl/random/internal/randen.cc", + "third-party/abseil-cpp/absl/random/internal/randen_detect.cc", + "third-party/abseil-cpp/absl/random/internal/randen_hwaes.cc", + "third-party/abseil-cpp/absl/random/internal/randen_round_keys.cc", + "third-party/abseil-cpp/absl/random/internal/randen_slow.cc", + "third-party/abseil-cpp/absl/random/internal/seed_material.cc", + "third-party/abseil-cpp/absl/random/seed_gen_exception.cc", + "third-party/abseil-cpp/absl/random/seed_sequences.cc", + // Abseil - status + "third-party/abseil-cpp/absl/status/internal/status_internal.cc", + "third-party/abseil-cpp/absl/status/status.cc", + "third-party/abseil-cpp/absl/status/status_payload_printer.cc", + "third-party/abseil-cpp/absl/status/statusor.cc", + // Abseil - strings + "third-party/abseil-cpp/absl/strings/ascii.cc", + "third-party/abseil-cpp/absl/strings/charconv.cc", + "third-party/abseil-cpp/absl/strings/cord.cc", + "third-party/abseil-cpp/absl/strings/cord_analysis.cc", + "third-party/abseil-cpp/absl/strings/cord_buffer.cc", + "third-party/abseil-cpp/absl/strings/escaping.cc", + "third-party/abseil-cpp/absl/strings/internal/charconv_bigint.cc", + "third-party/abseil-cpp/absl/strings/internal/charconv_parse.cc", + "third-party/abseil-cpp/absl/strings/internal/cord_internal.cc", + "third-party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc", + "third-party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc", + "third-party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc", + "third-party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc", + "third-party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc", + "third-party/abseil-cpp/absl/strings/internal/cordz_functions.cc", + "third-party/abseil-cpp/absl/strings/internal/cordz_handle.cc", + "third-party/abseil-cpp/absl/strings/internal/cordz_info.cc", + "third-party/abseil-cpp/absl/strings/internal/cordz_sample_token.cc", + "third-party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc", + "third-party/abseil-cpp/absl/strings/internal/escaping.cc", + "third-party/abseil-cpp/absl/strings/internal/memutil.cc", + "third-party/abseil-cpp/absl/strings/internal/ostringstream.cc", + "third-party/abseil-cpp/absl/strings/internal/pow10_helper.cc", + "third-party/abseil-cpp/absl/strings/internal/str_format/arg.cc", + "third-party/abseil-cpp/absl/strings/internal/str_format/bind.cc", + "third-party/abseil-cpp/absl/strings/internal/str_format/extension.cc", + "third-party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc", + "third-party/abseil-cpp/absl/strings/internal/str_format/output.cc", + "third-party/abseil-cpp/absl/strings/internal/str_format/parser.cc", + "third-party/abseil-cpp/absl/strings/internal/stringify_sink.cc", + "third-party/abseil-cpp/absl/strings/internal/utf8.cc", + "third-party/abseil-cpp/absl/strings/match.cc", + "third-party/abseil-cpp/absl/strings/numbers.cc", + "third-party/abseil-cpp/absl/strings/str_cat.cc", + "third-party/abseil-cpp/absl/strings/str_replace.cc", + "third-party/abseil-cpp/absl/strings/str_split.cc", + "third-party/abseil-cpp/absl/strings/string_view.cc", + "third-party/abseil-cpp/absl/strings/substitute.cc", + // Abseil - synchronization + "third-party/abseil-cpp/absl/synchronization/barrier.cc", + "third-party/abseil-cpp/absl/synchronization/blocking_counter.cc", + "third-party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc", + "third-party/abseil-cpp/absl/synchronization/internal/futex_waiter.cc", + "third-party/abseil-cpp/absl/synchronization/internal/graphcycles.cc", + "third-party/abseil-cpp/absl/synchronization/internal/kernel_timeout.cc", + "third-party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc", + "third-party/abseil-cpp/absl/synchronization/internal/pthread_waiter.cc", + "third-party/abseil-cpp/absl/synchronization/internal/sem_waiter.cc", + "third-party/abseil-cpp/absl/synchronization/internal/stdcpp_waiter.cc", + "third-party/abseil-cpp/absl/synchronization/internal/waiter_base.cc", + "third-party/abseil-cpp/absl/synchronization/internal/win32_waiter.cc", + "third-party/abseil-cpp/absl/synchronization/mutex.cc", + "third-party/abseil-cpp/absl/synchronization/notification.cc", + // Abseil - time + "third-party/abseil-cpp/absl/time/civil_time.cc", + "third-party/abseil-cpp/absl/time/clock.cc", + "third-party/abseil-cpp/absl/time/duration.cc", + "third-party/abseil-cpp/absl/time/format.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/civil_time_detail.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_fixed.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_if.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_impl.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc", + "third-party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc", + "third-party/abseil-cpp/absl/time/time.cc", + // Abseil - types + "third-party/abseil-cpp/absl/types/bad_any_cast.cc", + "third-party/abseil-cpp/absl/types/bad_optional_access.cc", + "third-party/abseil-cpp/absl/types/bad_variant_access.cc", + + // SentencePiece + "third-party/sentencepiece/src/bpe_model.cc", + "third-party/sentencepiece/src/char_model.cc", + "third-party/sentencepiece/src/error.cc", + "third-party/sentencepiece/src/filesystem.cc", + "third-party/sentencepiece/src/model_factory.cc", + "third-party/sentencepiece/src/model_interface.cc", + "third-party/sentencepiece/src/normalizer.cc", + "third-party/sentencepiece/src/sentencepiece_processor.cc", + "third-party/sentencepiece/src/unigram_model.cc", + "third-party/sentencepiece/src/util.cc", + "third-party/sentencepiece/src/word_model.cc", + "third-party/sentencepiece/src/builtin_pb/sentencepiece.pb.cc", + "third-party/sentencepiece/src/builtin_pb/sentencepiece_model.pb.cc", + + // Protobuf-lite (bundled with sentencepiece) + "third-party/sentencepiece/third_party/protobuf-lite/arena.cc", + "third-party/sentencepiece/third_party/protobuf-lite/arenastring.cc", + "third-party/sentencepiece/third_party/protobuf-lite/bytestream.cc", + "third-party/sentencepiece/third_party/protobuf-lite/coded_stream.cc", + "third-party/sentencepiece/third_party/protobuf-lite/common.cc", + "third-party/sentencepiece/third_party/protobuf-lite/extension_set.cc", + "third-party/sentencepiece/third_party/protobuf-lite/generated_enum_util.cc", + "third-party/sentencepiece/third_party/protobuf-lite/generated_message_table_driven_lite.cc", + "third-party/sentencepiece/third_party/protobuf-lite/generated_message_util.cc", + "third-party/sentencepiece/third_party/protobuf-lite/implicit_weak_message.cc", + "third-party/sentencepiece/third_party/protobuf-lite/int128.cc", + "third-party/sentencepiece/third_party/protobuf-lite/io_win32.cc", + "third-party/sentencepiece/third_party/protobuf-lite/message_lite.cc", + "third-party/sentencepiece/third_party/protobuf-lite/parse_context.cc", + "third-party/sentencepiece/third_party/protobuf-lite/repeated_field.cc", + "third-party/sentencepiece/third_party/protobuf-lite/status.cc", + "third-party/sentencepiece/third_party/protobuf-lite/statusor.cc", + "third-party/sentencepiece/third_party/protobuf-lite/stringpiece.cc", + "third-party/sentencepiece/third_party/protobuf-lite/stringprintf.cc", + "third-party/sentencepiece/third_party/protobuf-lite/structurally_valid.cc", + "third-party/sentencepiece/third_party/protobuf-lite/strutil.cc", + "third-party/sentencepiece/third_party/protobuf-lite/time.cc", + "third-party/sentencepiece/third_party/protobuf-lite/wire_format_lite.cc", + "third-party/sentencepiece/third_party/protobuf-lite/zero_copy_stream.cc", + "third-party/sentencepiece/third_party/protobuf-lite/zero_copy_stream_impl.cc", + "third-party/sentencepiece/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc", + ], + publicHeadersPath: "include", + cxxSettings: [ + .define("HAVE_PTHREAD", to: "1"), + .define("_USE_INTERNAL_STRING_VIEW"), + .headerSearchPath("swift-config"), + .headerSearchPath("third-party/sentencepiece"), + .headerSearchPath("third-party/sentencepiece/src"), + .headerSearchPath("third-party/sentencepiece/src/builtin_pb"), + .headerSearchPath("third-party/sentencepiece/third_party/protobuf-lite"), + .headerSearchPath("third-party/re2"), + .headerSearchPath("third-party/abseil-cpp"), + .headerSearchPath("third-party/json/single_include"), + .headerSearchPath("third-party/llama.cpp-unicode/include"), + .unsafeFlags(["-std=c++20", "-Wno-sign-compare", "-Wno-deprecated-declarations", "-Wno-attributes", "-include", "cstring"]), + ], + linkerSettings: [ + .linkedLibrary("c++"), + ] + ), + ], + cxxLanguageStandard: .cxx20 +) diff --git a/swift-config/config.h b/swift-config/config.h new file mode 100644 index 0000000..a31c2b7 --- /dev/null +++ b/swift-config/config.h @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +// Pre-generated config.h for Swift Package Manager builds. +// This replaces the CMake-generated config.h for sentencepiece. + +#ifndef CONFIG_H_ +#define CONFIG_H_ + +#define VERSION "0.2.1" +#define PACKAGE "sentencepiece" +#define PACKAGE_STRING "sentencepiece" + +#endif // CONFIG_H_