From 4daffd50a91222d96846332e865c9f89bd2835e1 Mon Sep 17 00:00:00 2001 From: Varsha Kumar Date: Mon, 29 Jun 2026 19:36:02 +0000 Subject: [PATCH 1/3] add build script for tiktoken 0.13.0 Signed-off-by: Varsha Kumar --- t/tiktoken/build_info.json | 9 +- t/tiktoken/tiktoken_v0.13.0_ubi_9.6.sh | 124 +++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 3 deletions(-) create mode 100755 t/tiktoken/tiktoken_v0.13.0_ubi_9.6.sh diff --git a/t/tiktoken/build_info.json b/t/tiktoken/build_info.json index 92ff85aad8..f8a0402dab 100644 --- a/t/tiktoken/build_info.json +++ b/t/tiktoken/build_info.json @@ -1,8 +1,8 @@ { - "maintainer": "aastha-sharma2", + "maintainer": "varsha.kumar@ibm.com", "package_name": "tiktoken", "github_url": "https://github.com/openai/tiktoken", - "version": "0.7.0", + "version": "0.13.0", "default_branch": "main", "build_script": "tiktoken_ubi_9.3.sh", "wheel_build" : true, @@ -10,10 +10,13 @@ "docker_build": false, "validate_build_script": true, "use_non_root_user": false, + "v0.13.0":{ + "build_script": "tiktoken_v0.13.0_ubi_9.6.sh" + }, "v0.9.0" :{ "build_script": "tiktoken_v0.9.0_ubi_9.6.sh" }, "*": { - "build_script": "tiktoken_ubi_9.3.sh" + "build_script": "tiktoken_ubi_9.6.sh" } } diff --git a/t/tiktoken/tiktoken_v0.13.0_ubi_9.6.sh b/t/tiktoken/tiktoken_v0.13.0_ubi_9.6.sh new file mode 100755 index 0000000000..95cb80591f --- /dev/null +++ b/t/tiktoken/tiktoken_v0.13.0_ubi_9.6.sh @@ -0,0 +1,124 @@ +#!/bin/bash -e +# ----------------------------------------------------------------------------- +# +# Package : tiktoken +# Version : v0.13.0 +# Source repo : https://github.com/openai/tiktoken +# Tested on : UBI:9.6 +# Language : Python +# Ci-Check : True +# Script License: Apache License, Version 2 or later +# Maintainer : Varsha Kumar +# +# Disclaimer: This script has been tested in root mode on given +# ========== platform using the mentioned version of the package. +# It may not work as expected with newer versions of the +# package and/or distribution. In such case, please +# contact "Maintainer" of this script. +# +# ---------------------------------------------------------------------------- + +# Variables +PACKAGE_NAME=tiktoken +PACKAGE_VERSION=${1:-0.13.0} +PACKAGE_URL=https://github.com/openai/tiktoken.git +PACKAGE_DIR=tiktoken + +# Install dependencies +yum install -y git python3.12 python3.12-devel python3.12-pip gcc-toolset-13 make wget sudo cmake +python3.12 -m pip install --upgrade pip +python3.12 -m pip install pytest tox nox hypothesis + +# Install Rust +echo "Installing Rust" +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +source "$HOME/.cargo/env" +rustup update +echo "Installed Rust" + + +export PATH=$PATH:/usr/local/bin/ +export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH +export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH + +OS_NAME=$(grep ^PRETTY_NAME /etc/os-release | cut -d= -f2) +SOURCE=Github + +# Clone or extract the package +if [[ "$PACKAGE_URL" == *github.com* ]]; then + if [ -d "$PACKAGE_DIR" ]; then + cd "$PACKAGE_DIR" || exit + else + if ! git clone "$PACKAGE_URL" "$PACKAGE_DIR"; then + echo "------------------$PACKAGE_NAME:clone_fails---------------------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | $SOURCE | Fail | Clone_Fails" + exit 1 + fi + cd "$PACKAGE_DIR" || exit + git checkout "$PACKAGE_VERSION" || exit + fi +else + if [ -d "$PACKAGE_DIR" ]; then + cd "$PACKAGE_DIR" || exit + else + if ! curl -L "$PACKAGE_URL" -o "$PACKAGE_DIR.tar.gz"; then + echo "------------------$PACKAGE_NAME:download_fails---------------------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | $SOURCE | Fail | Download_Fails" + exit 1 + fi + mkdir "$PACKAGE_DIR" + if ! tar -xzf "$PACKAGE_DIR.tar.gz" -C "$PACKAGE_DIR" --strip-components=1; then + echo "------------------$PACKAGE_NAME:untar_fails---------------------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | $SOURCE | Fail | Untar_Fails" + exit 1 + fi + cd "$PACKAGE_DIR" || exit + fi +fi + +# Install the package +if ! python3.12 -m pip install ./; then + echo "------------------$PACKAGE_NAME:install_fails------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | $SOURCE | Fail | Install_Failed" + exit 1 +fi + +# ------------------ Unified Test Execution Block ------------------ + +test_status=1 # 0 = success, non-zero = failure + +# Run pytest if any matching test files found +if ls */test_*.py > /dev/null 2>&1 && [ $test_status -ne 0 ]; then + echo "Running pytest..." + # Creating temporary directory because of circular import error + (cp -r tests /tmp/tiktoken_tests && cd /tmp && python3.12 -m pytest tiktoken_tests/) && test_status=0 || test_status=$? +fi + +# Run tox if tox.ini is present and previous tests failed +if [ -f "tox.ini" ] && [ $test_status -ne 0 ]; then + echo "Running tox..." + (python3.12 -m tox -e py39) && test_status=0 || test_status=$? +fi + +# Run nox if noxfile.py is present and previous tests failed +if [ -f "noxfile.py" ] && [ $test_status -ne 0 ]; then + echo "Running nox..." + (python3.12 -m nox) && test_status=0 || test_status=$? +fi + +# Final test result output +if [ $test_status -eq 0 ]; then + echo "------------------$PACKAGE_NAME:install_and_test_both_success-------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | $SOURCE | Pass | Both_Install_and_Test_Success" + exit 0 +else + echo "------------------$PACKAGE_NAME:install_success_but_test_fails---------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | $SOURCE | Fail | Install_success_but_test_Fails" + exit 2 +fi \ No newline at end of file From b87317a047dfe99054b26ae3e2306e000f09f76c Mon Sep 17 00:00:00 2001 From: Varsha Kumar Date: Tue, 30 Jun 2026 14:14:38 +0000 Subject: [PATCH 2/3] Changed line 20 to tiktoken_ubi_9.3.sh Signed-off-by: Varsha Kumar --- t/tiktoken/build_info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/tiktoken/build_info.json b/t/tiktoken/build_info.json index f8a0402dab..1acf470ac9 100644 --- a/t/tiktoken/build_info.json +++ b/t/tiktoken/build_info.json @@ -17,6 +17,6 @@ "build_script": "tiktoken_v0.9.0_ubi_9.6.sh" }, "*": { - "build_script": "tiktoken_ubi_9.6.sh" + "build_script": "tiktoken_ubi_9.3.sh" } } From fd165214be785fce56243067446dd764b6c702b8 Mon Sep 17 00:00:00 2001 From: Varsha Kumar Date: Tue, 30 Jun 2026 15:22:51 +0000 Subject: [PATCH 3/3] Added suggestion in line 13 Signed-off-by: Varsha Kumar --- t/tiktoken/build_info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/tiktoken/build_info.json b/t/tiktoken/build_info.json index 1acf470ac9..279ae8c442 100644 --- a/t/tiktoken/build_info.json +++ b/t/tiktoken/build_info.json @@ -10,7 +10,7 @@ "docker_build": false, "validate_build_script": true, "use_non_root_user": false, - "v0.13.0":{ + "0.13.0":{ "build_script": "tiktoken_v0.13.0_ubi_9.6.sh" }, "v0.9.0" :{