diff --git a/collector/utils/tokenizer/index.js b/collector/utils/tokenizer/index.js index 2086be2574b..e1231cc0cdf 100644 --- a/collector/utils/tokenizer/index.js +++ b/collector/utils/tokenizer/index.js @@ -2,7 +2,7 @@ const { getEncoding } = require("js-tiktoken"); class TikTokenTokenizer { static MAX_KB_ESTIMATE = 10; - static DIVISOR = 8; + static DIVISOR = 4; constructor() { if (TikTokenTokenizer.instance) {