From 54f3a7d2b8be39fdca70f5fc225fb702aae8224a Mon Sep 17 00:00:00 2001 From: ItzCrazyKns <95534749+ItzCrazyKns@users.noreply.github.com> Date: Tue, 31 Mar 2026 20:22:57 +0530 Subject: [PATCH] feat(split-text): export token count function --- src/lib/utils/splitText.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/utils/splitText.ts b/src/lib/utils/splitText.ts index 796bf4b4..08cf9b7c 100644 --- a/src/lib/utils/splitText.ts +++ b/src/lib/utils/splitText.ts @@ -4,7 +4,7 @@ const splitRegex = /(?<=\. |\n|! |\? |; |:\s|\d+\.\s|- |\* )/g; const enc = getEncoding('cl100k_base'); -const getTokenCount = (text: string): number => { +export const getTokenCount = (text: string): number => { try { return enc.encode(text).length; } catch {