diff --git a/src/lib/utils/splitText.ts b/src/lib/utils/splitText.ts index 796bf4b4..08cf9b7c 100644 --- a/src/lib/utils/splitText.ts +++ b/src/lib/utils/splitText.ts @@ -4,7 +4,7 @@ const splitRegex = /(?<=\. |\n|! |\? |; |:\s|\d+\.\s|- |\* )/g; const enc = getEncoding('cl100k_base'); -const getTokenCount = (text: string): number => { +export const getTokenCount = (text: string): number => { try { return enc.encode(text).length; } catch {