remi/Perplexica
1
0
Fork 0
mirror of https://github.com/ItzCrazyKns/Perplexica.git synced 2026-04-10 05:44:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(split-text): export token count function

Browse Source
This commit is contained in:
ItzCrazyKns
2026-03-31 20:22:57 +05:30
parent 1a124b8b07
commit 54f3a7d2b8
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/lib/utils/splitText.ts
View File

@@ -4,7 +4,7 @@ const splitRegex = /(?<=\. |\n|! |\? |; |:\s|\d+\.\s|- |\* )/g;
const enc = getEncoding('cl100k_base'); const enc = getEncoding('cl100k_base');
const getTokenCount = (text: string): number => { export const getTokenCount = (text: string): number => {
try { try {
return enc.encode(text).length; return enc.encode(text).length;
} catch { } catch {