remi/Perplexica
1
0
Fork 0
mirror of https://github.com/ItzCrazyKns/Perplexica.git synced 2025-06-20 16:58:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

User customizable context window for ollama models.

Browse Source
This commit is contained in:
Willie Zutz
2025-02-15 16:03:24 -07:00
parent 115e6b2a71
commit 2bdcbf20fb
10 changed files with 118 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/routes/images.ts
View File

@ -5,6 +5,7 @@ import { getAvailableChatModelProviders } from '../lib/providers';
import { HumanMessage, AIMessage } from '@langchain/core/messages';
import logger from '../utils/logger';
import { ChatOpenAI } from '@langchain/openai';
import { ChatOllama } from '@langchain/community/chat_models/ollama';
import {
getCustomOpenaiApiKey,
getCustomOpenaiApiUrl,
@ -16,6 +17,7 @@ const router = express.Router();
interface ChatModel {
provider: string;
model: string;
ollamaContextWindow?: number;
}
interface ImageSearchBody {
@ -61,6 +63,10 @@ router.post('/', async (req, res) => {
) {
llm = chatModelProviders[chatModelProvider][chatModel]
.model as unknown as BaseChatModel | undefined;
if (llm instanceof ChatOllama) {
llm.numCtx = body.chatModel?.ollamaContextWindow || 2048;
}
}
if (!llm) {

6
src/routes/search.ts
View File

@ -15,12 +15,14 @@ import {
getCustomOpenaiApiUrl,
getCustomOpenaiModelName,
} from '../config';
import { ChatOllama } from '@langchain/community/chat_models/ollama';
const router = express.Router();
interface chatModel {
provider: string;
model: string;
ollamaContextWindow?: number;
customOpenAIKey?: string;
customOpenAIBaseURL?: string;
}
@ -78,6 +80,7 @@ router.post('/', async (req, res) => {
const embeddingModel =
body.embeddingModel?.model ||
Object.keys(embeddingModelProviders[embeddingModelProvider])[0];
const ollamaContextWindow = body.chatModel?.ollamaContextWindow || 2048;
let llm: BaseChatModel | undefined;
let embeddings: Embeddings | undefined;
@ -99,6 +102,9 @@ router.post('/', async (req, res) => {
) {
llm = chatModelProviders[chatModelProvider][chatModel]
.model as unknown as BaseChatModel | undefined;
if (llm instanceof ChatOllama) {
llm.numCtx = ollamaContextWindow;
}
}
if (

5
src/routes/suggestions.ts
View File

@ -10,12 +10,14 @@ import {
getCustomOpenaiApiUrl,
getCustomOpenaiModelName,
} from '../config';
import { ChatOllama } from '@langchain/community/chat_models/ollama';
const router = express.Router();
interface ChatModel {
provider: string;
model: string;
ollamaContextWindow?: number;
}
interface SuggestionsBody {
@ -60,6 +62,9 @@ router.post('/', async (req, res) => {
) {
llm = chatModelProviders[chatModelProvider][chatModel]
.model as unknown as BaseChatModel | undefined;
if (llm instanceof ChatOllama) {
llm.numCtx = body.chatModel?.ollamaContextWindow || 2048;
}
}
if (!llm) {

6
src/routes/videos.ts
View File

@ -10,12 +10,14 @@ import {
getCustomOpenaiApiUrl,
getCustomOpenaiModelName,
} from '../config';
import { ChatOllama } from '@langchain/community/chat_models/ollama';
const router = express.Router();
interface ChatModel {
provider: string;
model: string;
ollamaContextWindow?: number;
}
interface VideoSearchBody {
@ -61,6 +63,10 @@ router.post('/', async (req, res) => {
) {
llm = chatModelProviders[chatModelProvider][chatModel]
.model as unknown as BaseChatModel | undefined;
if (llm instanceof ChatOllama) {
llm.numCtx = body.chatModel?.ollamaContextWindow || 2048;
}
}
if (!llm) {

6
src/websocket/connectionManager.ts
View File

@ -14,6 +14,7 @@ import {
getCustomOpenaiApiUrl,
getCustomOpenaiModelName,
} from '../config';
import { ChatOllama } from '@langchain/community/chat_models/ollama';
export const handleConnection = async (
ws: WebSocket,
@ -42,6 +43,8 @@ export const handleConnection = async (
searchParams.get('embeddingModel') ||
Object.keys(embeddingModelProviders[embeddingModelProvider])[0];
const ollamaContextWindow = searchParams.get('ollamaContextWindow');
let llm: BaseChatModel | undefined;
let embeddings: Embeddings | undefined;
@ -52,6 +55,9 @@ export const handleConnection = async (
) {
llm = chatModelProviders[chatModelProvider][chatModel]
.model as unknown as BaseChatModel | undefined;
if (llm instanceof ChatOllama) {
llm.numCtx = ollamaContextWindow ? parseInt(ollamaContextWindow) : 2048;
}
} else if (chatModelProvider == 'custom_openai') {
const customOpenaiApiKey = getCustomOpenaiApiKey();
const customOpenaiApiUrl = getCustomOpenaiApiUrl();

75
ui/app/settings/page.tsx
View File

@ -23,6 +23,7 @@ interface SettingsType {
customOpenaiApiKey: string;
customOpenaiApiUrl: string;
customOpenaiModelName: string;
ollamaContextWindow: number;
}
interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
@ -112,6 +113,11 @@ const Page = () => {
const [automaticImageSearch, setAutomaticImageSearch] = useState(false);
const [automaticVideoSearch, setAutomaticVideoSearch] = useState(false);
const [savingStates, setSavingStates] = useState<Record<string, boolean>>({});
const [contextWindowSize, setContextWindowSize] = useState(2048);
const [isCustomContextWindow, setIsCustomContextWindow] = useState(false);
const predefinedContextSizes = [
1024, 2048, 3072, 4096, 8192, 16384, 32768, 65536, 131072
];
useEffect(() => {
const fetchConfig = async () => {
@ -123,6 +129,7 @@ const Page = () => {
});
const data = (await res.json()) as SettingsType;
setConfig(data);
const chatModelProvidersKeys = Object.keys(data.chatModelProviders || {});
@ -171,6 +178,9 @@ const Page = () => {
setAutomaticVideoSearch(
localStorage.getItem('autoVideoSearch') === 'true',
);
const storedContextWindow = parseInt(localStorage.getItem('ollamaContextWindow') ?? '2048');
setContextWindowSize(storedContextWindow);
setIsCustomContextWindow(!predefinedContextSizes.includes(storedContextWindow));
setIsLoading(false);
};
@ -331,6 +341,8 @@ const Page = () => {
localStorage.setItem('embeddingModelProvider', value);
} else if (key === 'embeddingModel') {
localStorage.setItem('embeddingModel', value);
} else if (key === 'ollamaContextWindow') {
localStorage.setItem('ollamaContextWindow', value.toString());
}
} catch (err) {
console.error('Failed to save:', err);
@ -548,6 +560,69 @@ const Page = () => {
];
})()}
/>
{selectedChatModelProvider === 'ollama' && (
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Chat Context Window Size
</p>
<Select
value={isCustomContextWindow ? 'custom' : contextWindowSize.toString()}
onChange={(e) => {
const value = e.target.value;
if (value === 'custom') {
setIsCustomContextWindow(true);
} else {
setIsCustomContextWindow(false);
const numValue = parseInt(value);
setContextWindowSize(numValue);
setConfig((prev) => ({
...prev!,
ollamaContextWindow: numValue,
}));
saveConfig('ollamaContextWindow', numValue);
}
}}
options={[
...predefinedContextSizes.map(size => ({
value: size.toString(),
label: `${size.toLocaleString()} tokens`
})),
{ value: 'custom', label: 'Custom...' }
]}
/>
{isCustomContextWindow && (
<div className="mt-2">
<Input
type="number"
min={512}
value={contextWindowSize}
placeholder="Custom context window size (minimum 512)"
isSaving={savingStates['ollamaContextWindow']}
onChange={(e) => {
// Allow any value to be typed
const value = parseInt(e.target.value) || contextWindowSize;
setContextWindowSize(value);
}}
onSave={(value) => {
// Validate only when saving
const numValue = Math.max(512, parseInt(value) || 2048);
setContextWindowSize(numValue);
setConfig((prev) => ({
...prev!,
ollamaContextWindow: numValue,
}));
saveConfig('ollamaContextWindow', numValue);
}}
/>
</div>
)}
<p className="text-xs text-black/60 dark:text-white/60 mt-0.5">
{isCustomContextWindow
? "Adjust the context window size for Ollama models (minimum 512 tokens)"
: "Adjust the context window size for Ollama models"}
</p>
</div>
)}
</div>
)}
</div>

2
ui/components/ChatWindow.tsx
View File

@ -197,6 +197,8 @@ const useSocket = (
'openAIBaseURL',
localStorage.getItem('openAIBaseURL')!,
);
} else {
searchParams.append('ollamaContextWindow', localStorage.getItem('ollamaContextWindow') || '2048')
}
searchParams.append('embeddingModel', embeddingModel!);

5
ui/components/SearchImages.tsx
View File

@ -33,9 +33,9 @@ const SearchImages = ({
const chatModelProvider = localStorage.getItem('chatModelProvider');
const chatModel = localStorage.getItem('chatModel');
const customOpenAIBaseURL = localStorage.getItem('openAIBaseURL');
const customOpenAIKey = localStorage.getItem('openAIApiKey');
const ollamaContextWindow = localStorage.getItem('ollamaContextWindow') || '2048';
const res = await fetch(
`${process.env.NEXT_PUBLIC_API_URL}/images`,
@ -54,6 +54,9 @@ const SearchImages = ({
customOpenAIBaseURL: customOpenAIBaseURL,
customOpenAIKey: customOpenAIKey,
}),
...(chatModelProvider === 'ollama' && {
ollamaContextWindow: parseInt(ollamaContextWindow),
}),
},
}),
},

5
ui/components/SearchVideos.tsx
View File

@ -48,9 +48,9 @@ const Searchvideos = ({
const chatModelProvider = localStorage.getItem('chatModelProvider');
const chatModel = localStorage.getItem('chatModel');
const customOpenAIBaseURL = localStorage.getItem('openAIBaseURL');
const customOpenAIKey = localStorage.getItem('openAIApiKey');
const ollamaContextWindow = localStorage.getItem('ollamaContextWindow') || '2048';
const res = await fetch(
`${process.env.NEXT_PUBLIC_API_URL}/videos`,
@ -69,6 +69,9 @@ const Searchvideos = ({
customOpenAIBaseURL: customOpenAIBaseURL,
customOpenAIKey: customOpenAIKey,
}),
...(chatModelProvider === 'ollama' && {
ollamaContextWindow: parseInt(ollamaContextWindow),
}),
},
}),
},

4
ui/lib/actions.ts
View File

@ -6,6 +6,7 @@ export const getSuggestions = async (chatHisory: Message[]) => {
const customOpenAIKey = localStorage.getItem('openAIApiKey');
const customOpenAIBaseURL = localStorage.getItem('openAIBaseURL');
const ollamaContextWindow = localStorage.getItem('ollamaContextWindow') || '2048';
const res = await fetch(`${process.env.NEXT_PUBLIC_API_URL}/suggestions`, {
method: 'POST',
@ -21,6 +22,9 @@ export const getSuggestions = async (chatHisory: Message[]) => {
customOpenAIKey,
customOpenAIBaseURL,
}),
...(chatModelProvider === 'ollama' && {
ollamaContextWindow: parseInt(ollamaContextWindow),
}),
},
}),
});