Support for Ollama context window configuration

2025-12-02 09:48:16 +00:00 · 2025-04-20 01:37:10 -06:00
parent 73b5e8832e
commit c0705d1d9e
10 changed files with 170 additions and 54 deletions
--- a/src/app/api/chat/route.ts
+++ b/src/app/api/chat/route.ts
@@ -20,6 +20,7 @@ import {
  getCustomOpenaiApiUrl,
  getCustomOpenaiModelName,
 } from '@/lib/config';
+import { ChatOllama } from '@langchain/ollama';
 import { searchHandlers } from '@/lib/search';

 export const runtime = 'nodejs';
@@ -34,6 +35,7 @@ type Message = {
 type ChatModel = {
  provider: string;
  name: string;
+  ollamaContextWindow?: number;
 };

 type EmbeddingModel = {
@@ -232,6 +234,11 @@ export const POST = async (req: Request) => {
      }) as unknown as BaseChatModel;
    } else if (chatModelProvider && chatModel) {
      llm = chatModel.model;
+      
+      // Set context window size for Ollama models
+      if (llm instanceof ChatOllama && body.chatModel?.provider === 'ollama') {
+        llm.numCtx = body.chatModel.ollamaContextWindow || 2048;
+      }
    }

    if (!llm) {
--- a/src/app/api/search/route.ts
+++ b/src/app/api/search/route.ts
@@ -13,12 +13,14 @@ import {
  getCustomOpenaiModelName,
 } from '@/lib/config';
 import { searchHandlers } from '@/lib/search';
+import { ChatOllama } from '@langchain/ollama';

 interface chatModel {
  provider: string;
  name: string;
  customOpenAIKey?: string;
  customOpenAIBaseURL?: string;
+  ollamaContextWindow?: number;
 }

 interface embeddingModel {
@@ -96,6 +98,10 @@ export const POST = async (req: Request) => {
      llm = chatModelProviders[chatModelProvider][chatModel]
        .model as unknown as BaseChatModel | undefined;
    }
+    
+    if (llm instanceof ChatOllama && body.chatModel?.provider === 'ollama') {
+      llm.numCtx = body.chatModel.ollamaContextWindow || 2048;
+    }

    if (
      embeddingModelProviders[embeddingModelProvider] &&
--- a/src/app/api/suggestions/route.ts
+++ b/src/app/api/suggestions/route.ts
@@ -8,10 +8,12 @@ import { getAvailableChatModelProviders } from '@/lib/providers';
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
 import { ChatOpenAI } from '@langchain/openai';
+import { ChatOllama } from '@langchain/ollama';

 interface ChatModel {
  provider: string;
  model: string;
+  ollamaContextWindow?: number;
 }

 interface SuggestionsGenerationBody {
@@ -57,6 +59,10 @@ export const POST = async (req: Request) => {
      }) as unknown as BaseChatModel;
    } else if (chatModelProvider && chatModel) {
      llm = chatModel.model;
+      // Set context window size for Ollama models
+      if (llm instanceof ChatOllama && body.chatModel?.provider === 'ollama') {
+        llm.numCtx = body.chatModel.ollamaContextWindow || 2048;
+      }
    }

    if (!llm) {
--- a/src/components/ChatWindow.tsx
+++ b/src/components/ChatWindow.tsx
@@ -494,6 +494,8 @@ const sendMessage = async (
      }
    };

+    const ollamaContextWindow = localStorage.getItem('ollamaContextWindow') || '2048';
+
    const res = await fetch('/api/chat', {
      method: 'POST',
      headers: {
@@ -514,6 +516,9 @@ const sendMessage = async (
        chatModel: {
          name: chatModelProvider.name,
          provider: chatModelProvider.provider,
+          ...(chatModelProvider.provider === 'ollama' && {
+            ollamaContextWindow: parseInt(ollamaContextWindow),
+          }),
        },
        embeddingModel: {
          name: embeddingModelProvider.name,
--- a/src/components/MessageInput.tsx
+++ b/src/components/MessageInput.tsx
@@ -80,49 +80,11 @@ const MessageInput = ({
      }}
      className={cn(
        'bg-light-secondary dark:bg-dark-secondary p-4 flex items-center border border-light-200 dark:border-dark-200',
-        mode === 'multi' ? 'flex-col rounded-lg' : 'flex-row rounded-full',
+        mode === 'multi' ? 'flex-col rounded-lg' : 'flex-col md:flex-row rounded-lg md:rounded-full',
      )}
    >
      {mode === 'single' && (
-        <div className="flex flex-row items-center space-x-2">
-          <AttachSmall
-            fileIds={fileIds}
-            setFileIds={setFileIds}
-            files={files}
-            setFiles={setFiles}
-          />
-          <Optimization
-            optimizationMode={optimizationMode}
-            setOptimizationMode={setOptimizationMode}
-          />
-        </div>
-      )}
-      <TextareaAutosize
-        ref={inputRef}
-        value={message}
-        onChange={(e) => setMessage(e.target.value)}
-        onHeightChange={(height, props) => {
-          setTextareaRows(Math.ceil(height / props.rowHeight));
-        }}
-        className="transition bg-transparent dark:placeholder:text-white/50 placeholder:text-sm text-sm dark:text-white resize-none focus:outline-none w-full px-2 max-h-24 lg:max-h-36 xl:max-h-48 flex-grow flex-shrink"
-        placeholder="Ask a follow-up"
-      />
-      {mode === 'single' && (
-        <div className="flex flex-row items-center space-x-4">
-          <CopilotToggle
-            copilotEnabled={copilotEnabled}
-            setCopilotEnabled={setCopilotEnabled}
-          />
-          <button
-            disabled={message.trim().length === 0 || loading}
-            className="bg-[#24A0ED] text-white disabled:text-black/50 dark:disabled:text-white/50 hover:bg-opacity-85 transition duration-100 disabled:bg-[#e0e0dc79] dark:disabled:bg-[#ececec21] rounded-full p-2"
-          >
-            <ArrowUp className="bg-background" size={17} />
-          </button>
-        </div>
-      )}
-      {mode === 'multi' && (
-        <div className="flex flex-row items-center justify-between w-full pt-2">
+        <div className="flex flex-row items-center justify-between w-full mb-2 md:mb-0 md:w-auto">
          <div className="flex flex-row items-center space-x-2">
            <AttachSmall
              fileIds={fileIds}
@@ -135,14 +97,75 @@ const MessageInput = ({
              setOptimizationMode={setOptimizationMode}
            />
          </div>
-          <div className="flex flex-row items-center space-x-4">
+          <div className="md:hidden">
            <CopilotToggle
              copilotEnabled={copilotEnabled}
              setCopilotEnabled={setCopilotEnabled}
            />
+          </div>
+        </div>
+      )}
+      <div className="flex flex-row items-center w-full">
+        <TextareaAutosize
+          ref={inputRef}
+          value={message}
+          onChange={(e) => setMessage(e.target.value)}
+          onHeightChange={(height, props) => {
+            setTextareaRows(Math.ceil(height / props.rowHeight));
+          }}
+          className="transition bg-transparent dark:placeholder:text-white/50 placeholder:text-sm text-sm dark:text-white resize-none focus:outline-none w-full px-2 max-h-24 lg:max-h-36 xl:max-h-48 flex-grow flex-shrink"
+          placeholder="Ask a follow-up"
+        />
+        {mode === 'single' && (
+          <div className="flex flex-row items-center space-x-4">
+            <div className="hidden md:block">
+              <CopilotToggle
+                copilotEnabled={copilotEnabled}
+                setCopilotEnabled={setCopilotEnabled}
+              />
+            </div>
            <button
              disabled={message.trim().length === 0 || loading}
-              className="bg-[#24A0ED] text-white text-black/50 dark:disabled:text-white/50 hover:bg-opacity-85 transition duration-100 disabled:bg-[#e0e0dc79] dark:disabled:bg-[#ececec21] rounded-full p-2"
+              className="bg-[#24A0ED] text-white disabled:text-black/50 dark:disabled:text-white/50 hover:bg-opacity-85 transition duration-100 disabled:bg-[#e0e0dc79] dark:disabled:bg-[#ececec21] rounded-full p-2"
+            >
+              <ArrowUp className="bg-background" size={17} />
+            </button>
+          </div>
+        )}
+      </div>
+      
+      {mode === 'multi' && (
+        <div className="flex flex-col md:flex-row items-start md:items-center justify-between w-full pt-2">
+          <div className="flex flex-row items-center justify-between w-full md:w-auto mb-2 md:mb-0">
+            <div className="flex flex-row items-center space-x-2">
+              <AttachSmall
+                fileIds={fileIds}
+                setFileIds={setFileIds}
+                files={files}
+                setFiles={setFiles}
+              />
+              <Optimization
+                optimizationMode={optimizationMode}
+                setOptimizationMode={setOptimizationMode}
+              />
+            </div>
+            <div className="md:hidden">
+              <CopilotToggle
+                copilotEnabled={copilotEnabled}
+                setCopilotEnabled={setCopilotEnabled}
+              />
+            </div>
+          </div>
+          <div className="flex flex-row items-center space-x-4 self-end">
+            <div className="hidden md:block">
+              <CopilotToggle
+                copilotEnabled={copilotEnabled}
+                setCopilotEnabled={setCopilotEnabled}
+              />
+            </div>
+            <button
+              disabled={message.trim().length === 0 || loading}
+              className="bg-[#24A0ED] text-white disabled:text-black/50 dark:disabled:text-white/50 hover:bg-opacity-85 transition duration-100 disabled:bg-[#e0e0dc79] dark:disabled:bg-[#ececec21] rounded-full p-2"
            >
              <ArrowUp className="bg-background" size={17} />
            </button>
--- a/src/lib/providers/ollama.ts
+++ b/src/lib/providers/ollama.ts
@@ -6,8 +6,8 @@ export const PROVIDER_INFO = {
  key: 'ollama',
  displayName: 'Ollama',
 };
-import { ChatOllama } from '@langchain/community/chat_models/ollama';
-import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama';
+import { ChatOllama } from '@langchain/ollama';
+import { OllamaEmbeddings } from '@langchain/ollama';

 export const loadOllamaChatModels = async () => {
  const ollamaApiEndpoint = getOllamaApiEndpoint();
--- a/src/lib/search/metaSearchAgent.ts
+++ b/src/lib/search/metaSearchAgent.ts
@@ -239,7 +239,6 @@ class MetaSearchAgent implements MetaSearchAgentType {
    optimizationMode: 'speed' | 'balanced' | 'quality',
    systemInstructions: string,
  ) {
-      console.log("optimization mode", optimizationMode);
    return RunnableSequence.from([
      RunnableMap.from({
        systemInstructions: () => systemInstructions,