Update next.config.mjs

next.config.mjs

@@ -11,6 +11,13 @@ const nextConfig = {
     ],
   },
   serverExternalPackages: ['pdf-parse'],
+  outputFileTracingIncludes: {
+    '/api/**': [
+      './node_modules/@napi-rs/canvas/**',
+      './node_modules/@napi-rs/canvas-linux-x64-gnu/**',
+      './node_modules/@napi-rs/canvas-linux-x64-musl/**',
+    ],
+  },
   env: {
     NEXT_PUBLIC_VERSION: pkg.version,
   },

package.json

@@ -67,7 +67,9 @@
     "postcss": "^8",
     "prettier": "^3.2.5",
     "tailwindcss": "^3.3.0",
-    "typescript": "^5.9.3",
+    "typescript": "^5.9.3"
+  },
+  "optionalDependencies": {
     "@napi-rs/canvas": "^0.1.87"
   }
 }

src/app/api/images/route.ts

@@ -21,7 +21,10 @@ export const POST = async (req: Request) => {
 
     const images = await searchImages(
       {
-        chatHistory: body.chatHistory,
+        chatHistory: body.chatHistory.map(([role, content]) => ({
+          role: role === 'human' ? 'user' : 'assistant',
+          content,
+        })),
         query: body.query,
       },
       llm,

src/app/api/suggestions/route.ts

@@ -20,7 +20,10 @@ export const POST = async (req: Request) => {
 
     const suggestions = await generateSuggestions(
       {
-        chatHistory: body.chatHistory,
+        chatHistory: body.chatHistory.map(([role, content]) => ({
+          role: role === 'human' ? 'user' : 'assistant',
+          content,
+        })),
       },
       llm,
     );

src/app/api/videos/route.ts

@@ -21,7 +21,10 @@ export const POST = async (req: Request) => {
 
     const videos = await handleVideoSearch(
       {
-        chatHistory: body.chatHistory,
+        chatHistory: body.chatHistory.map(([role, content]) => ({
+          role: role === 'human' ? 'user' : 'assistant',
+          content,
+        })),
         query: body.query,
       },
       llm,

src/components/MessageBox.tsx

@@ -50,7 +50,14 @@ const MessageBox = ({
   dividerRef?: MutableRefObject<HTMLDivElement | null>;
   isLast: boolean;
 }) => {
-  const { loading, sendMessage, rewrite, messages, researchEnded } = useChat();
+  const {
+    loading,
+    sendMessage,
+    rewrite,
+    messages,
+    researchEnded,
+    chatHistory,
+  } = useChat();
 
   const parsedMessage = section.parsedTextBlocks.join('\n\n');
   const speechMessage = section.speechMessage || '';
@@ -265,11 +272,11 @@ const MessageBox = ({
           <div className="lg:sticky lg:top-20 flex flex-col items-center space-y-3 w-full lg:w-3/12 z-30 h-full pb-4">
             <SearchImages
               query={section.message.query}
-              chatHistory={messages}
+              chatHistory={chatHistory}
               messageId={section.message.messageId}
             />
             <SearchVideos
-              chatHistory={messages}
+              chatHistory={chatHistory}
               query={section.message.query}
               messageId={section.message.messageId}
             />

src/components/SearchImages.tsx

@@ -17,7 +17,7 @@ const SearchImages = ({
   messageId,
 }: {
   query: string;
-  chatHistory: Message[];
+  chatHistory: [string, string][];
   messageId: string;
 }) => {
   const [images, setImages] = useState<Image[] | null>(null);

src/components/SearchVideos.tsx

@@ -30,7 +30,7 @@ const Searchvideos = ({
   messageId,
 }: {
   query: string;
-  chatHistory: Message[];
+  chatHistory: [string, string][];
   messageId: string;
 }) => {
   const [videos, setVideos] = useState<Video[] | null>(null);

src/lib/hooks/useChat.tsx

@@ -175,7 +175,7 @@ const loadMessages = async (
   chatId: string,
   setMessages: (messages: Message[]) => void,
   setIsMessagesLoaded: (loaded: boolean) => void,
-  setChatHistory: (history: [string, string][]) => void,
+  chatHistory: React.MutableRefObject<[string, string][]>,
   setSources: (sources: string[]) => void,
   setNotFound: (notFound: boolean) => void,
   setFiles: (files: File[]) => void,
@@ -233,7 +233,7 @@ const loadMessages = async (
   setFiles(files);
   setFileIds(files.map((file: File) => file.fileId));
 
-  setChatHistory(history);
+  chatHistory.current = history;
   setSources(data.chat.sources);
   setIsMessagesLoaded(true);
 };
@@ -281,7 +281,7 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
 
   const [researchEnded, setResearchEnded] = useState(false);
 
-  const [chatHistory, setChatHistory] = useState<[string, string][]>([]);
+  const chatHistory = useRef<[string, string][]>([]);
   const [messages, setMessages] = useState<Message[]>([]);
 
   const [files, setFiles] = useState<File[]>([]);
@@ -402,7 +402,12 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
     });
   }, [messages]);
 
+  const isReconnectingRef = useRef(false);
+  const handledMessageEndRef = useRef<Set<string>>(new Set());
+
   const checkReconnect = async () => {
+    if (isReconnectingRef.current) return;
+
     setIsReady(true);
     console.debug(new Date(), 'app:ready');
 
@@ -414,6 +419,8 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
         setResearchEnded(false);
         setMessageAppeared(false);
 
+        isReconnectingRef.current = true;
+
         const res = await fetch(`/api/reconnect/${lastMsg.backendId}`, {
           method: 'POST',
         });
@@ -427,23 +434,27 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
 
         const messageHandler = getMessageHandler(lastMsg);
 
-        while (true) {
-          const { value, done } = await reader.read();
-          if (done) break;
+        try {
+          while (true) {
+            const { value, done } = await reader.read();
+            if (done) break;
 
-          partialChunk += decoder.decode(value, { stream: true });
+            partialChunk += decoder.decode(value, { stream: true });
 
-          try {
-            const messages = partialChunk.split('\n');
-            for (const msg of messages) {
-              if (!msg.trim()) continue;
-              const json = JSON.parse(msg);
-              messageHandler(json);
+            try {
+              const messages = partialChunk.split('\n');
+              for (const msg of messages) {
+                if (!msg.trim()) continue;
+                const json = JSON.parse(msg);
+                messageHandler(json);
+              }
+              partialChunk = '';
+            } catch (error) {
+              console.warn('Incomplete JSON, waiting for next chunk...');
             }
-            partialChunk = '';
-          } catch (error) {
-            console.warn('Incomplete JSON, waiting for next chunk...');
           }
+        } finally {
+          isReconnectingRef.current = false;
         }
       }
     }
@@ -463,7 +474,7 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
     if (params.chatId && params.chatId !== chatId) {
       setChatId(params.chatId);
       setMessages([]);
-      setChatHistory([]);
+      chatHistory.current = [];
       setFiles([]);
       setFileIds([]);
       setIsMessagesLoaded(false);
@@ -483,7 +494,7 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
         chatId,
         setMessages,
         setIsMessagesLoaded,
-        setChatHistory,
+        chatHistory,
         setSources,
         setNotFound,
         setFiles,
@@ -519,9 +530,7 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
 
     setMessages((prev) => prev.slice(0, index));
 
-    setChatHistory((prev) => {
-      return prev.slice(0, index * 2);
-    });
+    chatHistory.current = chatHistory.current.slice(0, index * 2);
 
     const messageToRewrite = messages[index];
     sendMessage(messageToRewrite.query, messageToRewrite.messageId, true);
@@ -621,12 +630,18 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
       }
 
       if (data.type === 'messageEnd') {
+        if (handledMessageEndRef.current.has(messageId)) {
+          return;
+        }
+
+        handledMessageEndRef.current.add(messageId);
+
         const currentMsg = messagesRef.current.find(
           (msg) => msg.messageId === messageId,
         );
 
         const newHistory: [string, string][] = [
-          ...chatHistory,
+          ...chatHistory.current,
           ['human', message.query],
           [
             'assistant',
@@ -635,7 +650,7 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
           ],
         ];
 
-        setChatHistory(newHistory);
+        chatHistory.current = newHistory;
 
         setMessages((prev) =>
           prev.map((msg) =>
@@ -652,13 +667,15 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
         const autoMediaSearch = getAutoMediaSearch();
 
         if (autoMediaSearch) {
-          document
-            .getElementById(`search-images-${lastMsg.messageId}`)
-            ?.click();
+          setTimeout(() => {
+            document
+              .getElementById(`search-images-${lastMsg.messageId}`)
+              ?.click();
 
-          document
-            .getElementById(`search-videos-${lastMsg.messageId}`)
-            ?.click();
+            document
+              .getElementById(`search-videos-${lastMsg.messageId}`)
+              ?.click();
+          }, 200);
         }
 
         // Check if there are sources and no suggestions
@@ -742,8 +759,11 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
         sources: sources,
         optimizationMode: optimizationMode,
         history: rewrite
-          ? chatHistory.slice(0, messageIndex === -1 ? undefined : messageIndex)
-          : chatHistory,
+          ? chatHistory.current.slice(
+              0,
+              messageIndex === -1 ? undefined : messageIndex,
+            )
+          : chatHistory.current,
         chatModel: {
           key: chatModelProvider.key,
           providerId: chatModelProvider.providerId,
@@ -790,7 +810,7 @@ export const ChatProvider = ({ children }: { children: React.ReactNode }) => {
       value={{
         messages,
         sections,
-        chatHistory,
+        chatHistory: chatHistory.current,
         files,
         fileIds,
         sources,

src/lib/prompts/media/image.ts

@@ -3,6 +3,7 @@ import { ChatTurnMessage } from '@/lib/types';
 export const imageSearchPrompt = `
 You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search the web for images.
 You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
+Make sure to make the querey standalone and not something very broad, use context from the answers in the conversation to make it specific so user can get best image search results.
 Output only the rephrased query in query key JSON format. Do not include any explanation or additional text.
 `;
 

src/lib/prompts/media/videos.ts

@@ -3,6 +3,7 @@ import { ChatTurnMessage } from '@/lib/types';
 export const videoSearchPrompt = `
 You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search Youtube for videos.
 You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
+Make sure to make the querey standalone and not something very broad, use context from the answers in the conversation to make it specific so user can get best video search results.
 Output only the rephrased query in query key JSON format. Do not include any explanation or additional text.
 `;