Merge branch 'ItzCrazyKns:master' into master

README.md

@@ -1,5 +1,21 @@
 # 🚀 Perplexica - An AI-powered search engine 🔎 <!-- omit in toc -->
 
+<div align="center" markdown="1">
+   <sup>Special thanks to:</sup>
+   <br>
+   <br>
+   <a href="https://www.warp.dev/perplexica">
+      <img alt="Warp sponsorship" width="400" src="https://github.com/user-attachments/assets/775dd593-9b5f-40f1-bf48-479faff4c27b">
+   </a>
+
+### [Warp, the AI Devtool that lives in your terminal](https://www.warp.dev/perplexica)
+
+[Available for MacOS, Linux, & Windows](https://www.warp.dev/perplexica)
+
+</div>
+
+<hr/>
+
 [![Discord](https://dcbadge.vercel.app/api/server/26aArMy8tT?style=flat&compact=true)](https://discord.gg/26aArMy8tT)
 
 ![preview](.assets/perplexica-screenshot.png?)
@@ -143,7 +159,6 @@ Perplexica runs on Next.js and handles all API requests. It works right away on
 
 [![Deploy to Sealos](https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg)](https://usw.sealos.io/?openapp=system-template%3FtemplateName%3Dperplexica)
 [![Deploy to RepoCloud](https://d16t0pc4846x52.cloudfront.net/deploylobe.svg)](https://repocloud.io/details/?app_id=267)
-[![Run on ClawCloud](https://raw.githubusercontent.com/ClawCloud/Run-Template/refs/heads/main/Run-on-ClawCloud.svg)](https://template.run.claw.cloud/?referralCode=U11MRQ8U9RM4&openapp=system-fastdeploy%3FtemplateName%3Dperplexica)
 
 ## Upcoming Features
 

sample.config.toml

@@ -25,8 +25,9 @@ API_URL = "" # Ollama API URL - http://host.docker.internal:11434
 [MODELS.DEEPSEEK]
 API_KEY = ""
 
-[MODELS.LM_STUDIO]
-API_URL = "" # LM Studio API URL - http://host.docker.internal:1234
-
 [API_ENDPOINTS]
 SEARXNG = "" # SearxNG API URL - http://localhost:32768
+TAVILY = "" # Tavily API key
+
+[SEARCH]
+ENGINE = "searxng" # "searxng" or "tavily"

src/app/api/config/route.ts

@@ -8,7 +8,8 @@ import {
   getOllamaApiEndpoint,
   getOpenaiApiKey,
   getDeepseekApiKey,
-  getLMStudioApiEndpoint,
+  getSearchEngine,
+  getTavilyApiKey,
   updateConfig,
 } from '@/lib/config';
 import {
@@ -52,7 +53,6 @@ export const GET = async (req: Request) => {
 
     config['openaiApiKey'] = getOpenaiApiKey();
     config['ollamaApiUrl'] = getOllamaApiEndpoint();
-    config['lmStudioApiUrl'] = getLMStudioApiEndpoint();
     config['anthropicApiKey'] = getAnthropicApiKey();
     config['groqApiKey'] = getGroqApiKey();
     config['geminiApiKey'] = getGeminiApiKey();
@@ -60,6 +60,8 @@ export const GET = async (req: Request) => {
     config['customOpenaiApiUrl'] = getCustomOpenaiApiUrl();
     config['customOpenaiApiKey'] = getCustomOpenaiApiKey();
     config['customOpenaiModelName'] = getCustomOpenaiModelName();
+    config['searchEngine'] = getSearchEngine();
+    config['tavilyApiKey'] = getTavilyApiKey();
 
     return Response.json({ ...config }, { status: 200 });
   } catch (err) {
@@ -95,15 +97,18 @@ export const POST = async (req: Request) => {
         DEEPSEEK: {
           API_KEY: config.deepseekApiKey,
         },
-        LM_STUDIO: {
-          API_URL: config.lmStudioApiUrl,
-        },
         CUSTOM_OPENAI: {
           API_URL: config.customOpenaiApiUrl,
           API_KEY: config.customOpenaiApiKey,
           MODEL_NAME: config.customOpenaiModelName,
         },
       },
+      SEARCH: {
+        ENGINE: config.searchEngine,
+      },
+      API_ENDPOINTS: {
+        TAVILY: config.tavilyApiKey || '',
+      },
     };
 
     updateConfig(updatedConfig);

src/app/api/discover/route.ts

@@ -1,4 +1,4 @@
-import { searchSearxng } from '@/lib/searxng';
+import { searchSearxng } from '../../../lib/searchEngines/searxng';
 
 const articleWebsites = [
   'yahoo.com',

src/app/settings/page.tsx

@@ -7,7 +7,6 @@ import { Switch } from '@headlessui/react';
 import ThemeSwitcher from '@/components/theme/Switcher';
 import { ImagesIcon, VideoIcon } from 'lucide-react';
 import Link from 'next/link';
-import { PROVIDER_METADATA } from '@/lib/providers';
 
 interface SettingsType {
   chatModelProviders: {
@@ -21,11 +20,12 @@ interface SettingsType {
   anthropicApiKey: string;
   geminiApiKey: string;
   ollamaApiUrl: string;
-  lmStudioApiUrl: string;
   deepseekApiKey: string;
   customOpenaiApiKey: string;
   customOpenaiApiUrl: string;
   customOpenaiModelName: string;
+  searchEngine: string;
+  tavilyApiKey?: string;
 }
 
 interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
@@ -147,6 +147,7 @@ const Page = () => {
   const [automaticImageSearch, setAutomaticImageSearch] = useState(false);
   const [automaticVideoSearch, setAutomaticVideoSearch] = useState(false);
   const [systemInstructions, setSystemInstructions] = useState<string>('');
+  const [searchEngine, setSearchEngine] = useState<string>('searxng');
   const [savingStates, setSavingStates] = useState<Record<string, boolean>>({});
 
   useEffect(() => {
@@ -209,6 +210,7 @@ const Page = () => {
       );
 
       setSystemInstructions(localStorage.getItem('systemInstructions')!);
+      setSearchEngine(localStorage.getItem('searchEngine') || 'searxng');
 
       setIsLoading(false);
     };
@@ -368,6 +370,10 @@ const Page = () => {
         localStorage.setItem('embeddingModel', value);
       } else if (key === 'systemInstructions') {
         localStorage.setItem('systemInstructions', value);
+      } else if (key === 'searchEngine') {
+        localStorage.setItem('searchEngine', value);
+      } else if (key === 'tavilyApiKey') {
+        localStorage.setItem('tavilyApiKey', value);
       }
     } catch (err) {
       console.error('Failed to save:', err);
@@ -510,6 +516,32 @@ const Page = () => {
                     />
                   </Switch>
                 </div>
+
+                <div className="flex flex-col space-y-1 mt-2">
+                  <p className="text-black/70 dark:text-white/70 text-sm">
+                    Search Engine
+                  </p>
+                  <Select
+                    value={searchEngine}
+                    onChange={(e) => {
+                      const value = e.target.value;
+                      setSearchEngine(value);
+                      saveConfig('searchEngine', value);
+                    }}
+                    options={[
+                      { value: 'searxng', label: 'SearxNG' },
+                      ...(config.tavilyApiKey ? [{ value: 'tavily', label: 'Tavily' }] : []),
+                    ]}
+                  />
+                  <p className="text-xs text-black/60 dark:text-white/60 mt-1">
+                    Select which search engine to use for web searches
+                  </p>
+                  {searchEngine === 'tavily' && !config.tavilyApiKey && (
+                    <p className="text-xs text-red-500 mt-1">
+                      Tavily API key is required to use this search engine
+                    </p>
+                  )}
+                </div>
               </div>
             </SettingsSection>
 
@@ -550,9 +582,8 @@ const Page = () => {
                         (provider) => ({
                           value: provider,
                           label:
-                            (PROVIDER_METADATA as any)[provider]?.displayName ||
                             provider.charAt(0).toUpperCase() +
-                              provider.slice(1),
+                            provider.slice(1),
                         }),
                       )}
                     />
@@ -693,9 +724,8 @@ const Page = () => {
                         (provider) => ({
                           value: provider,
                           label:
-                            (PROVIDER_METADATA as any)[provider]?.displayName ||
                             provider.charAt(0).toUpperCase() +
-                              provider.slice(1),
+                            provider.slice(1),
                         }),
                       )}
                     />
@@ -863,22 +893,29 @@ const Page = () => {
                   />
                 </div>
 
+                <div className="flex flex-col space-y-1 mt-4 pt-4 border-t border-light-200 dark:border-dark-200">
+                  <p className="text-black/90 dark:text-white/90 font-medium">Search Engine API Keys</p>
+                  <p className="text-sm text-black/60 dark:text-white/60 mt-0.5">
+                    API keys for search engines used in the application
+                  </p>
+                </div>
+
                 <div className="flex flex-col space-y-1">
                   <p className="text-black/70 dark:text-white/70 text-sm">
-                    LM Studio API URL
+                    Tavily API Key
                   </p>
                   <Input
                     type="text"
-                    placeholder="LM Studio API URL"
+                    placeholder="Tavily API key"
-                    value={config.lmStudioApiUrl}
+                    value={config.tavilyApiKey || ''}
-                    isSaving={savingStates['lmStudioApiUrl']}
+                    isSaving={savingStates['tavilyApiKey']}
                     onChange={(e) => {
                       setConfig((prev) => ({
                         ...prev!,
-                        lmStudioApiUrl: e.target.value,
+                        tavilyApiKey: e.target.value,
                       }));
                     }}
-                    onSave={(value) => saveConfig('lmStudioApiUrl', value)}
+                    onSave={(value) => saveConfig('tavilyApiKey', value)}
                   />
                 </div>
               </div>

src/lib/chains/imageSearchAgent.ts

@@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts';
 import formatChatHistoryAsString from '../utils/formatHistory';
 import { BaseMessage } from '@langchain/core/messages';
 import { StringOutputParser } from '@langchain/core/output_parsers';
-import { searchSearxng } from '../searxng';
+import { searchSearxng } from '../searchEngines/searxng';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const imageSearchChainPrompt = `

src/lib/chains/videoSearchAgent.ts

@@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts';
 import formatChatHistoryAsString from '../utils/formatHistory';
 import { BaseMessage } from '@langchain/core/messages';
 import { StringOutputParser } from '@langchain/core/output_parsers';
-import { searchSearxng } from '../searxng';
+import { searchSearxng } from '../searchEngines/searxng';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const VideoSearchChainPrompt = `

src/lib/config.ts

@@ -1,14 +1,7 @@
+import fs from 'fs';
+import path from 'path';
 import toml from '@iarna/toml';
 
-// Use dynamic imports for Node.js modules to prevent client-side errors
-let fs: any;
-let path: any;
-if (typeof window === 'undefined') {
-  // We're on the server
-  fs = require('fs');
-  path = require('path');
-}
-
 const configFileName = 'config.toml';
 
 interface Config {
@@ -35,9 +28,6 @@ interface Config {
     DEEPSEEK: {
       API_KEY: string;
     };
-    LM_STUDIO: {
-      API_URL: string;
-    };
     CUSTOM_OPENAI: {
       API_URL: string;
       API_KEY: string;
@@ -46,6 +36,10 @@ interface Config {
   };
   API_ENDPOINTS: {
     SEARXNG: string;
+    TAVILY: string;
+  };
+  SEARCH: {
+    ENGINE: string;
   };
 }
 
@@ -53,17 +47,10 @@ type RecursivePartial<T> = {
   [P in keyof T]?: RecursivePartial<T[P]>;
 };
 
-const loadConfig = () => {
+const loadConfig = () =>
-  // Server-side only
+  toml.parse(
-  if (typeof window === 'undefined') {
+    fs.readFileSync(path.join(process.cwd(), `${configFileName}`), 'utf-8'),
-    return toml.parse(
+  ) as any as Config;
-      fs.readFileSync(path.join(process.cwd(), `${configFileName}`), 'utf-8'),
-    ) as any as Config;
-  }
-
-  // Client-side fallback - settings will be loaded via API
-  return {} as Config;
-};
 
 export const getSimilarityMeasure = () =>
   loadConfig().GENERAL.SIMILARITY_MEASURE;
@@ -81,6 +68,12 @@ export const getGeminiApiKey = () => loadConfig().MODELS.GEMINI.API_KEY;
 export const getSearxngApiEndpoint = () =>
   process.env.SEARXNG_API_URL || loadConfig().API_ENDPOINTS.SEARXNG;
 
+export const getTavilyApiKey = () =>
+  process.env.TAVILY_API_KEY || loadConfig().API_ENDPOINTS.TAVILY;
+
+export const getSearchEngine = () =>
+  process.env.SEARCH_ENGINE || loadConfig().SEARCH?.ENGINE || 'searxng';
+
 export const getOllamaApiEndpoint = () => loadConfig().MODELS.OLLAMA.API_URL;
 
 export const getDeepseekApiKey = () => loadConfig().MODELS.DEEPSEEK.API_KEY;
@@ -94,9 +87,6 @@ export const getCustomOpenaiApiUrl = () =>
 export const getCustomOpenaiModelName = () =>
   loadConfig().MODELS.CUSTOM_OPENAI.MODEL_NAME;
 
-export const getLMStudioApiEndpoint = () =>
-  loadConfig().MODELS.LM_STUDIO.API_URL;
-
 const mergeConfigs = (current: any, update: any): any => {
   if (update === null || update === undefined) {
     return current;
@@ -129,13 +119,10 @@ const mergeConfigs = (current: any, update: any): any => {
 };
 
 export const updateConfig = (config: RecursivePartial<Config>) => {
-  // Server-side only
+  const currentConfig = loadConfig();
-  if (typeof window === 'undefined') {
+  const mergedConfig = mergeConfigs(currentConfig, config);
-    const currentConfig = loadConfig();
+  fs.writeFileSync(
-    const mergedConfig = mergeConfigs(currentConfig, config);
+    path.join(path.join(process.cwd(), `${configFileName}`)),
-    fs.writeFileSync(
+    toml.stringify(mergedConfig),
-      path.join(path.join(process.cwd(), `${configFileName}`)),
+  );
-      toml.stringify(mergedConfig),
-    );
-  }
 };

src/lib/providers/anthropic.ts

@@ -1,11 +1,6 @@
 import { ChatAnthropic } from '@langchain/anthropic';
 import { ChatModel } from '.';
 import { getAnthropicApiKey } from '../config';
-
-export const PROVIDER_INFO = {
-  key: 'anthropic',
-  displayName: 'Anthropic',
-};
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const anthropicChatModels: Record<string, string>[] = [

src/lib/providers/deepseek.ts

@@ -3,11 +3,6 @@ import { getDeepseekApiKey } from '../config';
 import { ChatModel } from '.';
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
-export const PROVIDER_INFO = {
-  key: 'deepseek',
-  displayName: 'Deepseek AI',
-};
-
 const deepseekChatModels: Record<string, string>[] = [
   {
     displayName: 'Deepseek Chat (Deepseek V3)',

src/lib/providers/gemini.ts

@@ -4,11 +4,6 @@ import {
 } from '@langchain/google-genai';
 import { getGeminiApiKey } from '../config';
 import { ChatModel, EmbeddingModel } from '.';
-
-export const PROVIDER_INFO = {
-  key: 'gemini',
-  displayName: 'Google Gemini',
-};
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { Embeddings } from '@langchain/core/embeddings';
 

src/lib/providers/groq.ts

@@ -1,11 +1,6 @@
 import { ChatOpenAI } from '@langchain/openai';
 import { getGroqApiKey } from '../config';
 import { ChatModel } from '.';
-
-export const PROVIDER_INFO = {
-  key: 'groq',
-  displayName: 'Groq',
-};
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const groqChatModels: Record<string, string>[] = [

src/lib/providers/index.ts

@@ -1,60 +1,18 @@
 import { Embeddings } from '@langchain/core/embeddings';
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
-import {
+import { loadOpenAIChatModels, loadOpenAIEmbeddingModels } from './openai';
-  loadOpenAIChatModels,
-  loadOpenAIEmbeddingModels,
-  PROVIDER_INFO as OpenAIInfo,
-  PROVIDER_INFO,
-} from './openai';
 import {
   getCustomOpenaiApiKey,
   getCustomOpenaiApiUrl,
   getCustomOpenaiModelName,
 } from '../config';
 import { ChatOpenAI } from '@langchain/openai';
-import {
+import { loadOllamaChatModels, loadOllamaEmbeddingModels } from './ollama';
-  loadOllamaChatModels,
+import { loadGroqChatModels } from './groq';
-  loadOllamaEmbeddingModels,
+import { loadAnthropicChatModels } from './anthropic';
-  PROVIDER_INFO as OllamaInfo,
+import { loadGeminiChatModels, loadGeminiEmbeddingModels } from './gemini';
-} from './ollama';
+import { loadTransformersEmbeddingsModels } from './transformers';
-import { loadGroqChatModels, PROVIDER_INFO as GroqInfo } from './groq';
+import { loadDeepseekChatModels } from './deepseek';
-import {
-  loadAnthropicChatModels,
-  PROVIDER_INFO as AnthropicInfo,
-} from './anthropic';
-import {
-  loadGeminiChatModels,
-  loadGeminiEmbeddingModels,
-  PROVIDER_INFO as GeminiInfo,
-} from './gemini';
-import {
-  loadTransformersEmbeddingsModels,
-  PROVIDER_INFO as TransformersInfo,
-} from './transformers';
-import {
-  loadDeepseekChatModels,
-  PROVIDER_INFO as DeepseekInfo,
-} from './deepseek';
-import {
-  loadLMStudioChatModels,
-  loadLMStudioEmbeddingsModels,
-  PROVIDER_INFO as LMStudioInfo,
-} from './lmstudio';
-
-export const PROVIDER_METADATA = {
-  openai: OpenAIInfo,
-  ollama: OllamaInfo,
-  groq: GroqInfo,
-  anthropic: AnthropicInfo,
-  gemini: GeminiInfo,
-  transformers: TransformersInfo,
-  deepseek: DeepseekInfo,
-  lmstudio: LMStudioInfo,
-  custom_openai: {
-    key: 'custom_openai',
-    displayName: 'Custom OpenAI',
-  },
-};
 
 export interface ChatModel {
   displayName: string;
@@ -76,7 +34,6 @@ export const chatModelProviders: Record<
   anthropic: loadAnthropicChatModels,
   gemini: loadGeminiChatModels,
   deepseek: loadDeepseekChatModels,
-  lmstudio: loadLMStudioChatModels,
 };
 
 export const embeddingModelProviders: Record<
@@ -87,7 +44,6 @@ export const embeddingModelProviders: Record<
   ollama: loadOllamaEmbeddingModels,
   gemini: loadGeminiEmbeddingModels,
   transformers: loadTransformersEmbeddingsModels,
-  lmstudio: loadLMStudioEmbeddingsModels,
 };
 
 export const getAvailableChatModelProviders = async () => {

src/lib/providers/lmstudio.ts

@@ -1,100 +0,0 @@
-import { getKeepAlive, getLMStudioApiEndpoint } from '../config';
-import axios from 'axios';
-import { ChatModel, EmbeddingModel } from '.';
-
-export const PROVIDER_INFO = {
-  key: 'lmstudio',
-  displayName: 'LM Studio',
-};
-import { ChatOpenAI } from '@langchain/openai';
-import { OpenAIEmbeddings } from '@langchain/openai';
-import { BaseChatModel } from '@langchain/core/language_models/chat_models';
-import { Embeddings } from '@langchain/core/embeddings';
-
-interface LMStudioModel {
-  id: string;
-  name?: string;
-}
-
-const ensureV1Endpoint = (endpoint: string): string =>
-  endpoint.endsWith('/v1') ? endpoint : `${endpoint}/v1`;
-
-const checkServerAvailability = async (endpoint: string): Promise<boolean> => {
-  try {
-    await axios.get(`${ensureV1Endpoint(endpoint)}/models`, {
-      headers: { 'Content-Type': 'application/json' },
-    });
-    return true;
-  } catch {
-    return false;
-  }
-};
-
-export const loadLMStudioChatModels = async () => {
-  const endpoint = getLMStudioApiEndpoint();
-
-  if (!endpoint) return {};
-  if (!(await checkServerAvailability(endpoint))) return {};
-
-  try {
-    const response = await axios.get(`${ensureV1Endpoint(endpoint)}/models`, {
-      headers: { 'Content-Type': 'application/json' },
-    });
-
-    const chatModels: Record<string, ChatModel> = {};
-
-    response.data.data.forEach((model: LMStudioModel) => {
-      chatModels[model.id] = {
-        displayName: model.name || model.id,
-        model: new ChatOpenAI({
-          openAIApiKey: 'lm-studio',
-          configuration: {
-            baseURL: ensureV1Endpoint(endpoint),
-          },
-          modelName: model.id,
-          temperature: 0.7,
-          streaming: true,
-          maxRetries: 3,
-        }) as unknown as BaseChatModel,
-      };
-    });
-
-    return chatModels;
-  } catch (err) {
-    console.error(`Error loading LM Studio models: ${err}`);
-    return {};
-  }
-};
-
-export const loadLMStudioEmbeddingsModels = async () => {
-  const endpoint = getLMStudioApiEndpoint();
-
-  if (!endpoint) return {};
-  if (!(await checkServerAvailability(endpoint))) return {};
-
-  try {
-    const response = await axios.get(`${ensureV1Endpoint(endpoint)}/models`, {
-      headers: { 'Content-Type': 'application/json' },
-    });
-
-    const embeddingsModels: Record<string, EmbeddingModel> = {};
-
-    response.data.data.forEach((model: LMStudioModel) => {
-      embeddingsModels[model.id] = {
-        displayName: model.name || model.id,
-        model: new OpenAIEmbeddings({
-          openAIApiKey: 'lm-studio',
-          configuration: {
-            baseURL: ensureV1Endpoint(endpoint),
-          },
-          modelName: model.id,
-        }) as unknown as Embeddings,
-      };
-    });
-
-    return embeddingsModels;
-  } catch (err) {
-    console.error(`Error loading LM Studio embeddings model: ${err}`);
-    return {};
-  }
-};

src/lib/providers/ollama.ts

@@ -1,11 +1,6 @@
 import axios from 'axios';
 import { getKeepAlive, getOllamaApiEndpoint } from '../config';
 import { ChatModel, EmbeddingModel } from '.';
-
-export const PROVIDER_INFO = {
-  key: 'ollama',
-  displayName: 'Ollama',
-};
 import { ChatOllama } from '@langchain/community/chat_models/ollama';
 import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama';
 

src/lib/providers/openai.ts

@@ -1,11 +1,6 @@
 import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai';
 import { getOpenaiApiKey } from '../config';
 import { ChatModel, EmbeddingModel } from '.';
-
-export const PROVIDER_INFO = {
-  key: 'openai',
-  displayName: 'OpenAI',
-};
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { Embeddings } from '@langchain/core/embeddings';
 
@@ -30,18 +25,6 @@ const openaiChatModels: Record<string, string>[] = [
     displayName: 'GPT-4 omni mini',
     key: 'gpt-4o-mini',
   },
-  {
-    displayName: 'GPT 4.1 nano',
-    key: 'gpt-4.1-nano',
-  },
-  {
-    displayName: 'GPT 4.1 mini',
-    key: 'gpt-4.1-mini',
-  },
-  {
-    displayName: 'GPT 4.1',
-    key: 'gpt-4.1',
-  },
 ];
 
 const openaiEmbeddingModels: Record<string, string>[] = [

src/lib/providers/transformers.ts

@@ -1,10 +1,5 @@
 import { HuggingFaceTransformersEmbeddings } from '../huggingfaceTransformer';
 
-export const PROVIDER_INFO = {
-  key: 'transformers',
-  displayName: 'Hugging Face',
-};
-
 export const loadTransformersEmbeddingsModels = async () => {
   try {
     const embeddingModels = {

src/lib/search/metaSearchAgent.ts

@@ -17,7 +17,9 @@ import LineListOutputParser from '../outputParsers/listLineOutputParser';
 import LineOutputParser from '../outputParsers/lineOutputParser';
 import { getDocumentsFromLinks } from '../utils/documents';
 import { Document } from 'langchain/document';
-import { searchSearxng } from '../searxng';
+import { searchTavily } from '../searchEngines/tavily';
+import { searchSearxng } from '../searchEngines/searxng';
+import { getSearchEngine } from '../config';
 import path from 'node:path';
 import fs from 'node:fs';
 import computeSimilarity from '../utils/computeSimilarity';
@@ -205,25 +207,42 @@ class MetaSearchAgent implements MetaSearchAgentType {
         } else {
           question = question.replace(/<think>.*?<\/think>/g, '');
 
-          const res = await searchSearxng(question, {
+          const searchEngine = getSearchEngine();
-            language: 'en',
-            engines: this.config.activeEngines,
-          });
 
-          const documents = res.results.map(
+          let res;
-            (result) =>
+          
-              new Document({
+          if (searchEngine === 'tavily') {
-                pageContent:
+            res = await searchTavily(question, {
-                  result.content ||
+              search_depth: 'basic',
-                  (this.config.activeEngines.includes('youtube')
+              max_results: 15,
-                    ? result.title
+              include_images: true,
-                    : '') /* Todo: Implement transcript grabbing using Youtubei (source: https://www.npmjs.com/package/youtubei) */,
+            });
-                metadata: {
+          } else {
-                  title: result.title,
+            // Default to SearxNG
-                  url: result.url,
+            res = await searchSearxng(question, {
-                  ...(result.img_src && { img_src: result.img_src }),
+              language: 'en',
-                },
+              engines: this.config.activeEngines,
-              }),
+            });
+          }
+
+          let documents: Document[] = [];
+          
+          documents = documents.concat(
+            res.results.map(
+              (result) =>
+                new Document({
+                  pageContent:
+                    result.content ||
+                    (this.config.activeEngines.includes('youtube')
+                      ? result.title
+                      : ''),
+                  metadata: {
+                    title: result.title,
+                    url: result.url,
+                    ...(result.img_src ? { img_src: result.img_src } : {}),
+                  },
+                }),
+            )
           );
 
           return { query: question, docs: documents };

src/lib/searchEngines/searxng.ts

@@ -1,5 +1,5 @@
 import axios from 'axios';
-import { getSearxngApiEndpoint } from './config';
+import { getSearxngApiEndpoint } from '../config';
 
 interface SearxngSearchOptions {
   categories?: string[];

src/lib/searchEngines/tavily.ts

@@ -0,0 +1,79 @@
+import axios from 'axios';
+import { getTavilyApiKey } from '../config';
+
+interface TavilySearchOptions {
+  topic?: 'general' | 'news';
+  search_depth?: 'basic' | 'advanced';
+  chunks_per_source?: number;
+  max_results?: number;
+  time_range?: 'day' | 'week' | 'month' | 'year' | 'd' | 'w' | 'm' | 'y';
+  days?: number;
+  include_answer?: boolean | 'basic' | 'advanced';
+  include_raw_content?: boolean;
+  include_images?: boolean;
+  include_image_descriptions?: boolean;
+  include_domains?: string[];
+  exclude_domains?: string[];
+}
+
+interface TavilySearchResult {
+  title: string;
+  url: string;
+  content: string;
+  score: number;
+  raw_content?: string;
+}
+
+interface TavilySearchResponse {
+  query: string;
+  answer?: string;
+  images?: Array<{
+    url: string;
+    description?: string;
+  }>;
+  results: TavilySearchResult[];
+  response_time: string;
+}
+
+export const searchTavily = async (
+  query: string,
+  opts?: TavilySearchOptions,
+) => {
+  const tavilyApiKey = getTavilyApiKey();
+  
+  if (!tavilyApiKey) {
+    throw new Error('Tavily API key is not configured');
+  }
+
+  const url = 'https://api.tavily.com/search';
+  
+  const response = await axios.post<TavilySearchResponse>(
+    url,
+    {
+      query,
+      ...opts,
+    },
+    {
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${tavilyApiKey}`,
+      },
+    }
+  );
+
+  const results = response.data.results;
+  
+  // Convert Tavily results to match the format expected by the rest of the application
+  const formattedResults = results.map(result => ({
+    title: result.title,
+    url: result.url,
+    content: result.content,
+    img_src: undefined, // Tavily doesn't provide image URLs in the standard response
+  }));
+
+  return { 
+    results: formattedResults, 
+    suggestions: [], // Tavily doesn't provide suggestions, so return empty array
+    answer: response.data.answer, // Include the AI-generated answer if available
+  };
+}; 

src/lib/utils/documents.ts

@@ -64,7 +64,7 @@ export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
         const splittedText = await splitter.splitText(parsedText);
         const title = res.data
           .toString('utf8')
-          .match(/<title.*>(.*?)<\/title>/)?.[1];
+          .match(/<title>(.*?)<\/title>/)?.[1];
 
         const linkDocs = splittedText.map((text) => {
           return new Document({