feat(tavily): integrate Tavily search engine with configuration and UI support

src/lib/config.ts

@@ -36,6 +36,10 @@ interface Config {
   };
   API_ENDPOINTS: {
     SEARXNG: string;
+    TAVILY: string;
+  };
+  SEARCH: {
+    ENGINE: string;
   };
 }
 
@@ -64,6 +68,12 @@ export const getGeminiApiKey = () => loadConfig().MODELS.GEMINI.API_KEY;
 export const getSearxngApiEndpoint = () =>
   process.env.SEARXNG_API_URL || loadConfig().API_ENDPOINTS.SEARXNG;
 
+export const getTavilyApiKey = () =>
+  process.env.TAVILY_API_KEY || loadConfig().API_ENDPOINTS.TAVILY;
+
+export const getSearchEngine = () =>
+  process.env.SEARCH_ENGINE || loadConfig().SEARCH?.ENGINE || 'searxng';
+
 export const getOllamaApiEndpoint = () => loadConfig().MODELS.OLLAMA.API_URL;
 
 export const getDeepseekApiKey = () => loadConfig().MODELS.DEEPSEEK.API_KEY;

src/lib/search/metaSearchAgent.ts

@@ -17,7 +17,9 @@ import LineListOutputParser from '../outputParsers/listLineOutputParser';
 import LineOutputParser from '../outputParsers/lineOutputParser';
 import { getDocumentsFromLinks } from '../utils/documents';
 import { Document } from 'langchain/document';
+import { searchTavily } from '../tavily';
 import { searchSearxng } from '../searxng';
+import { getSearchEngine } from '../config';
 import path from 'node:path';
 import fs from 'node:fs';
 import computeSimilarity from '../utils/computeSimilarity';
@@ -205,25 +207,47 @@ class MetaSearchAgent implements MetaSearchAgentType {
         } else {
           question = question.replace(/<think>.*?<\/think>/g, '');
 
-          const res = await searchSearxng(question, {
-            language: 'en',
-            engines: this.config.activeEngines,
-          });
+          const searchEngine = getSearchEngine();
+          console.log(`Using search engine: ${searchEngine}`);
 
-          const documents = res.results.map(
-            (result) =>
-              new Document({
-                pageContent:
-                  result.content ||
-                  (this.config.activeEngines.includes('youtube')
-                    ? result.title
-                    : '') /* Todo: Implement transcript grabbing using Youtubei (source: https://www.npmjs.com/package/youtubei) */,
-                metadata: {
-                  title: result.title,
-                  url: result.url,
-                  ...(result.img_src && { img_src: result.img_src }),
-                },
-              }),
+          let res;
+          
+          if (searchEngine === 'tavily') {
+            res = await searchTavily(question, {
+              search_depth: 'basic',
+              max_results: 15,
+              include_images: true,
+            });
+          } else {
+            // Default to SearxNG
+            res = await searchSearxng(question, {
+              language: 'en',
+              engines: this.config.activeEngines,
+            });
+          }
+
+          // If we have an AI-generated answer from Tavily, create a document for it
+          let documents: Document[] = [];
+          
+          
+
+          // Add the regular search results
+          documents = documents.concat(
+            res.results.map(
+              (result) =>
+                new Document({
+                  pageContent:
+                    result.content ||
+                    (this.config.activeEngines.includes('youtube')
+                      ? result.title
+                      : ''),
+                  metadata: {
+                    title: result.title,
+                    url: result.url,
+                    ...(result.img_src ? { img_src: result.img_src } : {}),
+                  },
+                }),
+            )
           );
 
           return { query: question, docs: documents };

src/lib/tavily.ts

@@ -0,0 +1,79 @@
+import axios from 'axios';
+import { getTavilyApiKey } from './config';
+
+interface TavilySearchOptions {
+  topic?: 'general' | 'news';
+  search_depth?: 'basic' | 'advanced';
+  chunks_per_source?: number;
+  max_results?: number;
+  time_range?: 'day' | 'week' | 'month' | 'year' | 'd' | 'w' | 'm' | 'y';
+  days?: number;
+  include_answer?: boolean | 'basic' | 'advanced';
+  include_raw_content?: boolean;
+  include_images?: boolean;
+  include_image_descriptions?: boolean;
+  include_domains?: string[];
+  exclude_domains?: string[];
+}
+
+interface TavilySearchResult {
+  title: string;
+  url: string;
+  content: string;
+  score: number;
+  raw_content?: string;
+}
+
+interface TavilySearchResponse {
+  query: string;
+  answer?: string;
+  images?: Array<{
+    url: string;
+    description?: string;
+  }>;
+  results: TavilySearchResult[];
+  response_time: string;
+}
+
+export const searchTavily = async (
+  query: string,
+  opts?: TavilySearchOptions,
+) => {
+  const tavilyApiKey = getTavilyApiKey();
+  
+  if (!tavilyApiKey) {
+    throw new Error('Tavily API key is not configured');
+  }
+
+  const url = 'https://api.tavily.com/search';
+  
+  const response = await axios.post<TavilySearchResponse>(
+    url,
+    {
+      query,
+      ...opts,
+    },
+    {
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${tavilyApiKey}`,
+      },
+    }
+  );
+
+  const results = response.data.results;
+  
+  // Convert Tavily results to match the format expected by the rest of the application
+  const formattedResults = results.map(result => ({
+    title: result.title,
+    url: result.url,
+    content: result.content,
+    img_src: undefined, // Tavily doesn't provide image URLs in the standard response
+  }));
+
+  return { 
+    results: formattedResults, 
+    suggestions: [], // Tavily doesn't provide suggestions, so return empty array
+    answer: response.data.answer, // Include the AI-generated answer if available
+  };
+};