add brave as a search engine

sample.config.toml

@@ -2,7 +2,7 @@
 PORT = 3001 # Port to run the server on
 SIMILARITY_MEASURE = "cosine" # "cosine" or "dot"
 KEEP_ALIVE = "5m" # How long to keep Ollama models loaded into memory. (Instead of using -1 use "-1m")
-SEARCH_ENGINE_BACKEND = "google" # "google" | "searxng" | "ddg" | "bing" | "brave"
+SEARCH_ENGINE_BACKEND = "searxng" # "google" | "searxng" | "bing" | "brave"
 
 [MODELS.OPENAI]
 API_KEY = ""

src/chains/imageSearchAgent.ts

@@ -9,6 +9,7 @@ import { BaseMessage } from '@langchain/core/messages';
 import { StringOutputParser } from '@langchain/core/output_parsers';
 import { searchSearxng } from '../lib/searchEngines/searxng';
 import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
 import { getSearchEngineBackend } from '../config';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
@@ -54,8 +55,7 @@ async function performImageSearch(query: string) {
             source: result.displayLink
           };
         }
-        })
-        .filter(Boolean);
+      }).filter(Boolean);
       break;
     }
 
@@ -76,6 +76,21 @@ async function performImageSearch(query: string) {
       break;
     }
 
+    case 'brave': {
+      const braveResult = await searchBraveAPI(query);
+      images = braveResult.results.map((result) => {
+        if (result.img_src && result.url && result.title) {
+          return {
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            source: result.url
+          };
+        }
+      }).filter(Boolean);
+      break;
+    }
+
     default:
       throw new Error(`Unknown search engine ${searchEngine}`);
   }

src/chains/videoSearchAgent.ts

@@ -9,6 +9,7 @@ import { BaseMessage } from '@langchain/core/messages';
 import { StringOutputParser } from '@langchain/core/output_parsers';
 import { searchSearxng } from '../lib/searchEngines/searxng';
 import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
 import { getSearchEngineBackend } from '../config';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
@@ -84,6 +85,22 @@ async function performVideoSearch(query: string) {
       break;
     }
 
+    case 'brave': {
+      const braveResult = await searchBraveAPI(youtubeQuery);
+      braveResult.results.forEach((result) => {
+        if (result.img_src && result.url && result.title) {
+          const videoId = new URL(result.url).searchParams.get('v');
+          videos.push({
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
+          });
+        }
+      });
+      break;
+    }
+
     default:
       throw new Error(`Unknown search engine ${searchEngine}`);
   }

src/lib/searchEngines/bing.ts

@@ -0,0 +1,116 @@
+import axios from 'axios';
+import { getBingSubscriptionKey } from '../../config';
+
+interface BingAPISearchResult {
+  _type: string;
+  name: string;
+  url: string;
+  displayUrl: string;
+  snippet?: string;
+  dateLastCrawled?: string;
+  thumbnailUrl?: string;
+  contentUrl?: string;
+  hostPageUrl?: string;
+  width?: number;
+  height?: number;
+  accentColor?: string;
+  contentSize?: string;
+  datePublished?: string;
+  encodingFormat?: string;
+  hostPageDisplayUrl?: string;
+  id?: string;
+  isLicensed?: boolean;
+  isFamilyFriendly?: boolean;
+  language?: string;
+  mediaUrl?: string;
+  motionThumbnailUrl?: string;
+  publisher?: string;
+  viewCount?: number;
+  webSearchUrl?: string;
+  primaryImageOfPage?: {
+    thumbnailUrl?: string;
+    width?: number;
+    height?: number;
+  };
+  provider?: Array<{
+    name: string;
+    _type: string;
+  }>;
+  video?: {
+    allowHttpsEmbed?: boolean;
+    embedHtml?: string;
+    allowMobileEmbed?: boolean;
+    viewCount?: number;
+  };
+  image?: {
+    thumbnail?: {
+      contentUrl?: string;
+      width?: number;
+      height?: number;
+    };
+    imageInsightsToken?: string;
+    imageId?: string;
+  };
+  metatags?: Array<{
+    [key: string]: string;
+    'og:type'?: string;
+    'og:image'?: string;
+    'og:video'?: string;
+  }>;
+  mentions?: Array<{
+    name: string;
+  }>;
+  entity?: {
+    entityPresentationInfo?: {
+      entityTypeHints?: string[];
+    };
+  };
+}
+
+
+export const searchBingAPI = async (query: string) => {
+  try {
+    const bingApiKey = await getBingSubscriptionKey();
+    const url = new URL(`https://api.cognitive.microsoft.com/bing/v7.0/search`);
+    url.searchParams.append('q', query);
+    url.searchParams.append('responseFilter', 'Webpages,Images,Videos,News');
+
+    const res = await axios.get(url.toString(), {
+      headers: {
+        'Ocp-Apim-Subscription-Key': bingApiKey,
+        'Accept': 'application/json'
+      }
+    });
+
+    if (res.data.error) {
+      throw new Error(`Bing API Error: ${res.data.error.message}`);
+    }
+
+    const originalres = res.data;
+    const webResults = originalres.webPages?.value || [];
+    const imageResults = originalres.images?.value || [];
+    const videoResults = originalres.videos?.value || [];
+
+    const results = webResults.map((item: any) => ({
+      title: item.name,
+      url: item.url,
+      content: item.snippet,
+      img_src: item.primaryImageOfPage?.thumbnailUrl
+             || imageResults.find((img: any) => img.hostPageUrl === item.url)?.thumbnailUrl
+             || videoResults.find((vid: any) => vid.hostPageUrl === item.url)?.thumbnailUrl,
+      ...(item.video && {
+        videoData: {
+          duration: item.video.duration,
+          embedUrl: item.video.embedHtml?.match(/src="(.*?)"/)?.[1]
+        }
+      })
+    }));
+
+    return { results, originalres };
+  } catch (error) {
+    const errorMessage = error.response?.data
+      ? JSON.stringify(error.response.data, null, 2)
+      : error.message || 'Unknown error';
+    throw new Error(`Bing API Error: ${errorMessage}`);
+  }
+};

src/lib/searchEngines/brave.ts

@@ -0,0 +1,96 @@
+import axios from 'axios';
+import { getBraveApiKey } from '../../config';
+
+interface BraveSearchResult {
+  title: string;
+  url: string;
+  content?: string;
+  img_src?: string;
+  age?: string;
+  family_friendly?: boolean;
+  language?: string;
+  video?: {
+    embedUrl?: string;
+    duration?: string;
+  };
+  rating?: {
+    value: number;
+    scale: number;
+  };
+  products?: Array<{
+    name: string;
+    price?: string;
+  }>;
+  recipe?: {
+    ingredients?: string[];
+    cookTime?: string;
+  };
+  meta?: {
+    fetched?: string;
+    lastCrawled?: string;
+  };
+}
+
+export const searchBraveAPI = async (
+  query: string,
+  numResults: number = 20
+): Promise<{ results: BraveSearchResult[]; originalres: any }> => {
+  try {
+    const braveApiKey = await getBraveApiKey();
+    const url = new URL(`https://api.search.brave.com/res/v1/web/search`);
+
+    url.searchParams.append('q', query);
+    url.searchParams.append('count', numResults.toString());
+
+    const res = await axios.get(url.toString(), {
+      headers: {
+        'X-Subscription-Token': braveApiKey,
+        'Accept': 'application/json'
+      }
+    });
+
+    if (res.data.error) {
+      throw new Error(`Brave API Error: ${res.data.error.message}`);
+    }
+
+    const originalres = res.data;
+    const webResults = originalres.web?.results || [];
+
+    const results: BraveSearchResult[] = webResults.map((item: any) => ({
+      title: item.title,
+      url: item.url,
+      content: item.description,
+      img_src: item.thumbnail?.src || item.deep_results?.images?.[0]?.src,
+      age: item.age,
+      family_friendly: item.family_friendly,
+      language: item.language,
+      video: item.video ? {
+        embedUrl: item.video.embed_url,
+        duration: item.video.duration
+      } : undefined,
+      rating: item.rating ? {
+        value: item.rating.value,
+        scale: item.rating.scale_max
+      } : undefined,
+      products: item.deep_results?.product_cluster?.map((p: any) => ({
+        name: p.name,
+        price: p.price
+      })),
+      recipe: item.recipe ? {
+        ingredients: item.recipe.ingredients,
+        cookTime: item.recipe.cook_time
+      } : undefined,
+      meta: {
+        fetched: item.meta?.fetched,
+        lastCrawled: item.meta?.last_crawled
+      }
+    }));
+
+    return { results, originalres };
+  } catch (error) {
+    const errorMessage = error.response?.data
+      ? JSON.stringify(error.response.data, null, 2)
+      : error.message || 'Unknown error';
+    throw new Error(`Brave API Error: ${errorMessage}`);
+  }
+};

src/routes/discover.ts

@@ -1,6 +1,7 @@
 import express from 'express';
 import { searchSearxng } from '../lib/searchEngines/searxng';
 import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
 import { getSearchEngineBackend } from '../config';
 import logger from '../utils/logger';
 
@@ -42,6 +43,20 @@ async function performSearch(query: string, site: string) {
       return searxResult.results;
     }
 
+    case 'brave': {
+      const braveResult = await searchBraveAPI(query);
+      return braveResult.results.map(item => ({
+        title: item.title,
+        url: item.url,
+        content: item.content,
+        thumbnail: item.img_src,
+        img_src: item.img_src,
+        iframe_src: null,
+        author: item.meta?.fetched || site,
+        publishedDate: item.meta?.lastCrawled
+      }));
+    }
+
     default:
       throw new Error(`Unknown search engine ${searchEngine}`);
   }

src/search/metaSearchAgent.ts

@@ -19,6 +19,8 @@ import { getDocumentsFromLinks } from '../utils/documents';
 import { Document } from 'langchain/document';
 import { searchSearxng } from '../lib/searchEngines/searxng';
 import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBingAPI } from '../lib/searchEngines/bing';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
 import { getSearchEngineBackend } from '../config';
 import path from 'path';
 import fs from 'fs';
@@ -219,6 +221,12 @@ class MetaSearchAgent implements MetaSearchAgentType {
             case 'google':
               res = await searchGooglePSE(question);
               break;
+            case 'bing':
+              res = await searchBingAPI(question);
+              break;
+            case 'brave':
+              res = await searchBraveAPI(question);
+              break;
             default:
               throw new Error(`Unknown search engine ${searchEngine}`);
           }