Merge branch 'ItzCrazyKns:master' into master

This reverts commit 31e4abf068.

.gitignore

@@ -2,6 +2,7 @@
 node_modules/
 npm-debug.log
 yarn-error.log
+package-lock.json
 
 # Build output
 /.next/
@@ -37,3 +38,6 @@ Thumbs.db
 # Db
 db.sqlite
 /searxng
+
+# Dev
+docker-compose-dev.yaml

README.md

@@ -44,7 +44,7 @@ Want to know more about its architecture and how it works? You can read it [here
   - **Normal Mode:** Processes your query and performs a web search.
 - **Focus Modes:** Special modes to better answer specific types of questions. Perplexica currently has 6 focus modes:
   - **All Mode:** Searches the entire web to find the best results.
-  - **Writing Assistant Mode:** Helpful for writing tasks that does not require searching the web.
+  - **Writing Assistant Mode:** Helpful for writing tasks that do not require searching the web.
   - **Academic Search Mode:** Finds articles and papers, ideal for academic research.
   - **YouTube Search Mode:** Finds YouTube videos based on the search query.
   - **Wolfram Alpha Search Mode:** Answers queries that need calculations or data analysis using Wolfram Alpha.
@@ -143,6 +143,7 @@ You can access Perplexica over your home network by following our networking gui
 
 ## One-Click Deployment
 
+[![Deploy to Sealos](https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg)](https://usw.sealos.io/?openapp=system-template%3FtemplateName%3Dperplexica)
 [![Deploy to RepoCloud](https://d16t0pc4846x52.cloudfront.net/deploylobe.svg)](https://repocloud.io/details/?app_id=267)
 
 ## Upcoming Features

docker-compose.yaml

@@ -4,7 +4,7 @@ services:
     volumes:
       - ./searxng:/etc/searxng:rw
     ports:
-      - 4000:8080
+      - "4000:8080"
     networks:
       - perplexica-network
     restart: unless-stopped
@@ -19,7 +19,7 @@ services:
     depends_on:
       - searxng
     ports:
-      - 3001:3001
+      - "3001:3001"
     volumes:
       - backend-dbstore:/home/perplexica/data
       - uploads:/home/perplexica/uploads
@@ -41,7 +41,7 @@ services:
     depends_on:
       - perplexica-backend
     ports:
-      - 3000:3000
+      - "3000:3000"
     networks:
       - perplexica-network
     restart: unless-stopped

docs/installation/UPDATING.md

@@ -10,23 +10,25 @@ To update Perplexica to the latest version, follow these steps:
    git clone https://github.com/ItzCrazyKns/Perplexica.git
    ```
 
-2. Navigate to the Project Directory.
+2. Navigate to the project directory.
 
-3. Pull latest images from registry.
+3. Check for changes in the configuration files. If the `sample.config.toml` file contains new fields, delete your existing `config.toml` file, rename `sample.config.toml` to `config.toml`, and update the configuration accordingly.
+
+4. Pull the latest images from the registry.
 
    ```bash
    docker compose pull
    ```
 
-4. Update and Recreate containers.
+5. Update and recreate the containers.
 
    ```bash
    docker compose up -d
    ```
 
-5. Once the command completes running go to http://localhost:3000 and verify the latest changes.
+6. Once the command completes, go to http://localhost:3000 and verify the latest changes.
 
-## For non Docker users
+## For non-Docker users
 
 1. Clone the latest version of Perplexica from GitHub:
 
@@ -34,7 +36,14 @@ To update Perplexica to the latest version, follow these steps:
    git clone https://github.com/ItzCrazyKns/Perplexica.git
    ```
 
-2. Navigate to the Project Directory
-3. Execute `npm i` in both the `ui` folder and the root directory.
-4. Once packages are updated, execute `npm run build` in both the `ui` folder and the root directory.
-5. Finally, start both the frontend and the backend by running `npm run start` in both the `ui` folder and the root directory.
+2. Navigate to the project directory.
+
+3. Check for changes in the configuration files. If the `sample.config.toml` file contains new fields, delete your existing `config.toml` file, rename `sample.config.toml` to `config.toml`, and update the configuration accordingly.
+
+4. Execute `npm i` in both the `ui` folder and the root directory.
+
+5. Once the packages are updated, execute `npm run build` in both the `ui` folder and the root directory.
+
+6. Finally, start both the frontend and the backend by running `npm run start` in both the `ui` folder and the root directory.
+
+---

package.json

@@ -30,8 +30,8 @@
     "@iarna/toml": "^2.2.5",
     "@langchain/anthropic": "^0.2.3",
     "@langchain/community": "^0.2.16",
-    "@langchain/openai": "^0.0.25",
     "@langchain/google-genai": "^0.0.23",
+    "@langchain/openai": "^0.0.25",
     "@xenova/transformers": "^2.17.1",
     "axios": "^1.6.8",
     "better-sqlite3": "^11.0.0",

sample.config.toml

@@ -3,6 +3,12 @@ PORT = 3001 # Port to run the server on
 SIMILARITY_MEASURE = "cosine" # "cosine" or "dot"
 KEEP_ALIVE = "5m" # How long to keep Ollama models loaded into memory. (Instead of using -1 use "-1m")
 
+[SEARCH_ENGINE_BACKENDS] # "google" | "searxng" | "bing" | "brave" | "yacy"
+SEARCH = "searxng"
+IMAGE = "searxng"
+VIDEO = "searxng"
+NEWS = "searxng"
+
 [MODELS.OPENAI]
 API_KEY = ""
 
@@ -22,5 +28,18 @@ API_URL = ""
 [MODELS.OLLAMA]
 API_URL = "" # Ollama API URL - http://host.docker.internal:11434
 
-[API_ENDPOINTS]
-SEARXNG = "http://localhost:32768" # SearxNG API URL
+[SEARCH_ENGINES.GOOGLE]
+API_KEY = ""
+CSE_ID = ""
+
+[SEARCH_ENGINES.SEARXNG]
+ENDPOINT = ""
+
+[SEARCH_ENGINES.BING]
+SUBSCRIPTION_KEY = ""
+
+[SEARCH_ENGINES.BRAVE]
+API_KEY = ""
+
+[SEARCH_ENGINES.YACY]
+ENDPOINT = ""

searxng/settings.yml

@@ -15,3 +15,5 @@ server:
 engines:
   - name: wolframalpha
     disabled: false
+  - name: qwant
+    disabled: true

src/chains/imageSearchAgent.ts

@@ -7,7 +7,12 @@ import { PromptTemplate } from '@langchain/core/prompts';
 import formatChatHistoryAsString from '../utils/formatHistory';
 import { BaseMessage } from '@langchain/core/messages';
 import { StringOutputParser } from '@langchain/core/output_parsers';
-import { searchSearxng } from '../lib/searxng';
+import { searchSearxng } from '../lib/searchEngines/searxng';
+import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
+import { searchYaCy } from '../lib/searchEngines/yacy';
+import { searchBingAPI } from '../lib/searchEngines/bing';
+import { getImageSearchEngineBackend } from '../config';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const imageSearchChainPrompt = `
@@ -36,6 +41,95 @@ type ImageSearchChainInput = {
   query: string;
 };
 
+async function performImageSearch(query: string) {
+  const searchEngine = getImageSearchEngineBackend();
+  let images = [];
+
+  switch (searchEngine) {
+    case 'google': {
+      const googleResult = await searchGooglePSE(query);
+      images = googleResult.results.map((result) => {
+        if (result.img_src && result.url && result.title) {
+          return {
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            source: result.displayLink
+          };
+        }
+      }).filter(Boolean);
+      break;
+    }
+
+    case 'searxng': {
+      const searxResult = await searchSearxng(query, {
+        engines: ['google images', 'bing images'],
+        pageno: 1,
+      });
+      searxResult.results.forEach((result) => {
+        if (result.img_src && result.url && result.title) {
+          images.push({
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+          });
+        }
+      });
+      break;
+    }
+
+    case 'brave': {
+      const braveResult = await searchBraveAPI(query);
+      images = braveResult.results.map((result) => {
+        if (result.img_src && result.url && result.title) {
+          return {
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            source: result.url
+          };
+        }
+      }).filter(Boolean);
+      break;
+    }
+
+    case 'yacy': {
+      const yacyResult = await searchYaCy(query);
+      images = yacyResult.results.map((result) => {
+        if (result.img_src && result.url && result.title) {
+          return {
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            source: result.url
+          }
+        }
+      }).filter(Boolean);
+      break;
+    }
+
+    case 'bing': {
+      const bingResult = await searchBingAPI(query);
+      images = bingResult.results.map((result) => {
+        if (result.img_src && result.url && result.title) {
+          return {
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            source: result.url
+          }
+        }
+      }).filter(Boolean);
+      break;
+    }
+
+    default:
+      throw new Error(`Unknown search engine ${searchEngine}`);
+  }
+
+  return images;
+}
+
 const strParser = new StringOutputParser();
 
 const createImageSearchChain = (llm: BaseChatModel) => {
@@ -52,22 +146,7 @@ const createImageSearchChain = (llm: BaseChatModel) => {
     llm,
     strParser,
     RunnableLambda.from(async (input: string) => {
-      const res = await searchSearxng(input, {
-        engines: ['bing images', 'google images'],
-      });
-
-      const images = [];
-
-      res.results.forEach((result) => {
-        if (result.img_src && result.url && result.title) {
-          images.push({
-            img_src: result.img_src,
-            url: result.url,
-            title: result.title,
-          });
-        }
-      });
-
+      const images = await performImageSearch(input);
       return images.slice(0, 10);
     }),
   ]);

src/chains/videoSearchAgent.ts

@@ -7,7 +7,11 @@ import { PromptTemplate } from '@langchain/core/prompts';
 import formatChatHistoryAsString from '../utils/formatHistory';
 import { BaseMessage } from '@langchain/core/messages';
 import { StringOutputParser } from '@langchain/core/output_parsers';
-import { searchSearxng } from '../lib/searxng';
+import { searchSearxng } from '../lib/searchEngines/searxng';
+import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
+import { searchBingAPI } from '../lib/searchEngines/bing';
+import { getVideoSearchEngineBackend } from '../config';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const VideoSearchChainPrompt = `
@@ -38,27 +42,33 @@ type VideoSearchChainInput = {
 
 const strParser = new StringOutputParser();
 
-const createVideoSearchChain = (llm: BaseChatModel) => {
-  return RunnableSequence.from([
-    RunnableMap.from({
-      chat_history: (input: VideoSearchChainInput) => {
-        return formatChatHistoryAsString(input.chat_history);
-      },
-      query: (input: VideoSearchChainInput) => {
-        return input.query;
-      },
-    }),
-    PromptTemplate.fromTemplate(VideoSearchChainPrompt),
-    llm,
-    strParser,
-    RunnableLambda.from(async (input: string) => {
-      const res = await searchSearxng(input, {
+async function performVideoSearch(query: string) {
+  const searchEngine = getVideoSearchEngineBackend();
+  const youtubeQuery = `${query} site:youtube.com`;
+  let videos = [];
+
+  switch (searchEngine) {
+    case 'google': {
+      const googleResult = await searchGooglePSE(youtubeQuery);
+      googleResult.results.forEach((result) => { // Use .results instead of .originalres
+        if (result.img_src && result.url && result.title) {
+          const videoId = new URL(result.url).searchParams.get('v');
+          videos.push({
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
+          });
+        }
+      });
+      break;
+    }
+
+    case 'searxng': {
+      const searxResult = await searchSearxng(query, {
         engines: ['youtube'],
       });
-
-      const videos = [];
-
-      res.results.forEach((result) => {
+      searxResult.results.forEach((result) => {
         if (
           result.thumbnail &&
           result.url &&
@@ -73,7 +83,69 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
           });
         }
       });
+      break;
+    }
 
+    case 'brave': {
+      const braveResult = await searchBraveAPI(youtubeQuery);
+      braveResult.results.forEach((result) => {
+        if (result.img_src && result.url && result.title) {
+          const videoId = new URL(result.url).searchParams.get('v');
+          videos.push({
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
+          });
+        }
+      });
+      break;
+    }
+
+    case 'yacy': {
+      console.log('Not available for yacy');
+      videos = [];
+      break;
+    }
+
+    case 'bing': {
+      const bingResult = await searchBingAPI(youtubeQuery);
+      bingResult.results.forEach((result) => {
+        if (result.img_src && result.url && result.title) {
+          const videoId = new URL(result.url).searchParams.get('v');
+          videos.push({
+            img_src: result.img_src,
+            url: result.url,
+            title: result.title,
+            iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
+          });
+        }
+      });
+      break;
+    }
+
+    default:
+      throw new Error(`Unknown search engine ${searchEngine}`);
+  }
+
+  return videos;
+}
+
+const createVideoSearchChain = (llm: BaseChatModel) => {
+  return RunnableSequence.from([
+    RunnableMap.from({
+      chat_history: (input: VideoSearchChainInput) => {
+        return formatChatHistoryAsString(input.chat_history);
+      },
+      query: (input: VideoSearchChainInput) => {
+        return input.query;
+      },
+    }),
+    PromptTemplate.fromTemplate(VideoSearchChainPrompt),
+    llm,
+    strParser,
+    RunnableLambda.from(async (input: string) => {
+      const videos = await performVideoSearch(input);
       return videos.slice(0, 10);
     }),
   ]);

src/config.ts

@@ -10,6 +10,12 @@ interface Config {
     SIMILARITY_MEASURE: string;
     KEEP_ALIVE: string;
   };
+  SEARCH_ENGINE_BACKENDS: {
+    SEARCH: string;
+    IMAGE: string;
+    VIDEO: string;
+    NEWS: string;
+  }
   MODELS: {
     OPENAI: {
       API_KEY: string;
@@ -32,8 +38,23 @@ interface Config {
       MODEL_NAME: string;
     };
   };
-  API_ENDPOINTS: {
-    SEARXNG: string;
+  SEARCH_ENGINES: {
+    GOOGLE: {
+      API_KEY: string;
+      CSE_ID: string;
+    };
+    SEARXNG: {
+      ENDPOINT: string;
+    };
+    BING: {
+      SUBSCRIPTION_KEY: string;
+    };
+    BRAVE: {
+      API_KEY: string;
+    };
+    YACY: {
+      ENDPOINT: string;
+    };
   };
 }
 
@@ -61,8 +82,31 @@ export const getAnthropicApiKey = () => loadConfig().MODELS.ANTHROPIC.API_KEY;
 
 export const getGeminiApiKey = () => loadConfig().MODELS.GEMINI.API_KEY;
 
+export const getSearchEngineBackend = () =>
+  loadConfig().SEARCH_ENGINE_BACKENDS.SEARCH;
+
+export const getImageSearchEngineBackend = () =>
+  loadConfig().SEARCH_ENGINE_BACKENDS.IMAGE || getSearchEngineBackend();
+
+export const getVideoSearchEngineBackend = () =>
+  loadConfig().SEARCH_ENGINE_BACKENDS.VIDEO || getSearchEngineBackend();
+
+export const getNewsSearchEngineBackend = () =>
+  loadConfig().SEARCH_ENGINE_BACKENDS.NEWS || getSearchEngineBackend();
+
+export const getGoogleApiKey = () => loadConfig().SEARCH_ENGINES.GOOGLE.API_KEY;
+
+export const getGoogleCseId = () => loadConfig().SEARCH_ENGINES.GOOGLE.CSE_ID;
+
+export const getBraveApiKey = () => loadConfig().SEARCH_ENGINES.BRAVE.API_KEY;
+
+export const getBingSubscriptionKey = () =>
+  loadConfig().SEARCH_ENGINES.BING.SUBSCRIPTION_KEY;
+
+export const getYacyJsonEndpoint = () => loadConfig().SEARCH_ENGINES.YACY.ENDPOINT;
+
 export const getSearxngApiEndpoint = () =>
-  process.env.SEARXNG_API_URL || loadConfig().API_ENDPOINTS.SEARXNG;
+  process.env.SEARXNG_API_URL || loadConfig().SEARCH_ENGINES.SEARXNG.ENDPOINT;
 
 export const getOllamaApiEndpoint = () => loadConfig().MODELS.OLLAMA.API_URL;
 

src/lib/searchEngines/bing.ts

@@ -0,0 +1,102 @@
+import axios from 'axios';
+import { getBingSubscriptionKey } from '../../config';
+
+interface BingAPISearchResult {
+  _type: string;
+  name: string;
+  url: string;
+  displayUrl: string;
+  snippet?: string;
+  dateLastCrawled?: string;
+  thumbnailUrl?: string;
+  contentUrl?: string;
+  hostPageUrl?: string;
+  width?: number;
+  height?: number;
+  accentColor?: string;
+  contentSize?: string;
+  datePublished?: string;
+  encodingFormat?: string;
+  hostPageDisplayUrl?: string;
+  id?: string;
+  isLicensed?: boolean;
+  isFamilyFriendly?: boolean;
+  language?: string;
+  mediaUrl?: string;
+  motionThumbnailUrl?: string;
+  publisher?: string;
+  viewCount?: number;
+  webSearchUrl?: string;
+  primaryImageOfPage?: {
+    thumbnailUrl?: string;
+    width?: number;
+    height?: number;
+  };
+  video?: {
+    allowHttpsEmbed?: boolean;
+    embedHtml?: string;
+    allowMobileEmbed?: boolean;
+    viewCount?: number;
+    duration?: string;
+  };
+  image?: {
+    thumbnail?: {
+      contentUrl?: string;
+      width?: number;
+      height?: number;
+    };
+    imageInsightsToken?: string;
+    imageId?: string;
+  };
+}
+
+export const searchBingAPI = async (query: string) => {
+  try {
+    const bingApiKey = await getBingSubscriptionKey();
+    const url = new URL(`https://api.cognitive.microsoft.com/bing/v7.0/search`);
+    url.searchParams.append('q', query);
+    url.searchParams.append('responseFilter', 'Webpages,Images,Videos');
+
+    const res = await axios.get(url.toString(), {
+      headers: {
+        'Ocp-Apim-Subscription-Key': bingApiKey,
+        'Accept': 'application/json'
+      }
+    });
+
+    if (res.data.error) {
+      throw new Error(`Bing API Error: ${res.data.error.message}`);
+    }
+
+    const originalres = res.data;
+
+    // Extract web, image, and video results
+    const webResults = originalres.webPages?.value || [];
+    const imageResults = originalres.images?.value || [];
+    const videoResults = originalres.videos?.value || [];
+
+    const results = webResults.map((item: BingAPISearchResult) => ({
+      title: item.name,
+      url: item.url,
+      content: item.snippet,
+      img_src: item.primaryImageOfPage?.thumbnailUrl
+             || imageResults.find((img: any) => img.hostPageUrl === item.url)?.thumbnailUrl
+             || videoResults.find((vid: any) => vid.hostPageUrl === item.url)?.thumbnailUrl,
+      ...(item.video && {
+        videoData: {
+          duration: item.video.duration,
+          embedUrl: item.video.embedHtml?.match(/src="(.*?)"/)?.[1]
+        },
+      publisher: item.publisher,
+      datePublished: item.datePublished
+      })
+    }));
+
+    return { results, originalres };
+  } catch (error) {
+    const errorMessage = error.response?.data
+      ? JSON.stringify(error.response.data, null, 2)
+      : error.message || 'Unknown error';
+    throw new Error(`Bing API Error: ${errorMessage}`);
+  }
+};

src/lib/searchEngines/brave.ts

@@ -0,0 +1,96 @@
+import axios from 'axios';
+import { getBraveApiKey } from '../../config';
+
+interface BraveSearchResult {
+  title: string;
+  url: string;
+  content?: string;
+  img_src?: string;
+  age?: string;
+  family_friendly?: boolean;
+  language?: string;
+  video?: {
+    embedUrl?: string;
+    duration?: string;
+  };
+  rating?: {
+    value: number;
+    scale: number;
+  };
+  products?: Array<{
+    name: string;
+    price?: string;
+  }>;
+  recipe?: {
+    ingredients?: string[];
+    cookTime?: string;
+  };
+  meta?: {
+    fetched?: string;
+    lastCrawled?: string;
+  };
+}
+
+export const searchBraveAPI = async (
+  query: string,
+  numResults: number = 20
+): Promise<{ results: BraveSearchResult[]; originalres: any }> => {
+  try {
+    const braveApiKey = await getBraveApiKey();
+    const url = new URL(`https://api.search.brave.com/res/v1/web/search`);
+
+    url.searchParams.append('q', query);
+    url.searchParams.append('count', numResults.toString());
+
+    const res = await axios.get(url.toString(), {
+      headers: {
+        'X-Subscription-Token': braveApiKey,
+        'Accept': 'application/json'
+      }
+    });
+
+    if (res.data.error) {
+      throw new Error(`Brave API Error: ${res.data.error.message}`);
+    }
+
+    const originalres = res.data;
+    const webResults = originalres.web?.results || [];
+
+    const results: BraveSearchResult[] = webResults.map((item: any) => ({
+      title: item.title,
+      url: item.url,
+      content: item.description,
+      img_src: item.thumbnail?.src || item.deep_results?.images?.[0]?.src,
+      age: item.age,
+      family_friendly: item.family_friendly,
+      language: item.language,
+      video: item.video ? {
+        embedUrl: item.video.embed_url,
+        duration: item.video.duration
+      } : undefined,
+      rating: item.rating ? {
+        value: item.rating.value,
+        scale: item.rating.scale_max
+      } : undefined,
+      products: item.deep_results?.product_cluster?.map((p: any) => ({
+        name: p.name,
+        price: p.price
+      })),
+      recipe: item.recipe ? {
+        ingredients: item.recipe.ingredients,
+        cookTime: item.recipe.cook_time
+      } : undefined,
+      meta: {
+        fetched: item.meta?.fetched,
+        lastCrawled: item.meta?.last_crawled
+      }
+    }));
+
+    return { results, originalres };
+  } catch (error) {
+    const errorMessage = error.response?.data
+      ? JSON.stringify(error.response.data, null, 2)
+      : error.message || 'Unknown error';
+    throw new Error(`Brave API Error: ${errorMessage}`);
+  }
+};

src/lib/searchEngines/google_pse.ts

@@ -0,0 +1,84 @@
+import axios from 'axios';
+import { getGoogleApiKey, getGoogleCseId } from '../../config';
+
+interface GooglePSESearchResult {
+  kind: string;
+  title: string;
+  htmlTitle: string;
+  link: string;
+  displayLink: string;
+  snippet?: string;
+  htmlSnippet?: string;
+  cacheId?: string;
+  formattedUrl: string;
+  htmlFormattedUrl: string;
+  pagemap?: {
+    videoobject: any;
+    cse_thumbnail?: Array<{
+      src: string;
+      width: string;
+      height: string;
+    }>;
+    metatags?: Array<{
+      [key: string]: string;
+      'author'?: string;
+    }>;
+    cse_image?: Array<{
+      src: string;
+    }>;
+  };
+  fileFormat?: string;
+  image?: {
+    contextLink: string;
+    thumbnailLink: string;
+  };
+  mime?: string;
+  labels?: Array<{
+    name: string;
+    displayName: string;
+  }>;
+}
+
+export const searchGooglePSE = async (query: string) => {
+  try {
+    const [googleApiKey, googleCseID] = await Promise.all([
+      getGoogleApiKey(),
+      getGoogleCseId()
+    ]);
+
+    const url = new URL(`https://www.googleapis.com/customsearch/v1`);
+    url.searchParams.append('q', query);
+    url.searchParams.append('cx', googleCseID);
+    url.searchParams.append('key', googleApiKey);
+
+    const res = await axios.get(url.toString());
+
+    if (res.data.error) {
+      throw new Error(`Google PSE Error: ${res.data.error.message}`);
+    }
+
+    const originalres = res.data.items;
+
+    const results = originalres.map((item: GooglePSESearchResult) => ({
+      title: item.title,
+      url: item.link,
+      content: item.snippet,
+      img_src: item.pagemap?.cse_image?.[0]?.src
+             || item.pagemap?.cse_thumbnail?.[0]?.src
+             || item.image?.thumbnailLink,
+      ...(item.pagemap?.videoobject?.[0] && {
+        videoData: {
+          duration: item.pagemap.videoobject[0].duration,
+          embedUrl: item.pagemap.videoobject[0].embedurl
+        }
+      })
+    }));
+
+    return { results, originalres };
+  } catch (error) {
+    const errorMessage = error.response?.data
+      ? JSON.stringify(error.response.data, null, 2)
+      : error.message || 'Unknown error';
+    throw new Error(`Google PSE Error: ${errorMessage}`);
+  }
+};

src/lib/searchEngines/searxng.ts

@@ -1,5 +1,5 @@
 import axios from 'axios';
-import { getSearxngApiEndpoint } from '../config';
+import { getSearxngApiEndpoint } from '../../config';
 
 interface SearxngSearchOptions {
   categories?: string[];

src/lib/searchEngines/yacy.ts

@@ -0,0 +1,83 @@
+import axios from 'axios';
+import { getYacyJsonEndpoint } from '../../config';
+
+interface YaCySearchResult {
+  channels: {
+    title: string;
+    description: string;
+    link: string;
+    image: {
+      url: string;
+      title: string;
+      link: string;
+    };
+    startIndex: string;
+    itemsPerPage: string;
+    searchTerms: string;
+    items: {
+      title: string;
+      link: string;
+      code: string;
+      description: string;
+      pubDate: string;
+      image?: string;
+      size: string;
+      sizename: string;
+      guid: string;
+      faviconUrl: string;
+      host: string;
+      path: string;
+      file: string;
+      urlhash: string;
+      ranking: string;
+    }[];
+    navigation: {
+      facetname: string;
+      displayname: string;
+      type: string;
+      min: string;
+      max: string;
+      mean: string;
+      elements: {
+        name: string;
+        count: string;
+        modifier: string;
+        url: string;
+      }[];
+    }[];
+  }[];
+}
+
+
+export const searchYaCy = async (
+  query: string,
+  numResults: number = 20
+) => {
+  try {
+    const yacyBaseUrl = getYacyJsonEndpoint();
+
+    const url = new URL(`${yacyBaseUrl}/yacysearch.json`);
+    url.searchParams.append('query', query);
+    url.searchParams.append('count', numResults.toString());
+
+    const res = await axios.get(url.toString());
+
+    const originalres = res.data as YaCySearchResult;
+
+    const results = originalres.channels[0].items.map(item => ({
+      title: item.title,
+      url: item.link,
+      content: item.description,
+      img_src: item.image || null,
+      pubDate: item.pubDate,
+      host: item.host,
+    }));
+
+    return { results, originalres };
+  } catch (error) {
+    const errorMessage = error.response?.data
+      ? JSON.stringify(error.response.data, null, 2)
+      : error.message || 'Unknown error';
+    throw new Error(`YaCy Error: ${errorMessage}`);
+  }
+};

src/routes/discover.ts

@@ -1,42 +1,125 @@
 import express from 'express';
-import { searchSearxng } from '../lib/searxng';
+import { searchSearxng } from '../lib/searchEngines/searxng';
+import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
+import { searchYaCy } from '../lib/searchEngines/yacy';
+import { searchBingAPI } from '../lib/searchEngines/bing';
+import { getNewsSearchEngineBackend } from '../config';
 import logger from '../utils/logger';
 
 const router = express.Router();
 
+async function performSearch(query: string, site: string) {
+  const searchEngine = getNewsSearchEngineBackend();
+  switch (searchEngine) {
+    case 'google': {
+      const googleResult = await searchGooglePSE(query);
+
+      return googleResult.originalres.map(item => {
+        const imageSources = [
+          item.pagemap?.cse_image?.[0]?.src,
+          item.pagemap?.cse_thumbnail?.[0]?.src,
+          item.pagemap?.metatags?.[0]?.['og:image'],
+          item.pagemap?.metatags?.[0]?.['twitter:image'],
+          item.pagemap?.metatags?.[0]?.['image'],
+        ].filter(Boolean); // Remove undefined values
+
+        return {
+          title: item.title,
+          url: item.link,
+          content: item.snippet,
+          thumbnail: imageSources[0], // First available image
+          img_src: imageSources[0],   // Same as thumbnail for consistency
+          iframe_src: null,
+          author: item.pagemap?.metatags?.[0]?.['og:site_name'] || site,
+          publishedDate: item.pagemap?.metatags?.[0]?.['article:published_time']
+        };
+      });
+    }
+
+    case 'searxng': {
+      const searxResult = await searchSearxng(query, {
+        engines: ['bing news'],
+        pageno: 1,
+      });
+      return searxResult.results;
+    }
+
+    case 'brave': {
+      const braveResult = await searchBraveAPI(query);
+      return braveResult.results.map(item => ({
+        title: item.title,
+        url: item.url,
+        content: item.content,
+        thumbnail: item.img_src,
+        img_src: item.img_src,
+        iframe_src: null,
+        author: item.meta?.fetched || site,
+        publishedDate: item.meta?.lastCrawled
+      }));
+    }
+
+    case 'yacy': {
+      const yacyResult = await searchYaCy(query);
+      return yacyResult.results.map((item) => ({
+        title: item.title,
+        url: item.url,
+        content: item.content,
+        thumbnail: item.img_src,
+        img_src: item.img_src,
+        iframe_src: null,
+        author: item?.host || site,
+        publishedDate: item?.pubDate
+      }))
+    }
+
+    case 'bing': {
+      const bingResult = await searchBingAPI(query);
+      return bingResult.results.map(item => ({
+        title: item.title,
+        url: item.url,
+        content: item.content,
+        thumbnail: item.img_src,
+        img_src: item.img_src,
+        iframe_src: null,
+        author: item?.publisher || site,
+        publishedDate: item?.datePublished
+      }))
+    }
+
+    default:
+      throw new Error(`Unknown search engine ${searchEngine}`);
+  }
+}
+
+
 router.get('/', async (req, res) => {
   try {
+    const queries = [
+      { site: 'businessinsider.com', topic: 'AI' },
+      { site: 'www.exchangewire.com', topic: 'AI' },
+      { site: 'yahoo.com', topic: 'AI' },
+      { site: 'businessinsider.com', topic: 'tech' },
+      { site: 'www.exchangewire.com', topic: 'tech' },
+      { site: 'yahoo.com', topic: 'tech' },
+    ];
+
     const data = (
-      await Promise.all([
-        searchSearxng('site:businessinsider.com AI', {
-          engines: ['bing news'],
-          pageno: 1,
-        }),
-        searchSearxng('site:www.exchangewire.com AI', {
-          engines: ['bing news'],
-          pageno: 1,
-        }),
-        searchSearxng('site:yahoo.com AI', {
-          engines: ['bing news'],
-          pageno: 1,
-        }),
-        searchSearxng('site:businessinsider.com tech', {
-          engines: ['bing news'],
-          pageno: 1,
-        }),
-        searchSearxng('site:www.exchangewire.com tech', {
-          engines: ['bing news'],
-          pageno: 1,
-        }),
-        searchSearxng('site:yahoo.com tech', {
-          engines: ['bing news'],
-          pageno: 1,
-        }),
-      ])
+      await Promise.all(
+        queries.map(async ({ site, topic }) => {
+          try {
+            const query = `site:${site} ${topic}`;
+            return await performSearch(query, site);
+          } catch (error) {
+            logger.error(`Error searching ${site}: ${error.message}`);
+            return [];
+          }
+        })
+      )
     )
-      .map((result) => result.results)
       .flat()
-      .sort(() => Math.random() - 0.5);
+      .sort(() => Math.random() - 0.5)
+      .filter(item => item.title && item.url && item.content);
 
     return res.json({ blogs: data });
   } catch (err: any) {

src/routes/search.ts

@@ -85,10 +85,12 @@ router.post('/', async (req, res) => {
     if (body.chatModel?.provider === 'custom_openai') {
       llm = new ChatOpenAI({
         modelName: body.chatModel?.model || getCustomOpenaiModelName(),
-        openAIApiKey: body.chatModel?.customOpenAIKey || getCustomOpenaiApiKey(),
+        openAIApiKey:
+          body.chatModel?.customOpenAIKey || getCustomOpenaiApiKey(),
         temperature: 0.7,
         configuration: {
-          baseURL: body.chatModel?.customOpenAIBaseURL || getCustomOpenaiApiUrl(),
+          baseURL:
+            body.chatModel?.customOpenAIBaseURL || getCustomOpenaiApiUrl(),
         },
       }) as unknown as BaseChatModel;
     } else if (

src/search/metaSearchAgent.ts

@@ -17,7 +17,12 @@ import LineListOutputParser from '../lib/outputParsers/listLineOutputParser';
 import LineOutputParser from '../lib/outputParsers/lineOutputParser';
 import { getDocumentsFromLinks } from '../utils/documents';
 import { Document } from 'langchain/document';
-import { searchSearxng } from '../lib/searxng';
+import { searchSearxng } from '../lib/searchEngines/searxng';
+import { searchGooglePSE } from '../lib/searchEngines/google_pse';
+import { searchBingAPI } from '../lib/searchEngines/bing';
+import { searchBraveAPI } from '../lib/searchEngines/brave';
+import { searchYaCy } from '../lib/searchEngines/yacy';
+import { getSearchEngineBackend } from '../config';
 import path from 'path';
 import fs from 'fs';
 import computeSimilarity from '../utils/computeSimilarity';
@@ -203,10 +208,36 @@ class MetaSearchAgent implements MetaSearchAgentType {
 
           return { query: question, docs: docs };
         } else {
-          const res = await searchSearxng(question, {
+
+          const searchEngine = getSearchEngineBackend();
+
+          let res;
+          switch (searchEngine) {
+            case 'searxng':
+              res = await searchSearxng(question, {
                 language: 'en',
                 engines: this.config.activeEngines,
               });
+              break;
+            case 'google':
+              res = await searchGooglePSE(question);
+              break;
+            case 'bing':
+              res = await searchBingAPI(question);
+              break;
+            case 'brave':
+              res = await searchBraveAPI(question);
+              break;
+            case 'yacy':
+              res = await searchYaCy(question);
+              break;
+            default:
+              throw new Error(`Unknown search engine ${searchEngine}`);
+          }
+
+          if (!res?.results) {
+            throw new Error(`No results found for search engine: ${searchEngine}`);
+          }
 
           const documents = res.results.map(
             (result) =>

ui/app/settings/page.tsx

@@ -223,11 +223,11 @@ const Page = () => {
         setChatModels(data.chatModelProviders || {});
         setEmbeddingModels(data.embeddingModelProviders || {});
 
-        const currentProvider = selectedChatModelProvider;
-        const newProviders = Object.keys(data.chatModelProviders || {});
+        const currentChatProvider = selectedChatModelProvider;
+        const newChatProviders = Object.keys(data.chatModelProviders || {});
 
-        if (!currentProvider && newProviders.length > 0) {
-          const firstProvider = newProviders[0];
+        if (!currentChatProvider && newChatProviders.length > 0) {
+          const firstProvider = newChatProviders[0];
           const firstModel = data.chatModelProviders[firstProvider]?.[0]?.name;
 
           if (firstModel) {
@@ -237,11 +237,11 @@ const Page = () => {
             localStorage.setItem('chatModel', firstModel);
           }
         } else if (
-          currentProvider &&
+          currentChatProvider &&
           (!data.chatModelProviders ||
-            !data.chatModelProviders[currentProvider] ||
-            !Array.isArray(data.chatModelProviders[currentProvider]) ||
-            data.chatModelProviders[currentProvider].length === 0)
+            !data.chatModelProviders[currentChatProvider] ||
+            !Array.isArray(data.chatModelProviders[currentChatProvider]) ||
+            data.chatModelProviders[currentChatProvider].length === 0)
         ) {
           const firstValidProvider = Object.entries(
             data.chatModelProviders || {},
@@ -267,6 +267,55 @@ const Page = () => {
           }
         }
 
+        const currentEmbeddingProvider = selectedEmbeddingModelProvider;
+        const newEmbeddingProviders = Object.keys(
+          data.embeddingModelProviders || {},
+        );
+
+        if (!currentEmbeddingProvider && newEmbeddingProviders.length > 0) {
+          const firstProvider = newEmbeddingProviders[0];
+          const firstModel =
+            data.embeddingModelProviders[firstProvider]?.[0]?.name;
+
+          if (firstModel) {
+            setSelectedEmbeddingModelProvider(firstProvider);
+            setSelectedEmbeddingModel(firstModel);
+            localStorage.setItem('embeddingModelProvider', firstProvider);
+            localStorage.setItem('embeddingModel', firstModel);
+          }
+        } else if (
+          currentEmbeddingProvider &&
+          (!data.embeddingModelProviders ||
+            !data.embeddingModelProviders[currentEmbeddingProvider] ||
+            !Array.isArray(
+              data.embeddingModelProviders[currentEmbeddingProvider],
+            ) ||
+            data.embeddingModelProviders[currentEmbeddingProvider].length === 0)
+        ) {
+          const firstValidProvider = Object.entries(
+            data.embeddingModelProviders || {},
+          ).find(
+            ([_, models]) => Array.isArray(models) && models.length > 0,
+          )?.[0];
+
+          if (firstValidProvider) {
+            setSelectedEmbeddingModelProvider(firstValidProvider);
+            setSelectedEmbeddingModel(
+              data.embeddingModelProviders[firstValidProvider][0].name,
+            );
+            localStorage.setItem('embeddingModelProvider', firstValidProvider);
+            localStorage.setItem(
+              'embeddingModel',
+              data.embeddingModelProviders[firstValidProvider][0].name,
+            );
+          } else {
+            setSelectedEmbeddingModelProvider(null);
+            setSelectedEmbeddingModel(null);
+            localStorage.removeItem('embeddingModelProvider');
+            localStorage.removeItem('embeddingModel');
+          }
+        }
+
         setConfig(data);
       }
 
@@ -278,6 +327,10 @@ const Page = () => {
         localStorage.setItem('chatModelProvider', value);
       } else if (key === 'chatModel') {
         localStorage.setItem('chatModel', value);
+      } else if (key === 'embeddingModelProvider') {
+        localStorage.setItem('embeddingModelProvider', value);
+      } else if (key === 'embeddingModel') {
+        localStorage.setItem('embeddingModel', value);
       }
     } catch (err) {
       console.error('Failed to save:', err);
@@ -436,7 +489,6 @@ const Page = () => {
                         const value = e.target.value;
                         setSelectedChatModelProvider(value);
                         saveConfig('chatModelProvider', value);
-                        // Auto-select first model of new provider
                         const firstModel =
                           config.chatModelProviders[value]?.[0]?.name;
                         if (firstModel) {
@@ -511,12 +563,16 @@ const Page = () => {
                       <Input
                         type="text"
                         placeholder="Model name"
-                        defaultValue={config.customOpenaiModelName}
-                        onChange={(e) =>
-                          setConfig({
-                            ...config,
+                        value={config.customOpenaiModelName}
+                        isSaving={savingStates['customOpenaiModelName']}
+                        onChange={(e: React.ChangeEvent<HTMLInputElement>) => {
+                          setConfig((prev) => ({
+                            ...prev!,
                             customOpenaiModelName: e.target.value,
-                          })
+                          }));
+                        }}
+                        onSave={(value) =>
+                          saveConfig('customOpenaiModelName', value)
                         }
                       />
                     </div>
@@ -527,12 +583,16 @@ const Page = () => {
                       <Input
                         type="text"
                         placeholder="Custom OpenAI API Key"
-                        defaultValue={config.customOpenaiApiKey}
-                        onChange={(e) =>
-                          setConfig({
-                            ...config,
+                        value={config.customOpenaiApiKey}
+                        isSaving={savingStates['customOpenaiApiKey']}
+                        onChange={(e: React.ChangeEvent<HTMLInputElement>) => {
+                          setConfig((prev) => ({
+                            ...prev!,
                             customOpenaiApiKey: e.target.value,
-                          })
+                          }));
+                        }}
+                        onSave={(value) =>
+                          saveConfig('customOpenaiApiKey', value)
                         }
                       />
                     </div>
@@ -543,17 +603,96 @@ const Page = () => {
                       <Input
                         type="text"
                         placeholder="Custom OpenAI Base URL"
-                        defaultValue={config.customOpenaiApiUrl}
-                        onChange={(e) =>
-                          setConfig({
-                            ...config,
+                        value={config.customOpenaiApiUrl}
+                        isSaving={savingStates['customOpenaiApiUrl']}
+                        onChange={(e: React.ChangeEvent<HTMLInputElement>) => {
+                          setConfig((prev) => ({
+                            ...prev!,
                             customOpenaiApiUrl: e.target.value,
-                          })
+                          }));
+                        }}
+                        onSave={(value) =>
+                          saveConfig('customOpenaiApiUrl', value)
                         }
                       />
                     </div>
                   </div>
                 )}
+
+              {config.embeddingModelProviders && (
+                <div className="flex flex-col space-y-4 mt-4 pt-4 border-t border-light-200 dark:border-dark-200">
+                  <div className="flex flex-col space-y-1">
+                    <p className="text-black/70 dark:text-white/70 text-sm">
+                      Embedding Model Provider
+                    </p>
+                    <Select
+                      value={selectedEmbeddingModelProvider ?? undefined}
+                      onChange={(e) => {
+                        const value = e.target.value;
+                        setSelectedEmbeddingModelProvider(value);
+                        saveConfig('embeddingModelProvider', value);
+                        const firstModel =
+                          config.embeddingModelProviders[value]?.[0]?.name;
+                        if (firstModel) {
+                          setSelectedEmbeddingModel(firstModel);
+                          saveConfig('embeddingModel', firstModel);
+                        }
+                      }}
+                      options={Object.keys(config.embeddingModelProviders).map(
+                        (provider) => ({
+                          value: provider,
+                          label:
+                            provider.charAt(0).toUpperCase() +
+                            provider.slice(1),
+                        }),
+                      )}
+                    />
+                  </div>
+
+                  {selectedEmbeddingModelProvider && (
+                    <div className="flex flex-col space-y-1">
+                      <p className="text-black/70 dark:text-white/70 text-sm">
+                        Embedding Model
+                      </p>
+                      <Select
+                        value={selectedEmbeddingModel ?? undefined}
+                        onChange={(e) => {
+                          const value = e.target.value;
+                          setSelectedEmbeddingModel(value);
+                          saveConfig('embeddingModel', value);
+                        }}
+                        options={(() => {
+                          const embeddingModelProvider =
+                            config.embeddingModelProviders[
+                              selectedEmbeddingModelProvider
+                            ];
+                          return embeddingModelProvider
+                            ? embeddingModelProvider.length > 0
+                              ? embeddingModelProvider.map((model) => ({
+                                  value: model.name,
+                                  label: model.displayName,
+                                }))
+                              : [
+                                  {
+                                    value: '',
+                                    label: 'No models available',
+                                    disabled: true,
+                                  },
+                                ]
+                            : [
+                                {
+                                  value: '',
+                                  label:
+                                    'Invalid provider, please check backend logs',
+                                  disabled: true,
+                                },
+                              ];
+                        })()}
+                      />
+                    </div>
+                  )}
+                </div>
+              )}
             </SettingsSection>
 
             <SettingsSection title="API Keys">

ui/components/MessageBox.tsx

@@ -68,7 +68,7 @@ const MessageBox = ({
   return (
     <div>
       {message.role === 'user' && (
-        <div className={cn('w-full', messageIndex === 0 ? 'pt-16' : 'pt-8')}>
+        <div className={cn('w-full', messageIndex === 0 ? 'pt-16' : 'pt-8', 'break-words')}>
           <h2 className="text-black dark:text-white font-medium text-3xl lg:w-9/12">
             {message.content}
           </h2>

ui/components/MessageInputActions/Attach.tsx

@@ -110,7 +110,7 @@ const Attach = ({
                 <button
                   type="button"
                   onClick={() => fileInputRef.current.click()}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <input
                     type="file"
@@ -128,7 +128,7 @@ const Attach = ({
                     setFiles([]);
                     setFileIds([]);
                   }}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <Trash size={14} />
                   <p className="text-xs">Clear</p>
@@ -145,7 +145,7 @@ const Attach = ({
                   <div className="bg-dark-100 flex items-center justify-center w-10 h-10 rounded-md">
                     <File size={16} className="text-white/70" />
                   </div>
-                  <p className="text-white/70 text-sm">
+                  <p className="text-black/70 dark:text-white/70 text-sm">
                     {file.fileName.length > 25
                       ? file.fileName.replace(/\.\w+$/, '').substring(0, 25) +
                         '...' +

ui/components/MessageInputActions/AttachSmall.tsx

@@ -82,7 +82,7 @@ const AttachSmall = ({
                 <button
                   type="button"
                   onClick={() => fileInputRef.current.click()}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <input
                     type="file"
@@ -100,7 +100,7 @@ const AttachSmall = ({
                     setFiles([]);
                     setFileIds([]);
                   }}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <Trash size={14} />
                   <p className="text-xs">Clear</p>
@@ -117,7 +117,7 @@ const AttachSmall = ({
                   <div className="bg-dark-100 flex items-center justify-center w-10 h-10 rounded-md">
                     <File size={16} className="text-white/70" />
                   </div>
-                  <p className="text-white/70 text-sm">
+                  <p className="text-black/70 dark:text-white/70 text-sm">
                     {file.fileName.length > 25
                       ? file.fileName.replace(/\.\w+$/, '').substring(0, 25) +
                         '...' +