remi/Perplexica
1
0
Fork 0
mirror of https://github.com/ItzCrazyKns/Perplexica.git synced 2025-10-30 19:08:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge e20d5ecc01 into 41b258e4d8

Browse Source
This commit is contained in:
OTYAK
2025-04-11 15:05:43 +00:00
committed by GitHub
parent 41b258e4d8 e20d5ecc01
commit f481eff50a
10 changed files with 206 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
sample.config.toml
View File

@@ -26,4 +26,8 @@ API_URL = "" # Ollama API URL - http://host.docker.internal:11434
API_KEY = ""
[API_ENDPOINTS]
SEARXNG = "" # SearxNG API URL - http://localhost:32768
SEARXNG = "" # SearxNG API URL - http://localhost:32768
TAVILY = "" # Tavily API key
[SEARCH]
ENGINE = "searxng" # "searxng" or "tavily"

10
src/app/api/config/route.ts
View File

@@ -8,6 +8,8 @@ import {
getOllamaApiEndpoint,
getOpenaiApiKey,
getDeepseekApiKey,
getSearchEngine,
getTavilyApiKey,
updateConfig,
} from '@/lib/config';
import {
@@ -58,6 +60,8 @@ export const GET = async (req: Request) => {
config['customOpenaiApiUrl'] = getCustomOpenaiApiUrl();
config['customOpenaiApiKey'] = getCustomOpenaiApiKey();
config['customOpenaiModelName'] = getCustomOpenaiModelName();
config['searchEngine'] = getSearchEngine();
config['tavilyApiKey'] = getTavilyApiKey();
return Response.json({ ...config }, { status: 200 });
} catch (err) {
@@ -99,6 +103,12 @@ export const POST = async (req: Request) => {
MODEL_NAME: config.customOpenaiModelName,
},
},
SEARCH: {
ENGINE: config.searchEngine,
},
API_ENDPOINTS: {
TAVILY: config.tavilyApiKey || '',
},
};
updateConfig(updatedConfig);

2
src/app/api/discover/route.ts
View File

@@ -1,4 +1,4 @@
import { searchSearxng } from '@/lib/searxng';
import { searchSearxng } from '../../../lib/searchEngines/searxng';
const articleWebsites = [
'yahoo.com',

60
src/app/settings/page.tsx
View File

@@ -24,6 +24,8 @@ interface SettingsType {
customOpenaiApiKey: string;
customOpenaiApiUrl: string;
customOpenaiModelName: string;
searchEngine: string;
tavilyApiKey?: string;
}
interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
@@ -145,6 +147,7 @@ const Page = () => {
const [automaticImageSearch, setAutomaticImageSearch] = useState(false);
const [automaticVideoSearch, setAutomaticVideoSearch] = useState(false);
const [systemInstructions, setSystemInstructions] = useState<string>('');
const [searchEngine, setSearchEngine] = useState<string>('searxng');
const [savingStates, setSavingStates] = useState<Record<string, boolean>>({});
useEffect(() => {
@@ -207,6 +210,7 @@ const Page = () => {
);
setSystemInstructions(localStorage.getItem('systemInstructions')!);
setSearchEngine(localStorage.getItem('searchEngine') || 'searxng');
setIsLoading(false);
};
@@ -366,6 +370,10 @@ const Page = () => {
localStorage.setItem('embeddingModel', value);
} else if (key === 'systemInstructions') {
localStorage.setItem('systemInstructions', value);
} else if (key === 'searchEngine') {
localStorage.setItem('searchEngine', value);
} else if (key === 'tavilyApiKey') {
localStorage.setItem('tavilyApiKey', value);
}
} catch (err) {
console.error('Failed to save:', err);
@@ -508,6 +516,32 @@ const Page = () => {
/>
</Switch>
</div>
<div className="flex flex-col space-y-1 mt-2">
<p className="text-black/70 dark:text-white/70 text-sm">
Search Engine
</p>
<Select
value={searchEngine}
onChange={(e) => {
const value = e.target.value;
setSearchEngine(value);
saveConfig('searchEngine', value);
}}
options={[
{ value: 'searxng', label: 'SearxNG' },
...(config.tavilyApiKey ? [{ value: 'tavily', label: 'Tavily' }] : []),
]}
/>
<p className="text-xs text-black/60 dark:text-white/60 mt-1">
Select which search engine to use for web searches
</p>
{searchEngine === 'tavily' && !config.tavilyApiKey && (
<p className="text-xs text-red-500 mt-1">
Tavily API key is required to use this search engine
</p>
)}
</div>
</div>
</SettingsSection>
@@ -858,6 +892,32 @@ const Page = () => {
onSave={(value) => saveConfig('deepseekApiKey', value)}
/>
</div>
<div className="flex flex-col space-y-1 mt-4 pt-4 border-t border-light-200 dark:border-dark-200">
<p className="text-black/90 dark:text-white/90 font-medium">Search Engine API Keys</p>
<p className="text-sm text-black/60 dark:text-white/60 mt-0.5">
API keys for search engines used in the application
</p>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Tavily API Key
</p>
<Input
type="text"
placeholder="Tavily API key"
value={config.tavilyApiKey || ''}
isSaving={savingStates['tavilyApiKey']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
tavilyApiKey: e.target.value,
}));
}}
onSave={(value) => saveConfig('tavilyApiKey', value)}
/>
</div>
</div>
</SettingsSection>
</div>

2
src/lib/chains/imageSearchAgent.ts
View File

@@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { searchSearxng } from '../searxng';
import { searchSearxng } from '../searchEngines/searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const imageSearchChainPrompt = `

2
src/lib/chains/videoSearchAgent.ts
View File

@@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { searchSearxng } from '../searxng';
import { searchSearxng } from '../searchEngines/searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const VideoSearchChainPrompt = `

10
src/lib/config.ts
View File

@@ -36,6 +36,10 @@ interface Config {
};
API_ENDPOINTS: {
SEARXNG: string;
TAVILY: string;
};
SEARCH: {
ENGINE: string;
};
}
@@ -64,6 +68,12 @@ export const getGeminiApiKey = () => loadConfig().MODELS.GEMINI.API_KEY;
export const getSearxngApiEndpoint = () =>
process.env.SEARXNG_API_URL || loadConfig().API_ENDPOINTS.SEARXNG;
export const getTavilyApiKey = () =>
process.env.TAVILY_API_KEY || loadConfig().API_ENDPOINTS.TAVILY;
export const getSearchEngine = () =>
process.env.SEARCH_ENGINE || loadConfig().SEARCH?.ENGINE || 'searxng';
export const getOllamaApiEndpoint = () => loadConfig().MODELS.OLLAMA.API_URL;
export const getDeepseekApiKey = () => loadConfig().MODELS.DEEPSEEK.API_KEY;

57
src/lib/search/metaSearchAgent.ts
View File

@@ -17,7 +17,9 @@ import LineListOutputParser from '../outputParsers/listLineOutputParser';
import LineOutputParser from '../outputParsers/lineOutputParser';
import { getDocumentsFromLinks } from '../utils/documents';
import { Document } from 'langchain/document';
import { searchSearxng } from '../searxng';
import { searchTavily } from '../searchEngines/tavily';
import { searchSearxng } from '../searchEngines/searxng';
import { getSearchEngine } from '../config';
import path from 'node:path';
import fs from 'node:fs';
import computeSimilarity from '../utils/computeSimilarity';
@@ -205,25 +207,42 @@ class MetaSearchAgent implements MetaSearchAgentType {
} else {
question = question.replace(/<think>.*?<\/think>/g, '');
const res = await searchSearxng(question, {
language: 'en',
engines: this.config.activeEngines,
});
const searchEngine = getSearchEngine();
const documents = res.results.map(
(result) =>
new Document({
pageContent:
result.content ||
(this.config.activeEngines.includes('youtube')
? result.title
: '') /* Todo: Implement transcript grabbing using Youtubei (source: https://www.npmjs.com/package/youtubei) */,
metadata: {
title: result.title,
url: result.url,
...(result.img_src && { img_src: result.img_src }),
},
}),
let res;
if (searchEngine === 'tavily') {
res = await searchTavily(question, {
search_depth: 'basic',
max_results: 15,
include_images: true,
});
} else {
// Default to SearxNG
res = await searchSearxng(question, {
language: 'en',
engines: this.config.activeEngines,
});
}
let documents: Document[] = [];
documents = documents.concat(
res.results.map(
(result) =>
new Document({
pageContent:
result.content ||
(this.config.activeEngines.includes('youtube')
? result.title
: ''),
metadata: {
title: result.title,
url: result.url,
...(result.img_src ? { img_src: result.img_src } : {}),
},
}),
)
);
return { query: question, docs: documents };

2
src/lib/searxng.ts → src/lib/searchEngines/searxng.ts
View File

@@ -1,5 +1,5 @@
import axios from 'axios';
import { getSearxngApiEndpoint } from './config';
import { getSearxngApiEndpoint } from '../config';
interface SearxngSearchOptions {
categories?: string[];

79
src/lib/searchEngines/tavily.ts Normal file
View File

@@ -0,0 +1,79 @@
import axios from 'axios';
import { getTavilyApiKey } from '../config';
interface TavilySearchOptions {
topic?: 'general' | 'news';
search_depth?: 'basic' | 'advanced';
chunks_per_source?: number;
max_results?: number;
time_range?: 'day' | 'week' | 'month' | 'year' | 'd' | 'w' | 'm' | 'y';
days?: number;
include_answer?: boolean | 'basic' | 'advanced';
include_raw_content?: boolean;
include_images?: boolean;
include_image_descriptions?: boolean;
include_domains?: string[];
exclude_domains?: string[];
}
interface TavilySearchResult {
title: string;
url: string;
content: string;
score: number;
raw_content?: string;
}
interface TavilySearchResponse {
query: string;
answer?: string;
images?: Array<{
url: string;
description?: string;
}>;
results: TavilySearchResult[];
response_time: string;
}
export const searchTavily = async (
query: string,
opts?: TavilySearchOptions,
) => {
const tavilyApiKey = getTavilyApiKey();
if (!tavilyApiKey) {
throw new Error('Tavily API key is not configured');
}
const url = 'https://api.tavily.com/search';
const response = await axios.post<TavilySearchResponse>(
url,
{
query,
...opts,
},
{
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${tavilyApiKey}`,
},
}
);
const results = response.data.results;
// Convert Tavily results to match the format expected by the rest of the application
const formattedResults = results.map(result => ({
title: result.title,
url: result.url,
content: result.content,
img_src: undefined, // Tavily doesn't provide image URLs in the standard response
}));
return {
results: formattedResults,
suggestions: [], // Tavily doesn't provide suggestions, so return empty array
answer: response.data.answer, // Include the AI-generated answer if available
};
};