remi/Perplexica
1
0
Fork 0
mirror of https://github.com/ItzCrazyKns/Perplexica.git synced 2025-04-30 00:02:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge e20d5ecc017bb7295d37ed0ad56dc3f9bb4c82a4 into 41b258e4d894972876205cdbcff9d2c05241c2b8

Browse Source
This commit is contained in:
OTYAK
2025-04-11 15:05:43 +00:00
committed by GitHub
parent 41b258e4d8 e20d5ecc01
commit f481eff50a
10 changed files with 206 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sample.config.toml
View File

@ -26,4 +26,8 @@ API_URL = "" # Ollama API URL - http://host.docker.internal:11434
API_KEY = ""
[API_ENDPOINTS]
SEARXNG = "" # SearxNG API URL - http://localhost:32768
SEARXNG = "" # SearxNG API URL - http://localhost:32768
TAVILY = "" # Tavily API key
[SEARCH]
ENGINE = "searxng" # "searxng" or "tavily"

10
src/app/api/config/route.ts
View File

@ -8,6 +8,8 @@ import {
getOllamaApiEndpoint,
getOpenaiApiKey,
getDeepseekApiKey,
getSearchEngine,
getTavilyApiKey,
updateConfig,
} from '@/lib/config';
import {
@ -58,6 +60,8 @@ export const GET = async (req: Request) => {
config['customOpenaiApiUrl'] = getCustomOpenaiApiUrl();
config['customOpenaiApiKey'] = getCustomOpenaiApiKey();
config['customOpenaiModelName'] = getCustomOpenaiModelName();
config['searchEngine'] = getSearchEngine();
config['tavilyApiKey'] = getTavilyApiKey();
return Response.json({ ...config }, { status: 200 });
} catch (err) {
@ -99,6 +103,12 @@ export const POST = async (req: Request) => {
MODEL_NAME: config.customOpenaiModelName,
},
},
SEARCH: {
ENGINE: config.searchEngine,
},
API_ENDPOINTS: {
TAVILY: config.tavilyApiKey || '',
},
};
updateConfig(updatedConfig);

2
src/app/api/discover/route.ts
View File

@ -1,4 +1,4 @@
import { searchSearxng } from '@/lib/searxng';
import { searchSearxng } from '../../../lib/searchEngines/searxng';
const articleWebsites = [
'yahoo.com',

60
src/app/settings/page.tsx
View File

@ -24,6 +24,8 @@ interface SettingsType {
customOpenaiApiKey: string;
customOpenaiApiUrl: string;
customOpenaiModelName: string;
searchEngine: string;
tavilyApiKey?: string;
}
interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
@ -145,6 +147,7 @@ const Page = () => {
const [automaticImageSearch, setAutomaticImageSearch] = useState(false);
const [automaticVideoSearch, setAutomaticVideoSearch] = useState(false);
const [systemInstructions, setSystemInstructions] = useState<string>('');
const [searchEngine, setSearchEngine] = useState<string>('searxng');
const [savingStates, setSavingStates] = useState<Record<string, boolean>>({});
useEffect(() => {
@ -207,6 +210,7 @@ const Page = () => {
);
setSystemInstructions(localStorage.getItem('systemInstructions')!);
setSearchEngine(localStorage.getItem('searchEngine') || 'searxng');
setIsLoading(false);
};
@ -366,6 +370,10 @@ const Page = () => {
localStorage.setItem('embeddingModel', value);
} else if (key === 'systemInstructions') {
localStorage.setItem('systemInstructions', value);
} else if (key === 'searchEngine') {
localStorage.setItem('searchEngine', value);
} else if (key === 'tavilyApiKey') {
localStorage.setItem('tavilyApiKey', value);
}
} catch (err) {
console.error('Failed to save:', err);
@ -508,6 +516,32 @@ const Page = () => {
/>
</Switch>
</div>
<div className="flex flex-col space-y-1 mt-2">
<p className="text-black/70 dark:text-white/70 text-sm">
Search Engine
</p>
<Select
value={searchEngine}
onChange={(e) => {
const value = e.target.value;
setSearchEngine(value);
saveConfig('searchEngine', value);
}}
options={[
{ value: 'searxng', label: 'SearxNG' },
...(config.tavilyApiKey ? [{ value: 'tavily', label: 'Tavily' }] : []),
]}
/>
<p className="text-xs text-black/60 dark:text-white/60 mt-1">
Select which search engine to use for web searches
</p>
{searchEngine === 'tavily' && !config.tavilyApiKey && (
<p className="text-xs text-red-500 mt-1">
Tavily API key is required to use this search engine
</p>
)}
</div>
</div>
</SettingsSection>
@ -858,6 +892,32 @@ const Page = () => {
onSave={(value) => saveConfig('deepseekApiKey', value)}
/>
</div>
<div className="flex flex-col space-y-1 mt-4 pt-4 border-t border-light-200 dark:border-dark-200">
<p className="text-black/90 dark:text-white/90 font-medium">Search Engine API Keys</p>
<p className="text-sm text-black/60 dark:text-white/60 mt-0.5">
API keys for search engines used in the application
</p>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Tavily API Key
</p>
<Input
type="text"
placeholder="Tavily API key"
value={config.tavilyApiKey || ''}
isSaving={savingStates['tavilyApiKey']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
tavilyApiKey: e.target.value,
}));
}}
onSave={(value) => saveConfig('tavilyApiKey', value)}
/>
</div>
</div>
</SettingsSection>
</div>

2
src/lib/chains/imageSearchAgent.ts
View File

@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { searchSearxng } from '../searxng';
import { searchSearxng } from '../searchEngines/searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const imageSearchChainPrompt = `

2
src/lib/chains/videoSearchAgent.ts
View File

@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { searchSearxng } from '../searxng';
import { searchSearxng } from '../searchEngines/searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const VideoSearchChainPrompt = `

10
src/lib/config.ts
View File

@ -36,6 +36,10 @@ interface Config {
};
API_ENDPOINTS: {
SEARXNG: string;
TAVILY: string;
};
SEARCH: {
ENGINE: string;
};
}
@ -64,6 +68,12 @@ export const getGeminiApiKey = () => loadConfig().MODELS.GEMINI.API_KEY;
export const getSearxngApiEndpoint = () =>
process.env.SEARXNG_API_URL || loadConfig().API_ENDPOINTS.SEARXNG;
export const getTavilyApiKey = () =>
process.env.TAVILY_API_KEY || loadConfig().API_ENDPOINTS.TAVILY;
export const getSearchEngine = () =>
process.env.SEARCH_ENGINE || loadConfig().SEARCH?.ENGINE || 'searxng';
export const getOllamaApiEndpoint = () => loadConfig().MODELS.OLLAMA.API_URL;
export const getDeepseekApiKey = () => loadConfig().MODELS.DEEPSEEK.API_KEY;

57
src/lib/search/metaSearchAgent.ts
View File

@ -17,7 +17,9 @@ import LineListOutputParser from '../outputParsers/listLineOutputParser';
import LineOutputParser from '../outputParsers/lineOutputParser';
import { getDocumentsFromLinks } from '../utils/documents';
import { Document } from 'langchain/document';
import { searchSearxng } from '../searxng';
import { searchTavily } from '../searchEngines/tavily';
import { searchSearxng } from '../searchEngines/searxng';
import { getSearchEngine } from '../config';
import path from 'node:path';
import fs from 'node:fs';
import computeSimilarity from '../utils/computeSimilarity';
@ -205,25 +207,42 @@ class MetaSearchAgent implements MetaSearchAgentType {
} else {
question = question.replace(/<think>.*?<\/think>/g, '');
const res = await searchSearxng(question, {
language: 'en',
engines: this.config.activeEngines,
});
const searchEngine = getSearchEngine();
const documents = res.results.map(
(result) =>
new Document({
pageContent:
result.content ||
(this.config.activeEngines.includes('youtube')
? result.title
: '') /* Todo: Implement transcript grabbing using Youtubei (source: https://www.npmjs.com/package/youtubei) */,
metadata: {
title: result.title,
url: result.url,
...(result.img_src && { img_src: result.img_src }),
},
}),
let res;
if (searchEngine === 'tavily') {
res = await searchTavily(question, {
search_depth: 'basic',
max_results: 15,
include_images: true,
});
} else {
// Default to SearxNG
res = await searchSearxng(question, {
language: 'en',
engines: this.config.activeEngines,
});
}
let documents: Document[] = [];
documents = documents.concat(
res.results.map(
(result) =>
new Document({
pageContent:
result.content ||
(this.config.activeEngines.includes('youtube')
? result.title
: ''),
metadata: {
title: result.title,
url: result.url,
...(result.img_src ? { img_src: result.img_src } : {}),
},
}),
)
);
return { query: question, docs: documents };

2
src/lib/searxng.ts → src/lib/searchEngines/searxng.ts
View File

@ -1,5 +1,5 @@
import axios from 'axios';
import { getSearxngApiEndpoint } from './config';
import { getSearxngApiEndpoint } from '../config';
interface SearxngSearchOptions {
categories?: string[];

79
src/lib/searchEngines/tavily.ts Normal file
View File

@ -0,0 +1,79 @@
import axios from 'axios';
import { getTavilyApiKey } from '../config';
interface TavilySearchOptions {
topic?: 'general' | 'news';
search_depth?: 'basic' | 'advanced';
chunks_per_source?: number;
max_results?: number;
time_range?: 'day' | 'week' | 'month' | 'year' | 'd' | 'w' | 'm' | 'y';
days?: number;
include_answer?: boolean | 'basic' | 'advanced';
include_raw_content?: boolean;
include_images?: boolean;
include_image_descriptions?: boolean;
include_domains?: string[];
exclude_domains?: string[];
}
interface TavilySearchResult {
title: string;
url: string;
content: string;
score: number;
raw_content?: string;
}
interface TavilySearchResponse {
query: string;
answer?: string;
images?: Array<{
url: string;
description?: string;
}>;
results: TavilySearchResult[];
response_time: string;
}
export const searchTavily = async (
query: string,
opts?: TavilySearchOptions,
) => {
const tavilyApiKey = getTavilyApiKey();
if (!tavilyApiKey) {
throw new Error('Tavily API key is not configured');
}
const url = 'https://api.tavily.com/search';
const response = await axios.post<TavilySearchResponse>(
url,
{
query,
...opts,
},
{
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${tavilyApiKey}`,
},
}
);
const results = response.data.results;
// Convert Tavily results to match the format expected by the rest of the application
const formattedResults = results.map(result => ({
title: result.title,
url: result.url,
content: result.content,
img_src: undefined, // Tavily doesn't provide image URLs in the standard response
}));
return {
results: formattedResults,
suggestions: [], // Tavily doesn't provide suggestions, so return empty array
answer: response.data.answer, // Include the AI-generated answer if available
};
};