Compare commits

..

3 Commits

Author SHA1 Message Date
ItzCrazyKns
59ab10110a Merge branch 'master' of https://github.com/HadiCherkaoui/Perplexica into pr/658 2025-03-02 15:35:14 +05:30
ItzCrazyKns
10f9cd2f79 feat(app): lint & beautify 2025-03-02 15:35:08 +05:30
Hadi Cherkaoui
82e1dd73b0 add config to ui 2025-03-02 11:00:56 +01:00
14 changed files with 442 additions and 107 deletions

View File

@@ -2,7 +2,6 @@
[![Discord](https://dcbadge.vercel.app/api/server/26aArMy8tT?style=flat&compact=true)](https://discord.gg/26aArMy8tT)
![preview](.assets/perplexica-screenshot.png?)
## Table of Contents <!-- omit in toc -->

View File

@@ -4,7 +4,7 @@ services:
volumes:
- ./searxng:/etc/searxng:rw
ports:
- "4000:8080"
- '4000:8080'
networks:
- perplexica-network
restart: unless-stopped
@@ -19,7 +19,7 @@ services:
depends_on:
- searxng
ports:
- "3001:3001"
- '3001:3001'
volumes:
- backend-dbstore:/home/perplexica/data
- uploads:/home/perplexica/uploads
@@ -41,7 +41,7 @@ services:
depends_on:
- perplexica-backend
ports:
- "3000:3000"
- '3000:3000'
networks:
- perplexica-network
restart: unless-stopped

View File

@@ -48,16 +48,18 @@ async function performImageSearch(query: string) {
switch (searchEngine) {
case 'google': {
const googleResult = await searchGooglePSE(query);
images = googleResult.results.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.displayLink
};
}
}).filter(Boolean);
images = googleResult.results
.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.displayLink,
};
}
})
.filter(Boolean);
break;
}
@@ -80,46 +82,52 @@ async function performImageSearch(query: string) {
case 'brave': {
const braveResult = await searchBraveAPI(query);
images = braveResult.results.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.url
};
}
}).filter(Boolean);
images = braveResult.results
.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.url,
};
}
})
.filter(Boolean);
break;
}
case 'yacy': {
const yacyResult = await searchYaCy(query);
images = yacyResult.results.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.url
images = yacyResult.results
.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.url,
};
}
}
}).filter(Boolean);
})
.filter(Boolean);
break;
}
case 'bing': {
const bingResult = await searchBingAPI(query);
images = bingResult.results.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.url
images = bingResult.results
.map((result) => {
if (result.img_src && result.url && result.title) {
return {
img_src: result.img_src,
url: result.url,
title: result.title,
source: result.url,
};
}
}
}).filter(Boolean);
})
.filter(Boolean);
break;
}

View File

@@ -50,14 +50,17 @@ async function performVideoSearch(query: string) {
switch (searchEngine) {
case 'google': {
const googleResult = await searchGooglePSE(youtubeQuery);
googleResult.results.forEach((result) => { // Use .results instead of .originalres
googleResult.results.forEach((result) => {
// Use .results instead of .originalres
if (result.img_src && result.url && result.title) {
const videoId = new URL(result.url).searchParams.get('v');
videos.push({
img_src: result.img_src,
url: result.url,
title: result.title,
iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
iframe_src: videoId
? `https://www.youtube.com/embed/${videoId}`
: null,
});
}
});
@@ -95,7 +98,9 @@ async function performVideoSearch(query: string) {
img_src: result.img_src,
url: result.url,
title: result.title,
iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
iframe_src: videoId
? `https://www.youtube.com/embed/${videoId}`
: null,
});
}
});
@@ -117,7 +122,9 @@ async function performVideoSearch(query: string) {
img_src: result.img_src,
url: result.url,
title: result.title,
iframe_src: videoId ? `https://www.youtube.com/embed/${videoId}` : null
iframe_src: videoId
? `https://www.youtube.com/embed/${videoId}`
: null,
});
}
});
@@ -159,4 +166,4 @@ const handleVideoSearch = (
return VideoSearchChain.invoke(input);
};
export default handleVideoSearch;
export default handleVideoSearch;

View File

@@ -15,7 +15,7 @@ interface Config {
IMAGE: string;
VIDEO: string;
NEWS: string;
}
};
MODELS: {
OPENAI: {
API_KEY: string;
@@ -103,7 +103,8 @@ export const getBraveApiKey = () => loadConfig().SEARCH_ENGINES.BRAVE.API_KEY;
export const getBingSubscriptionKey = () =>
loadConfig().SEARCH_ENGINES.BING.SUBSCRIPTION_KEY;
export const getYacyJsonEndpoint = () => loadConfig().SEARCH_ENGINES.YACY.ENDPOINT;
export const getYacyJsonEndpoint = () =>
loadConfig().SEARCH_ENGINES.YACY.ENDPOINT;
export const getSearxngApiEndpoint = () =>
process.env.SEARXNG_API_URL || loadConfig().SEARCH_ENGINES.SEARXNG.ENDPOINT;

View File

@@ -60,8 +60,8 @@ export const searchBingAPI = async (query: string) => {
const res = await axios.get(url.toString(), {
headers: {
'Ocp-Apim-Subscription-Key': bingApiKey,
'Accept': 'application/json'
}
Accept: 'application/json',
},
});
if (res.data.error) {
@@ -79,17 +79,20 @@ export const searchBingAPI = async (query: string) => {
title: item.name,
url: item.url,
content: item.snippet,
img_src: item.primaryImageOfPage?.thumbnailUrl
|| imageResults.find((img: any) => img.hostPageUrl === item.url)?.thumbnailUrl
|| videoResults.find((vid: any) => vid.hostPageUrl === item.url)?.thumbnailUrl,
img_src:
item.primaryImageOfPage?.thumbnailUrl ||
imageResults.find((img: any) => img.hostPageUrl === item.url)
?.thumbnailUrl ||
videoResults.find((vid: any) => vid.hostPageUrl === item.url)
?.thumbnailUrl,
...(item.video && {
videoData: {
duration: item.video.duration,
embedUrl: item.video.embedHtml?.match(/src="(.*?)"/)?.[1]
embedUrl: item.video.embedHtml?.match(/src="(.*?)"/)?.[1],
},
publisher: item.publisher,
datePublished: item.datePublished
})
publisher: item.publisher,
datePublished: item.datePublished,
}),
}));
return { results, originalres };
@@ -99,4 +102,4 @@ export const searchBingAPI = async (query: string) => {
: error.message || 'Unknown error';
throw new Error(`Bing API Error: ${errorMessage}`);
}
};
};

View File

@@ -33,7 +33,7 @@ interface BraveSearchResult {
export const searchBraveAPI = async (
query: string,
numResults: number = 20
numResults: number = 20,
): Promise<{ results: BraveSearchResult[]; originalres: any }> => {
try {
const braveApiKey = await getBraveApiKey();
@@ -45,8 +45,8 @@ export const searchBraveAPI = async (
const res = await axios.get(url.toString(), {
headers: {
'X-Subscription-Token': braveApiKey,
'Accept': 'application/json'
}
Accept: 'application/json',
},
});
if (res.data.error) {
@@ -64,26 +64,32 @@ export const searchBraveAPI = async (
age: item.age,
family_friendly: item.family_friendly,
language: item.language,
video: item.video ? {
embedUrl: item.video.embed_url,
duration: item.video.duration
} : undefined,
rating: item.rating ? {
value: item.rating.value,
scale: item.rating.scale_max
} : undefined,
video: item.video
? {
embedUrl: item.video.embed_url,
duration: item.video.duration,
}
: undefined,
rating: item.rating
? {
value: item.rating.value,
scale: item.rating.scale_max,
}
: undefined,
products: item.deep_results?.product_cluster?.map((p: any) => ({
name: p.name,
price: p.price
price: p.price,
})),
recipe: item.recipe ? {
ingredients: item.recipe.ingredients,
cookTime: item.recipe.cook_time
} : undefined,
recipe: item.recipe
? {
ingredients: item.recipe.ingredients,
cookTime: item.recipe.cook_time,
}
: undefined,
meta: {
fetched: item.meta?.fetched,
lastCrawled: item.meta?.last_crawled
}
lastCrawled: item.meta?.last_crawled,
},
}));
return { results, originalres };

View File

@@ -21,7 +21,7 @@ interface GooglePSESearchResult {
}>;
metatags?: Array<{
[key: string]: string;
'author'?: string;
author?: string;
}>;
cse_image?: Array<{
src: string;
@@ -43,7 +43,7 @@ export const searchGooglePSE = async (query: string) => {
try {
const [googleApiKey, googleCseID] = await Promise.all([
getGoogleApiKey(),
getGoogleCseId()
getGoogleCseId(),
]);
const url = new URL(`https://www.googleapis.com/customsearch/v1`);
@@ -63,15 +63,16 @@ export const searchGooglePSE = async (query: string) => {
title: item.title,
url: item.link,
content: item.snippet,
img_src: item.pagemap?.cse_image?.[0]?.src
|| item.pagemap?.cse_thumbnail?.[0]?.src
|| item.image?.thumbnailLink,
img_src:
item.pagemap?.cse_image?.[0]?.src ||
item.pagemap?.cse_thumbnail?.[0]?.src ||
item.image?.thumbnailLink,
...(item.pagemap?.videoobject?.[0] && {
videoData: {
duration: item.pagemap.videoobject[0].duration,
embedUrl: item.pagemap.videoobject[0].embedurl
}
})
embedUrl: item.pagemap.videoobject[0].embedurl,
},
}),
}));
return { results, originalres };

View File

@@ -48,11 +48,7 @@ interface YaCySearchResult {
}[];
}
export const searchYaCy = async (
query: string,
numResults: number = 20
) => {
export const searchYaCy = async (query: string, numResults: number = 20) => {
try {
const yacyBaseUrl = getYacyJsonEndpoint();
@@ -64,7 +60,7 @@ export const searchYaCy = async (
const originalres = res.data as YaCySearchResult;
const results = originalres.channels[0].items.map(item => ({
const results = originalres.channels[0].items.map((item) => ({
title: item.title,
url: item.link,
content: item.description,

View File

@@ -13,6 +13,16 @@ import {
getCustomOpenaiApiUrl,
getCustomOpenaiApiKey,
getCustomOpenaiModelName,
getSearchEngineBackend,
getImageSearchEngineBackend,
getVideoSearchEngineBackend,
getNewsSearchEngineBackend,
getSearxngApiEndpoint,
getGoogleApiKey,
getGoogleCseId,
getBingSubscriptionKey,
getBraveApiKey,
getYacyJsonEndpoint,
} from '../config';
import logger from '../utils/logger';
@@ -60,6 +70,21 @@ router.get('/', async (_, res) => {
config['customOpenaiApiUrl'] = getCustomOpenaiApiUrl();
config['customOpenaiApiKey'] = getCustomOpenaiApiKey();
config['customOpenaiModelName'] = getCustomOpenaiModelName();
// Add search engine configuration
config['searchEngineBackends'] = {
search: getSearchEngineBackend(),
image: getImageSearchEngineBackend(),
video: getVideoSearchEngineBackend(),
news: getNewsSearchEngineBackend(),
};
config['searxngEndpoint'] = getSearxngApiEndpoint();
config['googleApiKey'] = getGoogleApiKey();
config['googleCseId'] = getGoogleCseId();
config['bingSubscriptionKey'] = getBingSubscriptionKey();
config['braveApiKey'] = getBraveApiKey();
config['yacyEndpoint'] = getYacyJsonEndpoint();
res.status(200).json(config);
} catch (err: any) {
@@ -94,6 +119,30 @@ router.post('/', async (req, res) => {
MODEL_NAME: config.customOpenaiModelName,
},
},
SEARCH_ENGINE_BACKENDS: config.searchEngineBackends ? {
SEARCH: config.searchEngineBackends.search,
IMAGE: config.searchEngineBackends.image,
VIDEO: config.searchEngineBackends.video,
NEWS: config.searchEngineBackends.news,
} : undefined,
SEARCH_ENGINES: {
GOOGLE: {
API_KEY: config.googleApiKey,
CSE_ID: config.googleCseId,
},
SEARXNG: {
ENDPOINT: config.searxngEndpoint,
},
BING: {
SUBSCRIPTION_KEY: config.bingSubscriptionKey,
},
BRAVE: {
API_KEY: config.braveApiKey,
},
YACY: {
ENDPOINT: config.yacyEndpoint,
},
},
};
updateConfig(updatedConfig);

View File

@@ -15,7 +15,7 @@ async function performSearch(query: string, site: string) {
case 'google': {
const googleResult = await searchGooglePSE(query);
return googleResult.originalres.map(item => {
return googleResult.originalres.map((item) => {
const imageSources = [
item.pagemap?.cse_image?.[0]?.src,
item.pagemap?.cse_thumbnail?.[0]?.src,
@@ -29,10 +29,11 @@ async function performSearch(query: string, site: string) {
url: item.link,
content: item.snippet,
thumbnail: imageSources[0], // First available image
img_src: imageSources[0], // Same as thumbnail for consistency
img_src: imageSources[0], // Same as thumbnail for consistency
iframe_src: null,
author: item.pagemap?.metatags?.[0]?.['og:site_name'] || site,
publishedDate: item.pagemap?.metatags?.[0]?.['article:published_time']
publishedDate:
item.pagemap?.metatags?.[0]?.['article:published_time'],
};
});
}
@@ -47,7 +48,7 @@ async function performSearch(query: string, site: string) {
case 'brave': {
const braveResult = await searchBraveAPI(query);
return braveResult.results.map(item => ({
return braveResult.results.map((item) => ({
title: item.title,
url: item.url,
content: item.content,
@@ -55,7 +56,7 @@ async function performSearch(query: string, site: string) {
img_src: item.img_src,
iframe_src: null,
author: item.meta?.fetched || site,
publishedDate: item.meta?.lastCrawled
publishedDate: item.meta?.lastCrawled,
}));
}
@@ -69,13 +70,13 @@ async function performSearch(query: string, site: string) {
img_src: item.img_src,
iframe_src: null,
author: item?.host || site,
publishedDate: item?.pubDate
}))
publishedDate: item?.pubDate,
}));
}
case 'bing': {
const bingResult = await searchBingAPI(query);
return bingResult.results.map(item => ({
return bingResult.results.map((item) => ({
title: item.title,
url: item.url,
content: item.content,
@@ -83,8 +84,8 @@ async function performSearch(query: string, site: string) {
img_src: item.img_src,
iframe_src: null,
author: item?.publisher || site,
publishedDate: item?.datePublished
}))
publishedDate: item?.datePublished,
}));
}
default:
@@ -92,7 +93,6 @@ async function performSearch(query: string, site: string) {
}
}
router.get('/', async (req, res) => {
try {
const queries = [
@@ -114,12 +114,12 @@ router.get('/', async (req, res) => {
logger.error(`Error searching ${site}: ${error.message}`);
return [];
}
})
}),
)
)
.flat()
.sort(() => Math.random() - 0.5)
.filter(item => item.title && item.url && item.content);
.filter((item) => item.title && item.url && item.content);
return res.json({ blogs: data });
} catch (err: any) {

View File

@@ -208,7 +208,6 @@ class MetaSearchAgent implements MetaSearchAgentType {
return { query: question, docs: docs };
} else {
const searchEngine = getSearchEngineBackend();
let res;
@@ -236,7 +235,9 @@ class MetaSearchAgent implements MetaSearchAgentType {
}
if (!res?.results) {
throw new Error(`No results found for search engine: ${searchEngine}`);
throw new Error(
`No results found for search engine: ${searchEngine}`,
);
}
const documents = res.results.map(

View File

@@ -23,6 +23,18 @@ interface SettingsType {
customOpenaiApiKey: string;
customOpenaiApiUrl: string;
customOpenaiModelName: string;
searchEngineBackends: {
search: string;
image: string;
video: string;
news: string;
};
searxngEndpoint: string;
googleApiKey: string;
googleCseId: string;
bingSubscriptionKey: string;
braveApiKey: string;
yacyEndpoint: string;
}
interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
@@ -112,6 +124,12 @@ const Page = () => {
const [automaticImageSearch, setAutomaticImageSearch] = useState(false);
const [automaticVideoSearch, setAutomaticVideoSearch] = useState(false);
const [savingStates, setSavingStates] = useState<Record<string, boolean>>({});
const [searchEngineBackends, setSearchEngineBackends] = useState({
search: '',
image: '',
video: '',
news: '',
});
useEffect(() => {
const fetchConfig = async () => {
@@ -125,6 +143,16 @@ const Page = () => {
const data = (await res.json()) as SettingsType;
setConfig(data);
// Set search engine backends if they exist in the response
if (data.searchEngineBackends) {
setSearchEngineBackends({
search: data.searchEngineBackends.search || '',
image: data.searchEngineBackends.image || '',
video: data.searchEngineBackends.video || '',
news: data.searchEngineBackends.news || '',
});
}
const chatModelProvidersKeys = Object.keys(data.chatModelProviders || {});
const embeddingModelProvidersKeys = Object.keys(
data.embeddingModelProviders || {},
@@ -331,6 +359,8 @@ const Page = () => {
localStorage.setItem('embeddingModelProvider', value);
} else if (key === 'embeddingModel') {
localStorage.setItem('embeddingModel', value);
} else if (key === 'searchEngineBackends') {
localStorage.setItem('searchEngineBackends', value);
}
} catch (err) {
console.error('Failed to save:', err);
@@ -793,6 +823,234 @@ const Page = () => {
</div>
</div>
</SettingsSection>
<SettingsSection title="Search Engine Settings">
<div className="flex flex-col space-y-4">
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Default Search Engine
</p>
<Select
value={searchEngineBackends.search}
onChange={(e) => {
const value = e.target.value;
setSearchEngineBackends((prev) => ({
...prev,
search: value,
}));
saveConfig('searchEngineBackends', {
...searchEngineBackends,
search: value,
});
}}
options={[
{ value: 'searxng', label: 'SearXNG' },
{ value: 'google', label: 'Google' },
{ value: 'bing', label: 'Bing' },
{ value: 'brave', label: 'Brave' },
{ value: 'yacy', label: 'YaCy' },
]}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Image Search Engine
</p>
<Select
value={searchEngineBackends.image}
onChange={(e) => {
const value = e.target.value;
setSearchEngineBackends((prev) => ({
...prev,
image: value,
}));
saveConfig('searchEngineBackends', {
...searchEngineBackends,
image: value,
});
}}
options={[
{ value: '', label: 'Use Default Search Engine' },
{ value: 'searxng', label: 'SearXNG' },
{ value: 'google', label: 'Google' },
{ value: 'bing', label: 'Bing' },
{ value: 'brave', label: 'Brave' },
]}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Video Search Engine
</p>
<Select
value={searchEngineBackends.video}
onChange={(e) => {
const value = e.target.value;
setSearchEngineBackends((prev) => ({
...prev,
video: value,
}));
saveConfig('searchEngineBackends', {
...searchEngineBackends,
video: value,
});
}}
options={[
{ value: '', label: 'Use Default Search Engine' },
{ value: 'searxng', label: 'SearXNG' },
{ value: 'google', label: 'Google' },
{ value: 'bing', label: 'Bing' },
{ value: 'brave', label: 'Brave' },
]}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
News Search Engine
</p>
<Select
value={searchEngineBackends.news}
onChange={(e) => {
const value = e.target.value;
setSearchEngineBackends((prev) => ({
...prev,
news: value,
}));
saveConfig('searchEngineBackends', {
...searchEngineBackends,
news: value,
});
}}
options={[
{ value: '', label: 'Use Default Search Engine' },
{ value: 'searxng', label: 'SearXNG' },
{ value: 'google', label: 'Google' },
{ value: 'bing', label: 'Bing' },
{ value: 'brave', label: 'Brave' },
]}
/>
</div>
<div className="pt-4 border-t border-light-200 dark:border-dark-200">
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
SearXNG Endpoint
</p>
<Input
type="text"
placeholder="SearXNG API Endpoint"
value={config.searxngEndpoint || ''}
isSaving={savingStates['searxngEndpoint']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
searxngEndpoint: e.target.value,
}));
}}
onSave={(value) => saveConfig('searxngEndpoint', value)}
/>
</div>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Google API Key
</p>
<Input
type="text"
placeholder="Google API Key"
value={config.googleApiKey || ''}
isSaving={savingStates['googleApiKey']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
googleApiKey: e.target.value,
}));
}}
onSave={(value) => saveConfig('googleApiKey', value)}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Google CSE ID
</p>
<Input
type="text"
placeholder="Google Custom Search Engine ID"
value={config.googleCseId || ''}
isSaving={savingStates['googleCseId']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
googleCseId: e.target.value,
}));
}}
onSave={(value) => saveConfig('googleCseId', value)}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Bing Subscription Key
</p>
<Input
type="text"
placeholder="Bing Subscription Key"
value={config.bingSubscriptionKey || ''}
isSaving={savingStates['bingSubscriptionKey']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
bingSubscriptionKey: e.target.value,
}));
}}
onSave={(value) => saveConfig('bingSubscriptionKey', value)}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
Brave API Key
</p>
<Input
type="text"
placeholder="Brave API Key"
value={config.braveApiKey || ''}
isSaving={savingStates['braveApiKey']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
braveApiKey: e.target.value,
}));
}}
onSave={(value) => saveConfig('braveApiKey', value)}
/>
</div>
<div className="flex flex-col space-y-1">
<p className="text-black/70 dark:text-white/70 text-sm">
YaCy Endpoint
</p>
<Input
type="text"
placeholder="YaCy API Endpoint"
value={config.yacyEndpoint || ''}
isSaving={savingStates['yacyEndpoint']}
onChange={(e) => {
setConfig((prev) => ({
...prev!,
yacyEndpoint: e.target.value,
}));
}}
onSave={(value) => saveConfig('yacyEndpoint', value)}
/>
</div>
</div>
</SettingsSection>
</div>
)
)}

View File

@@ -68,7 +68,13 @@ const MessageBox = ({
return (
<div>
{message.role === 'user' && (
<div className={cn('w-full', messageIndex === 0 ? 'pt-16' : 'pt-8', 'break-words')}>
<div
className={cn(
'w-full',
messageIndex === 0 ? 'pt-16' : 'pt-8',
'break-words',
)}
>
<h2 className="text-black dark:text-white font-medium text-3xl lg:w-9/12">
{message.content}
</h2>