remi/Perplexica
1
0
Fork 0
mirror of https://github.com/ItzCrazyKns/Perplexica.git synced 2025-08-03 14:28:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(discover): prevent duplicate articles

Browse Source
This commit is contained in:
ItzCrazyKns
2025-08-01 20:41:07 +05:30
parent 88be3a045b
commit 37cd6d3ab5
1 changed files with 18 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/app/api/discover/route.ts
View File

@ -38,28 +38,30 @@ export const GET = async (req: Request) => {
let data = [];
if (mode === 'normal') {
const seenUrls = new Set();
data = (
await Promise.all([
...new Array(selectedTopic.links.length * selectedTopic.query.length)
.fill(0)
.map(async (_, i) => {
await Promise.all(
selectedTopic.links.flatMap((link) =>
selectedTopic.query.map(async (query) => {
return (
await searchSearxng(
`site:${selectedTopic.links[i % selectedTopic.links.length]} ${
selectedTopic.query[i % selectedTopic.query.length]
}`,
{
engines: ['bing news'],
pageno: 1,
language: 'en',
},
)
await searchSearxng(`site:${link} ${query}`, {
engines: ['bing news'],
pageno: 1,
language: 'en',
})
).results;
}),
])
),
)
)
.map((result) => result)
.flat()
.filter((item) => {
const url = item.url?.toLowerCase().trim();
if (seenUrls.has(url)) return false;
seenUrls.add(url);
return true;
})
.sort(() => Math.random() - 0.5);
} else {
data = (