From 37cd6d3ab57cbe7c4fbb62750fd764cda953d9c7 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <95534749+ItzCrazyKns@users.noreply.github.com>
Date: Fri, 1 Aug 2025 20:41:07 +0530
Subject: [PATCH] feat(discover): prevent duplicate articles

---
 src/app/api/discover/route.ts | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/app/api/discover/route.ts b/src/app/api/discover/route.ts
index c02d95c..415aee8 100644
--- a/src/app/api/discover/route.ts
+++ b/src/app/api/discover/route.ts
@@ -38,28 +38,30 @@ export const GET = async (req: Request) => {
     let data = [];
 
     if (mode === 'normal') {
+      const seenUrls = new Set();
+
       data = (
-        await Promise.all([
-          ...new Array(selectedTopic.links.length * selectedTopic.query.length)
-            .fill(0)
-            .map(async (_, i) => {
+        await Promise.all(
+          selectedTopic.links.flatMap((link) =>
+            selectedTopic.query.map(async (query) => {
               return (
-                await searchSearxng(
-                  `site:${selectedTopic.links[i % selectedTopic.links.length]} ${
-                    selectedTopic.query[i % selectedTopic.query.length]
-                  }`,
-                  {
-                    engines: ['bing news'],
-                    pageno: 1,
-                    language: 'en',
-                  },
-                )
+                await searchSearxng(`site:${link} ${query}`, {
+                  engines: ['bing news'],
+                  pageno: 1,
+                  language: 'en',
+                })
               ).results;
             }),
-        ])
+          ),
+        )
       )
-        .map((result) => result)
         .flat()
+        .filter((item) => {
+          const url = item.url?.toLowerCase().trim();
+          if (seenUrls.has(url)) return false;
+          seenUrls.add(url);
+          return true;
+        })
         .sort(() => Math.random() - 0.5);
     } else {
       data = (