feat(discover): prevent duplicate articles

src/app/api/discover/route.ts

@@ -38,28 +38,30 @@ export const GET = async (req: Request) => {
     let data = [];
 
     if (mode === 'normal') {
+      const seenUrls = new Set();
+
       data = (
-        await Promise.all([
+        await Promise.all(
-          ...new Array(selectedTopic.links.length * selectedTopic.query.length)
+          selectedTopic.links.flatMap((link) =>
-            .fill(0)
+            selectedTopic.query.map(async (query) => {
-            .map(async (_, i) => {
               return (
-                await searchSearxng(
+                await searchSearxng(`site:${link} ${query}`, {
-                  `site:${selectedTopic.links[i % selectedTopic.links.length]} ${
+                  engines: ['bing news'],
-                    selectedTopic.query[i % selectedTopic.query.length]
+                  pageno: 1,
-                  }`,
+                  language: 'en',
-                  {
+                })
-                    engines: ['bing news'],
-                    pageno: 1,
-                    language: 'en',
-                  },
-                )
               ).results;
             }),
-        ])
+          ),
+        )
       )
-        .map((result) => result)
         .flat()
+        .filter((item) => {
+          const url = item.url?.toLowerCase().trim();
+          if (seenUrls.has(url)) return false;
+          seenUrls.add(url);
+          return true;
+        })
         .sort(() => Math.random() - 0.5);
     } else {
       data = (