feat(discover): prevent duplicate articles

src/app/api/discover/route.ts

@@ -1,55 +1,72 @@
 import { searchSearxng } from '@/lib/searxng';
 
-const articleWebsites = [
+const websitesForTopic = {
-  'yahoo.com',
+  tech: {
-  'www.exchangewire.com',
+    query: ['technology news', 'latest tech', 'AI', 'science and innovation'],
-  'businessinsider.com',
+    links: ['techcrunch.com', 'wired.com', 'theverge.com'],
-  /* 'wired.com',
+  },
-  'mashable.com',
+  finance: {
-  'theverge.com',
+    query: ['finance news', 'economy', 'stock market', 'investing'],
-  'gizmodo.com',
+    links: ['bloomberg.com', 'cnbc.com', 'marketwatch.com'],
-  'cnet.com',
+  },
-  'venturebeat.com', */
+  art: {
-];
+    query: ['art news', 'culture', 'modern art', 'cultural events'],
+    links: ['artnews.com', 'hyperallergic.com', 'theartnewspaper.com'],
+  },
+  sports: {
+    query: ['sports news', 'latest sports', 'cricket football tennis'],
+    links: ['espn.com', 'bbc.com/sport', 'skysports.com'],
+  },
+  entertainment: {
+    query: ['entertainment news', 'movies', 'TV shows', 'celebrities'],
+    links: ['hollywoodreporter.com', 'variety.com', 'deadline.com'],
+  },
+};
 
-const topics = ['AI', 'tech']; /* TODO: Add UI to customize this */
+type Topic = keyof typeof websitesForTopic;
 
 export const GET = async (req: Request) => {
   try {
     const params = new URL(req.url).searchParams;
+
     const mode: 'normal' | 'preview' =
       (params.get('mode') as 'normal' | 'preview') || 'normal';
+    const topic: Topic = (params.get('topic') as Topic) || 'tech';
+
+    const selectedTopic = websitesForTopic[topic];
 
     let data = [];
 
     if (mode === 'normal') {
+      const seenUrls = new Set();
+
       data = (
-        await Promise.all([
+        await Promise.all(
-          ...new Array(articleWebsites.length * topics.length)
+          selectedTopic.links.flatMap((link) =>
-            .fill(0)
+            selectedTopic.query.map(async (query) => {
-            .map(async (_, i) => {
               return (
-                await searchSearxng(
+                await searchSearxng(`site:${link} ${query}`, {
-                  `site:${articleWebsites[i % articleWebsites.length]} ${
-                    topics[i % topics.length]
-                  }`,
-                  {
                   engines: ['bing news'],
                   pageno: 1,
                   language: 'en',
-                  },
+                })
-                )
               ).results;
             }),
-        ])
+          ),
+        )
       )
-        .map((result) => result)
         .flat()
+        .filter((item) => {
+          const url = item.url?.toLowerCase().trim();
+          if (seenUrls.has(url)) return false;
+          seenUrls.add(url);
+          return true;
+        })
         .sort(() => Math.random() - 0.5);
     } else {
       data = (
         await searchSearxng(
-          `site:${articleWebsites[Math.floor(Math.random() * articleWebsites.length)]} ${topics[Math.floor(Math.random() * topics.length)]}`,
+          `site:${selectedTopic.links[Math.floor(Math.random() * selectedTopic.links.length)]} ${selectedTopic.query[Math.floor(Math.random() * selectedTopic.query.length)]}`,
           {
             engines: ['bing news'],
             pageno: 1,

src/app/discover/page.tsx

@@ -4,6 +4,7 @@ import { Search } from 'lucide-react';
 import { useEffect, useState } from 'react';
 import Link from 'next/link';
 import { toast } from 'sonner';
+import { cn } from '@/lib/utils';
 
 interface Discover {
   title: string;
@@ -12,14 +13,38 @@ interface Discover {
   thumbnail: string;
 }
 
+const topics: { key: string; display: string }[] = [
+  {
+    display: 'Tech & Science',
+    key: 'tech',
+  },
+  {
+    display: 'Finance',
+    key: 'finance',
+  },
+  {
+    display: 'Art & Culture',
+    key: 'art',
+  },
+  {
+    display: 'Sports',
+    key: 'sports',
+  },
+  {
+    display: 'Entertainment',
+    key: 'entertainment',
+  },
+];
+
 const Page = () => {
   const [discover, setDiscover] = useState<Discover[] | null>(null);
   const [loading, setLoading] = useState(true);
+  const [activeTopic, setActiveTopic] = useState<string>(topics[0].key);
 
-  useEffect(() => {
+  const fetchArticles = async (topic: string) => {
-    const fetchData = async () => {
+    setLoading(true);
     try {
-        const res = await fetch(`/api/discover`, {
+      const res = await fetch(`/api/discover?topic=${topic}`, {
         method: 'GET',
         headers: {
           'Content-Type': 'application/json',
@@ -43,10 +68,39 @@ const Page = () => {
     }
   };
 
-    fetchData();
+  useEffect(() => {
-  }, []);
+    fetchArticles(activeTopic);
+  }, [activeTopic]);
 
-  return loading ? (
+  return (
+    <>
+      <div>
+        <div className="flex flex-col pt-4">
+          <div className="flex items-center">
+            <Search />
+            <h1 className="text-3xl font-medium p-2">Discover</h1>
+          </div>
+          <hr className="border-t border-[#2B2C2C] my-4 w-full" />
+        </div>
+
+        <div className="flex flex-row items-center space-x-2 overflow-x-auto">
+          {topics.map((t, i) => (
+            <div
+              key={i}
+              className={cn(
+                'border-[0.1px] rounded-full text-sm px-3 py-1 text-nowrap transition duration-200 cursor-pointer',
+                activeTopic === t.key
+                  ? 'text-cyan-300 bg-cyan-300/30 border-cyan-300/60'
+                  : 'border-white/30 text-white/70 hover:text-white hover:border-white/40 hover:bg-white/5',
+              )}
+              onClick={() => setActiveTopic(t.key)}
+            >
+              <span>{t.display}</span>
+            </div>
+          ))}
+        </div>
+
+        {loading ? (
           <div className="flex flex-row items-center justify-center min-h-screen">
             <svg
               aria-hidden="true"
@@ -66,17 +120,7 @@ const Page = () => {
             </svg>
           </div>
         ) : (
-    <>
+          <div className="grid lg:grid-cols-3 sm:grid-cols-2 grid-cols-1 gap-4 pb-28 pt-5 lg:pb-8 w-full justify-items-center lg:justify-items-start">
-      <div>
-        <div className="flex flex-col pt-4">
-          <div className="flex items-center">
-            <Search />
-            <h1 className="text-3xl font-medium p-2">Discover</h1>
-          </div>
-          <hr className="border-t border-[#2B2C2C] my-4 w-full" />
-        </div>
-
-        <div className="grid lg:grid-cols-3 sm:grid-cols-2 grid-cols-1 gap-4 pb-28 lg:pb-8 w-full justify-items-center lg:justify-items-start">
             {discover &&
               discover?.map((item, i) => (
                 <Link
@@ -105,6 +149,7 @@ const Page = () => {
                 </Link>
               ))}
           </div>
+        )}
       </div>
     </>
   );

src/lib/providers/gemini.ts

@@ -14,16 +14,12 @@ import { Embeddings } from '@langchain/core/embeddings';
 
 const geminiChatModels: Record<string, string>[] = [
   {
-    displayName: 'Gemini 2.5 Flash Preview 05-20',
+    displayName: 'Gemini 2.5 Flash',
-    key: 'gemini-2.5-flash-preview-05-20',
+    key: 'gemini-2.5-flash',
   },
   {
-    displayName: 'Gemini 2.5 Pro Preview',
+    displayName: 'Gemini 2.5 Pro',
-    key: 'gemini-2.5-pro-preview-05-06',
+    key: 'gemini-2.5-pro',
-  },
-  {
-    displayName: 'Gemini 2.5 Pro Experimental',
-    key: 'gemini-2.5-pro-preview-05-06',
   },
   {
     displayName: 'Gemini 2.0 Flash',