Merge 47c7bb688f into df4350f966

fix(ui): improve dark mode text color for attachment buttons

sample.config.toml

@@ -23,4 +23,7 @@ API_URL = ""
 API_URL = "" # Ollama API URL - http://host.docker.internal:11434
 
 [API_ENDPOINTS]
-SEARXNG = "http://localhost:32768" # SearxNG API URL
+SEARXNG = "http://localhost:32768" # SearxNG API URL
+
+[API_KEYS]
+JINA = ""

src/config.ts

@@ -35,6 +35,9 @@ interface Config {
   API_ENDPOINTS: {
     SEARXNG: string;
   };
+  API_KEYS: {
+    JINA: string;
+  }
 }
 
 type RecursivePartial<T> = {
@@ -75,6 +78,9 @@ export const getCustomOpenaiApiUrl = () =>
 export const getCustomOpenaiModelName = () =>
   loadConfig().MODELS.CUSTOM_OPENAI.MODEL_NAME;
 
+export const getJinaApiKey = () =>
+  loadConfig().API_KEYS.JINA;
+
 const mergeConfigs = (current: any, update: any): any => {
   if (update === null || update === undefined) {
     return current;

src/utils/documents.ts

@@ -1,22 +1,30 @@
 import axios from 'axios';
 import { htmlToText } from 'html-to-text';
-import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { RecursiveCharacterTextSplitter, MarkdownTextSplitter } from 'langchain/text_splitter';
 import { Document } from '@langchain/core/documents';
 import pdfParse from 'pdf-parse';
 import logger from './logger';
+import { getJinaApiKey } from '../config';
 
 export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
-  const splitter = new RecursiveCharacterTextSplitter();
 
+  links = links.map(link => link.startsWith('http://') || link.startsWith('https://')
+    ? link
+    : `https://${link}`)
+
+  if (getJinaApiKey()) {
+    return await getDocumentsFromJinaReader({ links });
+  }
+
+  return await getDocumentsFromLocal({ links });
+};
+
+const getDocumentsFromLocal = async ({links}: {links: string[]}) => {
+  const splitter = new RecursiveCharacterTextSplitter();
   let docs: Document[] = [];
 
   await Promise.all(
     links.map(async (link) => {
-      link =
-        link.startsWith('http://') || link.startsWith('https://')
-          ? link
-          : `https://${link}`;
-
       try {
         const res = await axios.get(link, {
           responseType: 'arraybuffer',
@@ -94,6 +102,62 @@ export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
       }
     }),
   );
-
+  return docs;
+};
+
+const getDocumentsFromJinaReader = async ({links}: { links: string[] }) => {
+  const splitter = new MarkdownTextSplitter();
+  let docs: Document[] = [];
+
+  await Promise.all(
+    links.map(async link => {
+      try {
+        const res = await axios.get(`https://r.jina.ai/${link}`, {
+          headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${getJinaApiKey()}`,
+          }
+        });
+
+        if(res.data.code === 200) {
+          const data = res.data.data
+          const splittedText = await splitter.splitText(data.content);
+          const linkDocs = splittedText.map((text) => {
+            return new Document({
+              pageContent: text,
+              metadata: {
+                title: data.title,
+                url: link,
+              },
+            });
+          });
+
+          docs.push(...linkDocs);
+          return;
+        } else {
+          docs.push(
+            new Document({
+              pageContent: `Failed to retrieve content from the link in the Jina reader API, code: ${res.data.code}`,
+              metadata: {
+                title: 'Failed to retrieve content',
+                url: link,
+              },
+            }),
+          );
+        }
+      } catch (err) {
+        docs.push(
+          new Document({
+            pageContent: `Failed to retrieve content from the link: ${err.message}`,
+            metadata: {
+              title: 'Failed to retrieve content',
+              url: link,
+            },
+          }),
+        );
+      }
+    })
+  );
   return docs;
 };

ui/components/MessageBox.tsx

@@ -68,7 +68,7 @@ const MessageBox = ({
   return (
     <div>
       {message.role === 'user' && (
-        <div className={cn('w-full', messageIndex === 0 ? 'pt-16' : 'pt-8')}>
+        <div className={cn('w-full', messageIndex === 0 ? 'pt-16' : 'pt-8', 'break-words')}>
           <h2 className="text-black dark:text-white font-medium text-3xl lg:w-9/12">
             {message.content}
           </h2>

ui/components/MessageInputActions/Attach.tsx

@@ -110,7 +110,7 @@ const Attach = ({
                 <button
                   type="button"
                   onClick={() => fileInputRef.current.click()}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <input
                     type="file"
@@ -128,7 +128,7 @@ const Attach = ({
                     setFiles([]);
                     setFileIds([]);
                   }}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <Trash size={14} />
                   <p className="text-xs">Clear</p>
@@ -145,7 +145,7 @@ const Attach = ({
                   <div className="bg-dark-100 flex items-center justify-center w-10 h-10 rounded-md">
                     <File size={16} className="text-white/70" />
                   </div>
-                  <p className="text-white/70 text-sm">
+                  <p className="text-black/70 dark:text-white/70 text-sm">
                     {file.fileName.length > 25
                       ? file.fileName.replace(/\.\w+$/, '').substring(0, 25) +
                         '...' +

ui/components/MessageInputActions/AttachSmall.tsx

@@ -82,7 +82,7 @@ const AttachSmall = ({
                 <button
                   type="button"
                   onClick={() => fileInputRef.current.click()}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <input
                     type="file"
@@ -100,7 +100,7 @@ const AttachSmall = ({
                     setFiles([]);
                     setFileIds([]);
                   }}
-                  className="flex flex-row items-center space-x-1 text-white/70 hover:text-white transition duration-200"
+                  className="flex flex-row items-center space-x-1 text-black/70 dark:text-white/70 hover:text-black hover:dark:text-white transition duration-200"
                 >
                   <Trash size={14} />
                   <p className="text-xs">Clear</p>
@@ -117,7 +117,7 @@ const AttachSmall = ({
                   <div className="bg-dark-100 flex items-center justify-center w-10 h-10 rounded-md">
                     <File size={16} className="text-white/70" />
                   </div>
-                  <p className="text-white/70 text-sm">
+                  <p className="text-black/70 dark:text-white/70 text-sm">
                     {file.fileName.length > 25
                       ? file.fileName.replace(/\.\w+$/, '').substring(0, 25) +
                         '...' +