feat(app): remove backend

src/app/api/chat/route.ts

@@ -0,0 +1,360 @@
+import prompts from '@/lib/prompts';
+import MetaSearchAgent from '@/lib/search/metaSearchAgent';
+import crypto from 'crypto';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+import { EventEmitter } from 'stream';
+import {
+  chatModelProviders,
+  embeddingModelProviders,
+  getAvailableChatModelProviders,
+  getAvailableEmbeddingModelProviders,
+} from '@/lib/providers';
+import db from '@/lib/db';
+import { chats, messages as messagesSchema } from '@/lib/db/schema';
+import { and, eq, gt } from 'drizzle-orm';
+import { getFileDetails } from '@/lib/utils/files';
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { ChatOpenAI } from '@langchain/openai';
+import {
+  getCustomOpenaiApiKey,
+  getCustomOpenaiApiUrl,
+  getCustomOpenaiModelName,
+} from '@/lib/config';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+const searchHandlers: Record<string, MetaSearchAgent> = {
+  webSearch: new MetaSearchAgent({
+    activeEngines: [],
+    queryGeneratorPrompt: prompts.webSearchRetrieverPrompt,
+    responsePrompt: prompts.webSearchResponsePrompt,
+    rerank: true,
+    rerankThreshold: 0.3,
+    searchWeb: true,
+    summarizer: true,
+  }),
+  academicSearch: new MetaSearchAgent({
+    activeEngines: ['arxiv', 'google scholar', 'pubmed'],
+    queryGeneratorPrompt: prompts.academicSearchRetrieverPrompt,
+    responsePrompt: prompts.academicSearchResponsePrompt,
+    rerank: true,
+    rerankThreshold: 0,
+    searchWeb: true,
+    summarizer: false,
+  }),
+  writingAssistant: new MetaSearchAgent({
+    activeEngines: [],
+    queryGeneratorPrompt: '',
+    responsePrompt: prompts.writingAssistantPrompt,
+    rerank: true,
+    rerankThreshold: 0,
+    searchWeb: false,
+    summarizer: false,
+  }),
+  wolframAlphaSearch: new MetaSearchAgent({
+    activeEngines: ['wolframalpha'],
+    queryGeneratorPrompt: prompts.wolframAlphaSearchRetrieverPrompt,
+    responsePrompt: prompts.wolframAlphaSearchResponsePrompt,
+    rerank: false,
+    rerankThreshold: 0,
+    searchWeb: true,
+    summarizer: false,
+  }),
+  youtubeSearch: new MetaSearchAgent({
+    activeEngines: ['youtube'],
+    queryGeneratorPrompt: prompts.youtubeSearchRetrieverPrompt,
+    responsePrompt: prompts.youtubeSearchResponsePrompt,
+    rerank: true,
+    rerankThreshold: 0.3,
+    searchWeb: true,
+    summarizer: false,
+  }),
+  redditSearch: new MetaSearchAgent({
+    activeEngines: ['reddit'],
+    queryGeneratorPrompt: prompts.redditSearchRetrieverPrompt,
+    responsePrompt: prompts.redditSearchResponsePrompt,
+    rerank: true,
+    rerankThreshold: 0.3,
+    searchWeb: true,
+    summarizer: false,
+  }),
+};
+
+type Message = {
+  messageId: string;
+  chatId: string;
+  content: string;
+};
+
+type ChatModel = {
+  provider: string;
+  name: string;
+};
+
+type EmbeddingModel = {
+  provider: string;
+  name: string;
+};
+
+type Body = {
+  message: Message;
+  optimizationMode: 'speed' | 'balanced' | 'quality';
+  focusMode: string;
+  history: Array<[string, string]>;
+  files: Array<string>;
+  chatModel: ChatModel;
+  embeddingModel: EmbeddingModel;
+};
+
+const handleEmitterEvents = async (
+  stream: EventEmitter,
+  writer: WritableStreamDefaultWriter,
+  encoder: TextEncoder,
+  aiMessageId: string,
+  chatId: string,
+) => {
+  let recievedMessage = '';
+  let sources: any[] = [];
+
+  stream.on('data', (data) => {
+    const parsedData = JSON.parse(data);
+    if (parsedData.type === 'response') {
+      writer.write(
+        encoder.encode(
+          JSON.stringify({
+            type: 'message',
+            data: parsedData.data,
+            messageId: aiMessageId,
+          }) + '\n',
+        ),
+      );
+
+      recievedMessage += parsedData.data;
+    } else if (parsedData.type === 'sources') {
+      writer.write(
+        encoder.encode(
+          JSON.stringify({
+            type: 'sources',
+            data: parsedData.data,
+            messageId: aiMessageId,
+          }) + '\n',
+        ),
+      );
+
+      sources = parsedData.data;
+    }
+  });
+  stream.on('end', () => {
+    writer.write(
+      encoder.encode(
+        JSON.stringify({
+          type: 'messageEnd',
+          messageId: aiMessageId,
+        }) + '\n',
+      ),
+    );
+    writer.close();
+
+    db.insert(messagesSchema)
+      .values({
+        content: recievedMessage,
+        chatId: chatId,
+        messageId: aiMessageId,
+        role: 'assistant',
+        metadata: JSON.stringify({
+          createdAt: new Date(),
+          ...(sources && sources.length > 0 && { sources }),
+        }),
+      })
+      .execute();
+  });
+  stream.on('error', (data) => {
+    const parsedData = JSON.parse(data);
+    writer.write(
+      encoder.encode(
+        JSON.stringify({
+          type: 'error',
+          data: parsedData.data,
+        }),
+      ),
+    );
+    writer.close();
+  });
+};
+
+const handleHistorySave = async (
+  message: Message,
+  humanMessageId: string,
+  focusMode: string,
+  files: string[],
+) => {
+  const chat = await db.query.chats.findFirst({
+    where: eq(chats.id, message.chatId),
+  });
+
+  if (!chat) {
+    await db
+      .insert(chats)
+      .values({
+        id: message.chatId,
+        title: message.content,
+        createdAt: new Date().toString(),
+        focusMode: focusMode,
+        files: files.map(getFileDetails),
+      })
+      .execute();
+  }
+
+  const messageExists = await db.query.messages.findFirst({
+    where: eq(messagesSchema.messageId, humanMessageId),
+  });
+
+  if (!messageExists) {
+    await db
+      .insert(messagesSchema)
+      .values({
+        content: message.content,
+        chatId: message.chatId,
+        messageId: humanMessageId,
+        role: 'user',
+        metadata: JSON.stringify({
+          createdAt: new Date(),
+        }),
+      })
+      .execute();
+  } else {
+    await db
+      .delete(messagesSchema)
+      .where(
+        and(
+          gt(messagesSchema.id, messageExists.id),
+          eq(messagesSchema.chatId, message.chatId),
+        ),
+      )
+      .execute();
+  }
+};
+
+export const POST = async (req: Request) => {
+  try {
+    const body = (await req.json()) as Body;
+    const { message } = body;
+
+    if (message.content === '') {
+      return Response.json(
+        {
+          message: 'Please provide a message to process',
+        },
+        { status: 400 },
+      );
+    }
+
+    const [chatModelProviders, embeddingModelProviders] = await Promise.all([
+      getAvailableChatModelProviders(),
+      getAvailableEmbeddingModelProviders(),
+    ]);
+
+    const chatModelProvider =
+      chatModelProviders[
+        body.chatModel?.provider || Object.keys(chatModelProviders)[0]
+      ];
+    const chatModel =
+      chatModelProvider[
+        body.chatModel?.name || Object.keys(chatModelProvider)[0]
+      ];
+
+    const embeddingProvider =
+      embeddingModelProviders[
+        body.embeddingModel?.provider || Object.keys(embeddingModelProviders)[0]
+      ];
+    const embeddingModel =
+      embeddingProvider[
+        body.embeddingModel?.name || Object.keys(embeddingProvider)[0]
+      ];
+
+    let llm: BaseChatModel | undefined;
+    let embedding = embeddingModel.model;
+
+    if (body.chatModel?.provider === 'custom_openai') {
+      llm = new ChatOpenAI({
+        openAIApiKey: getCustomOpenaiApiKey(),
+        modelName: getCustomOpenaiModelName(),
+        temperature: 0.7,
+        configuration: {
+          baseURL: getCustomOpenaiApiUrl(),
+        },
+      }) as unknown as BaseChatModel;
+    } else if (chatModelProvider && chatModel) {
+      llm = chatModel.model;
+    }
+
+    if (!llm) {
+      return Response.json({ error: 'Invalid chat model' }, { status: 400 });
+    }
+
+    if (!embedding) {
+      return Response.json(
+        { error: 'Invalid embedding model' },
+        { status: 400 },
+      );
+    }
+
+    const humanMessageId =
+      message.messageId ?? crypto.randomBytes(7).toString('hex');
+    const aiMessageId = crypto.randomBytes(7).toString('hex');
+
+    const history: BaseMessage[] = body.history.map((msg) => {
+      if (msg[0] === 'human') {
+        return new HumanMessage({
+          content: msg[1],
+        });
+      } else {
+        return new AIMessage({
+          content: msg[1],
+        });
+      }
+    });
+
+    const handler = searchHandlers[body.focusMode];
+
+    if (!handler) {
+      return Response.json(
+        {
+          message: 'Invalid focus mode',
+        },
+        { status: 400 },
+      );
+    }
+
+    const stream = await handler.searchAndAnswer(
+      message.content,
+      history,
+      llm,
+      embedding,
+      body.optimizationMode,
+      body.files,
+    );
+
+    const responseStream = new TransformStream();
+    const writer = responseStream.writable.getWriter();
+    const encoder = new TextEncoder();
+
+    handleEmitterEvents(stream, writer, encoder, aiMessageId, message.chatId);
+    handleHistorySave(message, humanMessageId, body.focusMode, body.files);
+
+    return new Response(responseStream.readable, {
+      headers: {
+        'Content-Type': 'text/event-stream',
+        Connection: 'keep-alive',
+        'Cache-Control': 'no-cache, no-transform',
+      },
+    });
+  } catch (err) {
+    console.error('An error ocurred while processing chat request:', err);
+    return Response.json(
+      { message: 'An error ocurred while processing chat request' },
+      { status: 500 },
+    );
+  }
+};

src/app/api/chats/[id]/route.ts

@@ -0,0 +1,69 @@
+import db from '@/lib/db';
+import { chats, messages } from '@/lib/db/schema';
+import { eq } from 'drizzle-orm';
+
+export const GET = async (
+  req: Request,
+  { params }: { params: Promise<{ id: string }> },
+) => {
+  try {
+    const { id } = await params;
+
+    const chatExists = await db.query.chats.findFirst({
+      where: eq(chats.id, id),
+    });
+
+    if (!chatExists) {
+      return Response.json({ message: 'Chat not found' }, { status: 404 });
+    }
+
+    const chatMessages = await db.query.messages.findMany({
+      where: eq(messages.chatId, id),
+    });
+
+    return Response.json(
+      {
+        chat: chatExists,
+        messages: chatMessages,
+      },
+      { status: 200 },
+    );
+  } catch (err) {
+    console.error('Error in getting chat by id: ', err);
+    return Response.json(
+      { message: 'An error has occurred.' },
+      { status: 500 },
+    );
+  }
+};
+
+export const DELETE = async (
+  req: Request,
+  { params }: { params: Promise<{ id: string }> },
+) => {
+  try {
+    const { id } = await params;
+
+    const chatExists = await db.query.chats.findFirst({
+      where: eq(chats.id, id),
+    });
+
+    if (!chatExists) {
+      return Response.json({ message: 'Chat not found' }, { status: 404 });
+    }
+
+    await db.delete(chats).where(eq(chats.id, id)).execute();
+    await db.delete(messages).where(eq(messages.chatId, id)).execute();
+
+    return Response.json(
+      { message: 'Chat deleted successfully' },
+      { status: 200 },
+    );
+  } catch (err) {
+    console.error('Error in deleting chat by id: ', err);
+    return Response.json(
+      { message: 'An error has occurred.' },
+      { status: 500 },
+    );
+  }
+};

src/app/api/chats/route.ts

@@ -0,0 +1,15 @@
+import db from '@/lib/db';
+
+export const GET = async (req: Request) => {
+  try {
+    let chats = await db.query.chats.findMany();
+    chats = chats.reverse();
+    return Response.json({ chats: chats }, { status: 200 });
+  } catch (err) {
+    console.error('Error in getting chats: ', err);
+    return Response.json(
+      { message: 'An error has occurred.' },
+      { status: 500 },
+    );
+  }
+};

src/app/api/config/route.ts

@@ -0,0 +1,109 @@
+import {
+  getAnthropicApiKey,
+  getCustomOpenaiApiKey,
+  getCustomOpenaiApiUrl,
+  getCustomOpenaiModelName,
+  getGeminiApiKey,
+  getGroqApiKey,
+  getOllamaApiEndpoint,
+  getOpenaiApiKey,
+  updateConfig,
+} from '@/lib/config';
+import {
+  getAvailableChatModelProviders,
+  getAvailableEmbeddingModelProviders,
+} from '@/lib/providers';
+
+export const GET = async (req: Request) => {
+  try {
+    const config: Record<string, any> = {};
+
+    const [chatModelProviders, embeddingModelProviders] = await Promise.all([
+      getAvailableChatModelProviders(),
+      getAvailableEmbeddingModelProviders(),
+    ]);
+
+    config['chatModelProviders'] = {};
+    config['embeddingModelProviders'] = {};
+
+    for (const provider in chatModelProviders) {
+      config['chatModelProviders'][provider] = Object.keys(
+        chatModelProviders[provider],
+      ).map((model) => {
+        return {
+          name: model,
+          displayName: chatModelProviders[provider][model].displayName,
+        };
+      });
+    }
+
+    for (const provider in embeddingModelProviders) {
+      config['embeddingModelProviders'][provider] = Object.keys(
+        embeddingModelProviders[provider],
+      ).map((model) => {
+        return {
+          name: model,
+          displayName: embeddingModelProviders[provider][model].displayName,
+        };
+      });
+    }
+
+    config['openaiApiKey'] = getOpenaiApiKey();
+    config['ollamaApiUrl'] = getOllamaApiEndpoint();
+    config['anthropicApiKey'] = getAnthropicApiKey();
+    config['groqApiKey'] = getGroqApiKey();
+    config['geminiApiKey'] = getGeminiApiKey();
+    config['customOpenaiApiUrl'] = getCustomOpenaiApiUrl();
+    config['customOpenaiApiKey'] = getCustomOpenaiApiKey();
+    config['customOpenaiModelName'] = getCustomOpenaiModelName();
+
+    return Response.json({ ...config }, { status: 200 });
+  } catch (err) {
+    console.error('An error ocurred while getting config:', err);
+    return Response.json(
+      { message: 'An error ocurred while getting config' },
+      { status: 500 },
+    );
+  }
+};
+
+export const POST = async (req: Request) => {
+  try {
+    const config = await req.json();
+
+    const updatedConfig = {
+      MODELS: {
+        OPENAI: {
+          API_KEY: config.openaiApiKey,
+        },
+        GROQ: {
+          API_KEY: config.groqApiKey,
+        },
+        ANTHROPIC: {
+          API_KEY: config.anthropicApiKey,
+        },
+        GEMINI: {
+          API_KEY: config.geminiApiKey,
+        },
+        OLLAMA: {
+          API_URL: config.ollamaApiUrl,
+        },
+        CUSTOM_OPENAI: {
+          API_URL: config.customOpenaiApiUrl,
+          API_KEY: config.customOpenaiApiKey,
+          MODEL_NAME: config.customOpenaiModelName,
+        },
+      },
+    };
+
+    updateConfig(updatedConfig);
+
+    return Response.json({ message: 'Config updated' }, { status: 200 });
+  } catch (err) {
+    console.error('An error ocurred while updating config:', err);
+    return Response.json(
+      { message: 'An error ocurred while updating config' },
+      { status: 500 },
+    );
+  }
+};

src/app/api/discover/route.ts

@@ -0,0 +1,61 @@
+import { searchSearxng } from '@/lib/searxng';
+
+const articleWebsites = [
+  'yahoo.com',
+  'www.exchangewire.com',
+  'businessinsider.com',
+  /* 'wired.com',
+  'mashable.com',
+  'theverge.com',
+  'gizmodo.com',
+  'cnet.com',
+  'venturebeat.com', */
+];
+
+const topics = ['AI', 'tech']; /* TODO: Add UI to customize this */
+
+export const GET = async (req: Request) => {
+  try {
+    const data = (
+      await Promise.all([
+        ...new Array(articleWebsites.length * topics.length)
+          .fill(0)
+          .map(async (_, i) => {
+            return (
+              await searchSearxng(
+                `site:${articleWebsites[i % articleWebsites.length]} ${
+                  topics[i % topics.length]
+                }`,
+                {
+                  engines: ['bing news'],
+                  pageno: 1,
+                },
+              )
+            ).results;
+          }),
+      ])
+    )
+      .map((result) => result)
+      .flat()
+      .sort(() => Math.random() - 0.5);
+
+    return Response.json(
+      {
+        blogs: data,
+      },
+      {
+        status: 200,
+      },
+    );
+  } catch (err) {
+    console.error(`An error ocurred in discover route: ${err}`);
+    return Response.json(
+      {
+        message: 'An error has occurred',
+      },
+      {
+        status: 500,
+      },
+    );
+  }
+};

src/app/api/images/route.ts

@@ -0,0 +1,83 @@
+import handleImageSearch from '@/lib/chains/imageSearchAgent';
+import {
+  getCustomOpenaiApiKey,
+  getCustomOpenaiApiUrl,
+  getCustomOpenaiModelName,
+} from '@/lib/config';
+import { getAvailableChatModelProviders } from '@/lib/providers';
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+import { ChatOpenAI } from '@langchain/openai';
+
+interface ChatModel {
+  provider: string;
+  model: string;
+}
+
+interface ImageSearchBody {
+  query: string;
+  chatHistory: any[];
+  chatModel?: ChatModel;
+}
+
+export const POST = async (req: Request) => {
+  try {
+    const body: ImageSearchBody = await req.json();
+
+    const chatHistory = body.chatHistory
+      .map((msg: any) => {
+        if (msg.role === 'user') {
+          return new HumanMessage(msg.content);
+        } else if (msg.role === 'assistant') {
+          return new AIMessage(msg.content);
+        }
+      })
+      .filter((msg) => msg !== undefined) as BaseMessage[];
+
+    const chatModelProviders = await getAvailableChatModelProviders();
+
+    const chatModelProvider =
+      chatModelProviders[
+        body.chatModel?.provider || Object.keys(chatModelProviders)[0]
+      ];
+    const chatModel =
+      chatModelProvider[
+        body.chatModel?.model || Object.keys(chatModelProvider)[0]
+      ];
+
+    let llm: BaseChatModel | undefined;
+
+    if (body.chatModel?.provider === 'custom_openai') {
+      llm = new ChatOpenAI({
+        openAIApiKey: getCustomOpenaiApiKey(),
+        modelName: getCustomOpenaiModelName(),
+        temperature: 0.7,
+        configuration: {
+          baseURL: getCustomOpenaiApiUrl(),
+        },
+      }) as unknown as BaseChatModel;
+    } else if (chatModelProvider && chatModel) {
+      llm = chatModel.model;
+    }
+
+    if (!llm) {
+      return Response.json({ error: 'Invalid chat model' }, { status: 400 });
+    }
+
+    const images = await handleImageSearch(
+      {
+        chat_history: chatHistory,
+        query: body.query,
+      },
+      llm,
+    );
+
+    return Response.json({ images }, { status: 200 });
+  } catch (err) {
+    console.error(`An error ocurred while searching images: ${err}`);
+    return Response.json(
+      { message: 'An error ocurred while searching images' },
+      { status: 500 },
+    );
+  }
+};

src/app/api/models/route.ts

@@ -0,0 +1,47 @@
+import {
+  getAvailableChatModelProviders,
+  getAvailableEmbeddingModelProviders,
+} from '@/lib/providers';
+
+export const GET = async (req: Request) => {
+  try {
+    const [chatModelProviders, embeddingModelProviders] = await Promise.all([
+      getAvailableChatModelProviders(),
+      getAvailableEmbeddingModelProviders(),
+    ]);
+
+    Object.keys(chatModelProviders).forEach((provider) => {
+      Object.keys(chatModelProviders[provider]).forEach((model) => {
+        delete (chatModelProviders[provider][model] as { model?: unknown })
+          .model;
+      });
+    });
+
+    Object.keys(embeddingModelProviders).forEach((provider) => {
+      Object.keys(embeddingModelProviders[provider]).forEach((model) => {
+        delete (embeddingModelProviders[provider][model] as { model?: unknown })
+          .model;
+      });
+    });
+
+    return Response.json(
+      {
+        chatModelProviders,
+        embeddingModelProviders,
+      },
+      {
+        status: 200,
+      },
+    );
+  } catch (err) {
+    console.error('An error ocurred while fetching models', err);
+    return Response.json(
+      {
+        message: 'An error has occurred.',
+      },
+      {
+        status: 500,
+      },
+    );
+  }
+};

src/app/api/suggestions/route.ts

@@ -0,0 +1,81 @@
+import generateSuggestions from '@/lib/chains/suggestionGeneratorAgent';
+import {
+  getCustomOpenaiApiKey,
+  getCustomOpenaiApiUrl,
+  getCustomOpenaiModelName,
+} from '@/lib/config';
+import { getAvailableChatModelProviders } from '@/lib/providers';
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+import { ChatOpenAI } from '@langchain/openai';
+
+interface ChatModel {
+  provider: string;
+  model: string;
+}
+
+interface SuggestionsGenerationBody {
+  chatHistory: any[];
+  chatModel?: ChatModel;
+}
+
+export const POST = async (req: Request) => {
+  try {
+    const body: SuggestionsGenerationBody = await req.json();
+
+    const chatHistory = body.chatHistory
+      .map((msg: any) => {
+        if (msg.role === 'user') {
+          return new HumanMessage(msg.content);
+        } else if (msg.role === 'assistant') {
+          return new AIMessage(msg.content);
+        }
+      })
+      .filter((msg) => msg !== undefined) as BaseMessage[];
+
+    const chatModelProviders = await getAvailableChatModelProviders();
+
+    const chatModelProvider =
+      chatModelProviders[
+        body.chatModel?.provider || Object.keys(chatModelProviders)[0]
+      ];
+    const chatModel =
+      chatModelProvider[
+        body.chatModel?.model || Object.keys(chatModelProvider)[0]
+      ];
+
+    let llm: BaseChatModel | undefined;
+
+    if (body.chatModel?.provider === 'custom_openai') {
+      llm = new ChatOpenAI({
+        openAIApiKey: getCustomOpenaiApiKey(),
+        modelName: getCustomOpenaiModelName(),
+        temperature: 0.7,
+        configuration: {
+          baseURL: getCustomOpenaiApiUrl(),
+        },
+      }) as unknown as BaseChatModel;
+    } else if (chatModelProvider && chatModel) {
+      llm = chatModel.model;
+    }
+
+    if (!llm) {
+      return Response.json({ error: 'Invalid chat model' }, { status: 400 });
+    }
+
+    const suggestions = await generateSuggestions(
+      {
+        chat_history: chatHistory,
+      },
+      llm,
+    );
+
+    return Response.json({ suggestions }, { status: 200 });
+  } catch (err) {
+    console.error(`An error ocurred while generating suggestions: ${err}`);
+    return Response.json(
+      { message: 'An error ocurred while generating suggestions' },
+      { status: 500 },
+    );
+  }
+};

src/app/api/uploads/route.ts

@@ -0,0 +1,134 @@
+import { NextResponse } from 'next/server';
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { getAvailableEmbeddingModelProviders } from '@/lib/providers';
+import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
+import { DocxLoader } from '@langchain/community/document_loaders/fs/docx';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import { Document } from 'langchain/document';
+
+interface FileRes {
+  fileName: string;
+  fileExtension: string;
+  fileId: string;
+}
+
+const uploadDir = path.join(process.cwd(), 'uploads');
+
+if (!fs.existsSync(uploadDir)) {
+  fs.mkdirSync(uploadDir, { recursive: true });
+}
+
+const splitter = new RecursiveCharacterTextSplitter({
+  chunkSize: 500,
+  chunkOverlap: 100,
+});
+
+export async function POST(req: Request) {
+  try {
+    const formData = await req.formData();
+
+    const files = formData.getAll('files') as File[];
+    const embedding_model = formData.get('embedding_model');
+    const embedding_model_provider = formData.get('embedding_model_provider');
+
+    if (!embedding_model || !embedding_model_provider) {
+      return NextResponse.json(
+        { message: 'Missing embedding model or provider' },
+        { status: 400 },
+      );
+    }
+
+    const embeddingModels = await getAvailableEmbeddingModelProviders();
+    const provider =
+      embedding_model_provider ?? Object.keys(embeddingModels)[0];
+    const embeddingModel =
+      embedding_model ?? Object.keys(embeddingModels[provider as string])[0];
+
+    let embeddingsModel =
+      embeddingModels[provider as string]?.[embeddingModel as string]?.model;
+    if (!embeddingsModel) {
+      return NextResponse.json(
+        { message: 'Invalid embedding model selected' },
+        { status: 400 },
+      );
+    }
+
+    const processedFiles: FileRes[] = [];
+
+    await Promise.all(
+      files.map(async (file: any) => {
+        const fileExtension = file.name.split('.').pop();
+        if (!['pdf', 'docx', 'txt'].includes(fileExtension!)) {
+          return NextResponse.json(
+            { message: 'File type not supported' },
+            { status: 400 },
+          );
+        }
+
+        const uniqueFileName = `${crypto.randomBytes(16).toString('hex')}.${fileExtension}`;
+        const filePath = path.join(uploadDir, uniqueFileName);
+
+        const buffer = Buffer.from(await file.arrayBuffer());
+        fs.writeFileSync(filePath, new Uint8Array(buffer));
+
+        let docs: any[] = [];
+        if (fileExtension === 'pdf') {
+          const loader = new PDFLoader(filePath);
+          docs = await loader.load();
+        } else if (fileExtension === 'docx') {
+          const loader = new DocxLoader(filePath);
+          docs = await loader.load();
+        } else if (fileExtension === 'txt') {
+          const text = fs.readFileSync(filePath, 'utf-8');
+          docs = [
+            new Document({ pageContent: text, metadata: { title: file.name } }),
+          ];
+        }
+
+        const splitted = await splitter.splitDocuments(docs);
+
+        const extractedDataPath = filePath.replace(/\.\w+$/, '-extracted.json');
+        fs.writeFileSync(
+          extractedDataPath,
+          JSON.stringify({
+            title: file.name,
+            contents: splitted.map((doc) => doc.pageContent),
+          }),
+        );
+
+        const embeddings = await embeddingsModel.embedDocuments(
+          splitted.map((doc) => doc.pageContent),
+        );
+        const embeddingsDataPath = filePath.replace(
+          /\.\w+$/,
+          '-embeddings.json',
+        );
+        fs.writeFileSync(
+          embeddingsDataPath,
+          JSON.stringify({
+            title: file.name,
+            embeddings,
+          }),
+        );
+
+        processedFiles.push({
+          fileName: file.name,
+          fileExtension: fileExtension,
+          fileId: uniqueFileName.replace(/\.\w+$/, ''),
+        });
+      }),
+    );
+
+    return NextResponse.json({
+      files: processedFiles,
+    });
+  } catch (error) {
+    console.error('Error uploading file:', error);
+    return NextResponse.json(
+      { message: 'An error has occurred.' },
+      { status: 500 },
+    );
+  }
+}

src/app/api/videos/route.ts

@@ -0,0 +1,83 @@
+import handleVideoSearch from '@/lib/chains/videoSearchAgent';
+import {
+  getCustomOpenaiApiKey,
+  getCustomOpenaiApiUrl,
+  getCustomOpenaiModelName,
+} from '@/lib/config';
+import { getAvailableChatModelProviders } from '@/lib/providers';
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+import { ChatOpenAI } from '@langchain/openai';
+
+interface ChatModel {
+  provider: string;
+  model: string;
+}
+
+interface VideoSearchBody {
+  query: string;
+  chatHistory: any[];
+  chatModel?: ChatModel;
+}
+
+export const POST = async (req: Request) => {
+  try {
+    const body: VideoSearchBody = await req.json();
+
+    const chatHistory = body.chatHistory
+      .map((msg: any) => {
+        if (msg.role === 'user') {
+          return new HumanMessage(msg.content);
+        } else if (msg.role === 'assistant') {
+          return new AIMessage(msg.content);
+        }
+      })
+      .filter((msg) => msg !== undefined) as BaseMessage[];
+
+    const chatModelProviders = await getAvailableChatModelProviders();
+
+    const chatModelProvider =
+      chatModelProviders[
+        body.chatModel?.provider || Object.keys(chatModelProviders)[0]
+      ];
+    const chatModel =
+      chatModelProvider[
+        body.chatModel?.model || Object.keys(chatModelProvider)[0]
+      ];
+
+    let llm: BaseChatModel | undefined;
+
+    if (body.chatModel?.provider === 'custom_openai') {
+      llm = new ChatOpenAI({
+        openAIApiKey: getCustomOpenaiApiKey(),
+        modelName: getCustomOpenaiModelName(),
+        temperature: 0.7,
+        configuration: {
+          baseURL: getCustomOpenaiApiUrl(),
+        },
+      }) as unknown as BaseChatModel;
+    } else if (chatModelProvider && chatModel) {
+      llm = chatModel.model;
+    }
+
+    if (!llm) {
+      return Response.json({ error: 'Invalid chat model' }, { status: 400 });
+    }
+
+    const videos = await handleVideoSearch(
+      {
+        chat_history: chatHistory,
+        query: body.query,
+      },
+      llm,
+    );
+
+    return Response.json({ videos }, { status: 200 });
+  } catch (err) {
+    console.error(`An error ocurred while searching videos: ${err}`);
+    return Response.json(
+      { message: 'An error ocurred while searching videos' },
+      { status: 500 },
+    );
+  }
+};