diff --git a/sample.config.toml b/sample.config.toml index 980e99d..1db2125 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -25,5 +25,8 @@ API_URL = "" # Ollama API URL - http://host.docker.internal:11434 [MODELS.DEEPSEEK] API_KEY = "" +[MODELS.LM_STUDIO] +API_URL = "" # LM Studio API URL - http://host.docker.internal:1234 + [API_ENDPOINTS] -SEARXNG = "" # SearxNG API URL - http://localhost:32768 \ No newline at end of file +SEARXNG = "" # SearxNG API URL - http://localhost:32768 diff --git a/src/lib/config.ts b/src/lib/config.ts index 2831214..7c6d495 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -28,6 +28,9 @@ interface Config { DEEPSEEK: { API_KEY: string; }; + LM_STUDIO: { + API_URL: string; + }; CUSTOM_OPENAI: { API_URL: string; API_KEY: string; @@ -77,6 +80,8 @@ export const getCustomOpenaiApiUrl = () => export const getCustomOpenaiModelName = () => loadConfig().MODELS.CUSTOM_OPENAI.MODEL_NAME; +export const getLMStudioApiEndpoint = () => loadConfig().MODELS.LM_STUDIO.API_URL; + const mergeConfigs = (current: any, update: any): any => { if (update === null || update === undefined) { return current; diff --git a/src/lib/providers/index.ts b/src/lib/providers/index.ts index eef212f..0a4a6db 100644 --- a/src/lib/providers/index.ts +++ b/src/lib/providers/index.ts @@ -13,6 +13,7 @@ import { loadAnthropicChatModels } from './anthropic'; import { loadGeminiChatModels, loadGeminiEmbeddingModels } from './gemini'; import { loadTransformersEmbeddingsModels } from './transformers'; import { loadDeepseekChatModels } from './deepseek'; +import { loadLMStudioChatModels, loadLMStudioEmbeddingsModels } from './lmstudio'; export interface ChatModel { displayName: string; @@ -34,6 +35,7 @@ export const chatModelProviders: Record< anthropic: loadAnthropicChatModels, gemini: loadGeminiChatModels, deepseek: loadDeepseekChatModels, + lmstudio: loadLMStudioChatModels, }; export const embeddingModelProviders: Record< @@ -44,6 +46,7 @@ export const embeddingModelProviders: Record< ollama: loadOllamaEmbeddingModels, gemini: loadGeminiEmbeddingModels, transformers: loadTransformersEmbeddingsModels, + lmstudio: loadLMStudioEmbeddingsModels, }; export const getAvailableChatModelProviders = async () => { diff --git a/src/lib/providers/lmstudio.ts b/src/lib/providers/lmstudio.ts new file mode 100644 index 0000000..fd8eb75 --- /dev/null +++ b/src/lib/providers/lmstudio.ts @@ -0,0 +1,101 @@ +import { getKeepAlive, getLMStudioApiEndpoint } from '../config'; +import axios from 'axios'; +import { ChatModel, EmbeddingModel } from '.'; +import { ChatOpenAI } from '@langchain/openai'; +import { OpenAIEmbeddings } from '@langchain/openai'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import { Embeddings } from '@langchain/core/embeddings'; + +interface LMStudioModel { + id: string; + name?: string; +} + +const ensureV1Endpoint = (endpoint: string): string => + endpoint.endsWith('/v1') ? endpoint : `${endpoint}/v1`; + +const checkServerAvailability = async (endpoint: string): Promise => { + try { + const keepAlive = getKeepAlive(); + await axios.get(`${ensureV1Endpoint(endpoint)}/models`, { + timeout: parseInt(keepAlive) * 1000 || 5000, + headers: { 'Content-Type': 'application/json' }, + }); + return true; + } catch { + return false; + } +}; + +export const loadLMStudioChatModels = async () => { + const endpoint = getLMStudioApiEndpoint(); + const keepAlive = getKeepAlive(); + + if (!endpoint) return {}; + if (!await checkServerAvailability(endpoint)) return {}; + + try { + const response = await axios.get(`${ensureV1Endpoint(endpoint)}/models`, { + timeout: parseInt(keepAlive) * 1000 || 5000, + headers: { 'Content-Type': 'application/json' }, + }); + + const chatModels: Record = {}; + + response.data.data.forEach((model: LMStudioModel) => { + chatModels[model.id] = { + displayName: model.name || model.id, + model: new ChatOpenAI({ + openAIApiKey: 'lm-studio', + configuration: { + baseURL: ensureV1Endpoint(endpoint), + }, + modelName: model.id, + temperature: 0.7, + streaming: true, + maxRetries: 3 + }) as unknown as BaseChatModel, + }; + }); + + return chatModels; + } catch (err) { + console.error(`Error loading LM Studio models: ${err}`); + return {}; + } +}; + +export const loadLMStudioEmbeddingsModels = async () => { + const endpoint = getLMStudioApiEndpoint(); + const keepAlive = getKeepAlive(); + + if (!endpoint) return {}; + if (!await checkServerAvailability(endpoint)) return {}; + + try { + const response = await axios.get(`${ensureV1Endpoint(endpoint)}/models`, { + timeout: parseInt(keepAlive) * 1000 || 5000, + headers: { 'Content-Type': 'application/json' }, + }); + + const embeddingsModels: Record = {}; + + response.data.data.forEach((model: LMStudioModel) => { + embeddingsModels[model.id] = { + displayName: model.name || model.id, + model: new OpenAIEmbeddings({ + openAIApiKey: 'lm-studio', + configuration: { + baseURL: ensureV1Endpoint(endpoint), + }, + modelName: model.id, + }) as unknown as Embeddings, + }; + }); + + return embeddingsModels; + } catch (err) { + console.error(`Error loading LM Studio embeddings model: ${err}`); + return {}; + } +};