feat(providers): add models.json for models list

src/lib/providers/transformers.ts

@@ -1,31 +1,30 @@
-import { HuggingFaceTransformersEmbeddings } from '../huggingfaceTransformer';
+import { EmbeddingModel, getModelsList, RawModel } from '.'
+import { HuggingFaceTransformersEmbeddings } from '../huggingfaceTransformer'
+
+const loadModels = () => {
+  return getModelsList()?.embeddingModels[
+    'transformers'
+  ] as unknown as RawModel[]
+}
 
 export const loadTransformersEmbeddingsModels = async () => {
   try {
-    const embeddingModels = {
-      'xenova-bge-small-en-v1.5': {
-        displayName: 'BGE Small',
-        model: new HuggingFaceTransformersEmbeddings({
-          modelName: 'Xenova/bge-small-en-v1.5',
-        }),
-      },
-      'xenova-gte-small': {
-        displayName: 'GTE Small',
-        model: new HuggingFaceTransformersEmbeddings({
-          modelName: 'Xenova/gte-small',
-        }),
-      },
-      'xenova-bert-base-multilingual-uncased': {
-        displayName: 'Bert Multilingual',
-        model: new HuggingFaceTransformersEmbeddings({
-          modelName: 'Xenova/bert-base-multilingual-uncased',
-        }),
-      },
-    };
+    const models = loadModels()
 
-    return embeddingModels;
+    const embeddingModels: Record<string, EmbeddingModel> = {}
+
+    models.forEach(model => {
+      embeddingModels[model.key] = {
+        displayName: model.displayName,
+        model: new HuggingFaceTransformersEmbeddings({
+          modelName: model.key,
+        }),
+      }
+    })
+
+    return embeddingModels
   } catch (err) {
-    console.error(`Error loading Transformers embeddings model: ${err}`);
-    return {};
+    console.error(`Error loading Transformers embeddings model: ${err}`)
+    return {}
   }
-};
+}