feat(db): fix migration errors, explicitly migrate based on index

app.dockerfile

@@ -1,4 +1,6 @@
-FROM node:20.18.0-slim AS builder
+FROM node:24.5.0-slim AS builder
+
+RUN apt-get update && apt-get install -y python3 python3-pip sqlite3 && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/perplexica
 
@@ -8,6 +10,7 @@ RUN yarn install --frozen-lockfile --network-timeout 600000
 COPY tsconfig.json next.config.mjs next-env.d.ts postcss.config.js drizzle.config.ts tailwind.config.ts ./
 COPY src ./src
 COPY public ./public
+COPY drizzle ./drizzle
 
 RUN mkdir -p /home/perplexica/data
 RUN yarn build
@@ -15,7 +18,9 @@ RUN yarn build
 RUN yarn add --dev @vercel/ncc
 RUN yarn ncc build ./src/lib/db/migrate.ts -o migrator
 
-FROM node:20.18.0-slim
+FROM node:24.5.0-slim
+
+RUN apt-get update && apt-get install -y python3 python3-pip sqlite3 && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/perplexica
 
@@ -32,4 +37,6 @@ RUN mkdir /home/perplexica/uploads
 
 COPY entrypoint.sh ./entrypoint.sh
 RUN chmod +x ./entrypoint.sh
-CMD ["./entrypoint.sh"]
+RUN sed -i 's/\r$//' ./entrypoint.sh || true
+
+CMD ["/home/perplexica/entrypoint.sh"]

drizzle/0001_acoustic_wild_pack.sql

@@ -1,51 +0,0 @@
-PRAGMA foreign_keys=OFF;
---> statement-breakpoint
-
-CREATE TABLE `__new_messages` (
-	`id` integer PRIMARY KEY NOT NULL,
-	`type` text NOT NULL,
-	`chatId` text NOT NULL,
-	`createdAt` text DEFAULT CURRENT_TIMESTAMP NOT NULL,
-	`messageId` text NOT NULL,
-	`content` text,
-	`sources` text DEFAULT '[]'
-);
---> statement-breakpoint
-
-INSERT INTO `__new_messages`("id", "type", "chatId", "createdAt", "messageId", "content", "sources") 
-SELECT 
-    id,
-    COALESCE(type, 'user') as type,
-    chatId,
-    CASE
-        WHEN metadata IS NOT NULL AND instr(metadata, '\"createdAt\":\"') > 0 THEN
-            substr(
-                metadata,
-                instr(metadata, '\"createdAt\":\"') + 14,
-                CASE
-                    WHEN instr(substr(metadata, instr(metadata, '\"createdAt\":\"') + 14), '\"') > 0 THEN
-                        instr(substr(metadata, instr(metadata, '\"createdAt\":\"') + 14), '\"') - 1
-                    ELSE 24
-                END
-            )
-        ELSE CURRENT_TIMESTAMP
-    END as createdAt,
-    messageId,
-    content,
-    '[]' as sources
-FROM `messages`;
---> statement-breakpoint
-
-DROP TABLE `messages`;
---> statement-breakpoint
-ALTER TABLE `__new_messages` RENAME TO `messages`;
---> statement-breakpoint
-
-PRAGMA foreign_keys=ON;
---> statement-breakpoint
-
-CREATE INDEX IF NOT EXISTS `idx_messages_chatId` ON `messages` (`chatId`);
---> statement-breakpoint
-CREATE INDEX IF NOT EXISTS `idx_messages_type` ON `messages` (`type`);
---> statement-breakpoint
-CREATE INDEX IF NOT EXISTS `idx_messages_createdAt` ON `messages` (`createdAt`);

drizzle/0001_wise_rockslide.sql

@@ -0,0 +1 @@
+/* Do nothing */

drizzle/meta/0001_snapshot.json

@@ -1,7 +1,7 @@
 {
   "version": "6",
   "dialect": "sqlite",
-  "id": "766180d3-caab-4b9e-9f35-61ee7b652903",
+  "id": "6dedf55f-0e44-478f-82cf-14a21ac686f8",
   "prevId": "ef3a044b-0f34-40b5-babb-2bb3a909ba27",
   "tables": {
     "chats": {

drizzle/meta/_journal.json

@@ -12,8 +12,8 @@
     {
       "idx": 1,
       "version": "6",
-      "when": 1756793457846,
+      "when": 1758863991284,
-      "tag": "0001_acoustic_wild_pack",
+      "tag": "0001_wise_rockslide",
       "breakpoints": true
     }
   ]

package.json

@@ -5,11 +5,11 @@
   "author": "ItzCrazyKns",
   "scripts": {
     "dev": "next dev",
-    "build": "npm run db:push && next build",
+    "build": "npm run db:migrate && next build",
     "start": "next start",
     "lint": "next lint",
     "format:write": "prettier . --write",
-    "db:push": "drizzle-kit push"
+    "db:migrate": "node ./src/lib/db/migrate.ts"
   },
   "dependencies": {
     "@headlessui/react": "^2.2.0",

src/lib/db/migrate.ts

@@ -1,5 +1,89 @@
-import db from './';
+import Database from 'better-sqlite3';
-import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
 import path from 'path';
+import fs from 'fs';
 
-migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') });
+const db = new Database(path.join(process.cwd(), 'data', 'db.sqlite'));
+
+const migrationsFolder = path.join(process.cwd(), 'drizzle');
+
+db.exec(`
+  CREATE TABLE IF NOT EXISTS ran_migrations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL UNIQUE,
+    run_on DATETIME DEFAULT CURRENT_TIMESTAMP
+  );
+`);
+
+function sanitizeSql(content: string) {
+    return content
+        .split(/\r?\n/)
+        .filter((l) => !l.trim().startsWith('-->') && !l.includes('statement-breakpoint'))
+        .join('\n');
+}
+
+fs.readdirSync(migrationsFolder)
+    .filter((f) => f.endsWith('.sql'))
+    .sort()
+    .forEach((file) => {
+        const filePath = path.join(migrationsFolder, file);
+        let content = fs.readFileSync(filePath, 'utf-8');
+        content = sanitizeSql(content);
+
+        const migrationName = file.split('_')[0] || file;
+
+        const already = db.prepare('SELECT 1 FROM ran_migrations WHERE name = ?').get(migrationName);
+        if (already) {
+            console.log(`Skipping already-applied migration: ${file}`);
+            return;
+        }
+
+        try {
+            if (migrationName === '0001') {
+                const messages = db.prepare('SELECT id, type, metadata, content, chatId, messageId FROM messages').all();
+
+                db.exec(`
+                    CREATE TABLE IF NOT EXISTS messages_with_sources (
+                        id INTEGER PRIMARY KEY,
+                        type TEXT NOT NULL,
+                        chatId TEXT NOT NULL,
+                        createdAt TEXT NOT NULL,
+                        messageId TEXT NOT NULL,
+                        content TEXT,
+                        sources TEXT DEFAULT '[]'
+                    );
+                `);
+
+                const insertMessage = db.prepare(`
+                    INSERT INTO messages_with_sources (type, chatId, createdAt, messageId, content, sources)
+                    VALUES (?, ?, ?, ?, ?, ?)
+                `);
+
+                messages.forEach((msg: any) => {
+                    if (msg.type === 'user') {
+                        msg.metadata = JSON.parse(msg.metadata || '{}');
+                        insertMessage.run('user', msg.chatId, msg.metadata['createdAt'], msg.messageId, msg.content, '[]');
+                    } else if (msg.type === 'assistant') {
+                        msg.metadata = JSON.parse(msg.metadata || '{}');
+                        insertMessage.run('assistant', msg.chatId, msg.metadata['createdAt'], msg.messageId, msg.content, '[]');
+                        const sources = msg.metadata['sources'] || '[]'
+                        if (sources && sources.length > 0) {
+                            insertMessage.run('source', msg.chatId, msg.metadata['createdAt'], `${msg.messageId}-source`, '', JSON.stringify(sources));
+                        }
+                    }
+                });
+
+                db.exec('DROP TABLE messages;');
+                db.exec('ALTER TABLE messages_with_sources RENAME TO messages;');
+
+            } else {
+                db.exec(content);
+            }
+
+            db.prepare('INSERT OR IGNORE INTO ran_migrations (name) VALUES (?)').run(migrationName);
+            console.log(`Applied migration: ${file}`);
+
+        } catch (err) {
+            console.error(`Failed to apply migration ${file}:`, err);
+            throw err;
+        }
+    });