From 650c69e04fd28dd6cfb5aa0aed4b0ebb2647d492 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns <95534749+ItzCrazyKns@users.noreply.github.com> Date: Fri, 26 Sep 2025 13:44:39 +0530 Subject: [PATCH] feat(db): fix migration errors, explicitly migrate based on index --- app.dockerfile | 13 ++++- drizzle/0001_acoustic_wild_pack.sql | 51 ---------------- drizzle/0001_wise_rockslide.sql | 1 + drizzle/meta/0001_snapshot.json | 2 +- drizzle/meta/_journal.json | 4 +- package.json | 4 +- src/lib/db/migrate.ts | 90 ++++++++++++++++++++++++++++- 7 files changed, 103 insertions(+), 62 deletions(-) delete mode 100644 drizzle/0001_acoustic_wild_pack.sql create mode 100644 drizzle/0001_wise_rockslide.sql diff --git a/app.dockerfile b/app.dockerfile index c3c0fd0..ab4f4cc 100644 --- a/app.dockerfile +++ b/app.dockerfile @@ -1,4 +1,6 @@ -FROM node:20.18.0-slim AS builder +FROM node:24.5.0-slim AS builder + +RUN apt-get update && apt-get install -y python3 python3-pip sqlite3 && rm -rf /var/lib/apt/lists/* WORKDIR /home/perplexica @@ -8,6 +10,7 @@ RUN yarn install --frozen-lockfile --network-timeout 600000 COPY tsconfig.json next.config.mjs next-env.d.ts postcss.config.js drizzle.config.ts tailwind.config.ts ./ COPY src ./src COPY public ./public +COPY drizzle ./drizzle RUN mkdir -p /home/perplexica/data RUN yarn build @@ -15,7 +18,9 @@ RUN yarn build RUN yarn add --dev @vercel/ncc RUN yarn ncc build ./src/lib/db/migrate.ts -o migrator -FROM node:20.18.0-slim +FROM node:24.5.0-slim + +RUN apt-get update && apt-get install -y python3 python3-pip sqlite3 && rm -rf /var/lib/apt/lists/* WORKDIR /home/perplexica @@ -32,4 +37,6 @@ RUN mkdir /home/perplexica/uploads COPY entrypoint.sh ./entrypoint.sh RUN chmod +x ./entrypoint.sh -CMD ["./entrypoint.sh"] \ No newline at end of file +RUN sed -i 's/\r$//' ./entrypoint.sh || true + +CMD ["/home/perplexica/entrypoint.sh"] \ No newline at end of file diff --git a/drizzle/0001_acoustic_wild_pack.sql b/drizzle/0001_acoustic_wild_pack.sql deleted file mode 100644 index 977e6b3..0000000 --- a/drizzle/0001_acoustic_wild_pack.sql +++ /dev/null @@ -1,51 +0,0 @@ -PRAGMA foreign_keys=OFF; ---> statement-breakpoint - -CREATE TABLE `__new_messages` ( - `id` integer PRIMARY KEY NOT NULL, - `type` text NOT NULL, - `chatId` text NOT NULL, - `createdAt` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `messageId` text NOT NULL, - `content` text, - `sources` text DEFAULT '[]' -); ---> statement-breakpoint - -INSERT INTO `__new_messages`("id", "type", "chatId", "createdAt", "messageId", "content", "sources") -SELECT - id, - COALESCE(type, 'user') as type, - chatId, - CASE - WHEN metadata IS NOT NULL AND instr(metadata, '\"createdAt\":\"') > 0 THEN - substr( - metadata, - instr(metadata, '\"createdAt\":\"') + 14, - CASE - WHEN instr(substr(metadata, instr(metadata, '\"createdAt\":\"') + 14), '\"') > 0 THEN - instr(substr(metadata, instr(metadata, '\"createdAt\":\"') + 14), '\"') - 1 - ELSE 24 - END - ) - ELSE CURRENT_TIMESTAMP - END as createdAt, - messageId, - content, - '[]' as sources -FROM `messages`; ---> statement-breakpoint - -DROP TABLE `messages`; ---> statement-breakpoint -ALTER TABLE `__new_messages` RENAME TO `messages`; ---> statement-breakpoint - -PRAGMA foreign_keys=ON; ---> statement-breakpoint - -CREATE INDEX IF NOT EXISTS `idx_messages_chatId` ON `messages` (`chatId`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_messages_type` ON `messages` (`type`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_messages_createdAt` ON `messages` (`createdAt`); \ No newline at end of file diff --git a/drizzle/0001_wise_rockslide.sql b/drizzle/0001_wise_rockslide.sql new file mode 100644 index 0000000..b48d53b --- /dev/null +++ b/drizzle/0001_wise_rockslide.sql @@ -0,0 +1 @@ +/* Do nothing */ \ No newline at end of file diff --git a/drizzle/meta/0001_snapshot.json b/drizzle/meta/0001_snapshot.json index e131360..2773290 100644 --- a/drizzle/meta/0001_snapshot.json +++ b/drizzle/meta/0001_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "766180d3-caab-4b9e-9f35-61ee7b652903", + "id": "6dedf55f-0e44-478f-82cf-14a21ac686f8", "prevId": "ef3a044b-0f34-40b5-babb-2bb3a909ba27", "tables": { "chats": { diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 9c3456e..e67ba06 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -12,8 +12,8 @@ { "idx": 1, "version": "6", - "when": 1756793457846, - "tag": "0001_acoustic_wild_pack", + "when": 1758863991284, + "tag": "0001_wise_rockslide", "breakpoints": true } ] diff --git a/package.json b/package.json index 5715c2a..4cc7e8d 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,11 @@ "author": "ItzCrazyKns", "scripts": { "dev": "next dev", - "build": "npm run db:push && next build", + "build": "npm run db:migrate && next build", "start": "next start", "lint": "next lint", "format:write": "prettier . --write", - "db:push": "drizzle-kit push" + "db:migrate": "node ./src/lib/db/migrate.ts" }, "dependencies": { "@headlessui/react": "^2.2.0", diff --git a/src/lib/db/migrate.ts b/src/lib/db/migrate.ts index c3ebff6..309c12d 100644 --- a/src/lib/db/migrate.ts +++ b/src/lib/db/migrate.ts @@ -1,5 +1,89 @@ -import db from './'; -import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import Database from 'better-sqlite3'; import path from 'path'; +import fs from 'fs'; -migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') }); +const db = new Database(path.join(process.cwd(), 'data', 'db.sqlite')); + +const migrationsFolder = path.join(process.cwd(), 'drizzle'); + +db.exec(` + CREATE TABLE IF NOT EXISTS ran_migrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + run_on DATETIME DEFAULT CURRENT_TIMESTAMP + ); +`); + +function sanitizeSql(content: string) { + return content + .split(/\r?\n/) + .filter((l) => !l.trim().startsWith('-->') && !l.includes('statement-breakpoint')) + .join('\n'); +} + +fs.readdirSync(migrationsFolder) + .filter((f) => f.endsWith('.sql')) + .sort() + .forEach((file) => { + const filePath = path.join(migrationsFolder, file); + let content = fs.readFileSync(filePath, 'utf-8'); + content = sanitizeSql(content); + + const migrationName = file.split('_')[0] || file; + + const already = db.prepare('SELECT 1 FROM ran_migrations WHERE name = ?').get(migrationName); + if (already) { + console.log(`Skipping already-applied migration: ${file}`); + return; + } + + try { + if (migrationName === '0001') { + const messages = db.prepare('SELECT id, type, metadata, content, chatId, messageId FROM messages').all(); + + db.exec(` + CREATE TABLE IF NOT EXISTS messages_with_sources ( + id INTEGER PRIMARY KEY, + type TEXT NOT NULL, + chatId TEXT NOT NULL, + createdAt TEXT NOT NULL, + messageId TEXT NOT NULL, + content TEXT, + sources TEXT DEFAULT '[]' + ); + `); + + const insertMessage = db.prepare(` + INSERT INTO messages_with_sources (type, chatId, createdAt, messageId, content, sources) + VALUES (?, ?, ?, ?, ?, ?) + `); + + messages.forEach((msg: any) => { + if (msg.type === 'user') { + msg.metadata = JSON.parse(msg.metadata || '{}'); + insertMessage.run('user', msg.chatId, msg.metadata['createdAt'], msg.messageId, msg.content, '[]'); + } else if (msg.type === 'assistant') { + msg.metadata = JSON.parse(msg.metadata || '{}'); + insertMessage.run('assistant', msg.chatId, msg.metadata['createdAt'], msg.messageId, msg.content, '[]'); + const sources = msg.metadata['sources'] || '[]' + if (sources && sources.length > 0) { + insertMessage.run('source', msg.chatId, msg.metadata['createdAt'], `${msg.messageId}-source`, '', JSON.stringify(sources)); + } + } + }); + + db.exec('DROP TABLE messages;'); + db.exec('ALTER TABLE messages_with_sources RENAME TO messages;'); + + } else { + db.exec(content); + } + + db.prepare('INSERT OR IGNORE INTO ran_migrations (name) VALUES (?)').run(migrationName); + console.log(`Applied migration: ${file}`); + + } catch (err) { + console.error(`Failed to apply migration ${file}:`, err); + throw err; + } + }); \ No newline at end of file