// Local message store: SQLite (WAL) with an FTS5 index for search. import Database from 'better-sqlite3' import fs from 'node:fs' import path from 'node:path' import type { Conversation, Direction, MediaRef, Message, MessageKind, MessageStatus, SearchHit } from '@shared/types' import { SNIPPET_END, SNIPPET_START } from '@shared/types' import { fixVoipDst } from './voipms' const SCHEMA_VERSION = 10 export interface ReadState { did: string contact: string lastRead: number // read up to this message time markedUnread: boolean unreadTs?: number | null // when read/unread last changed archivedAt?: number | null archivedTs?: number | null // when archived/unarchived last changed } export interface NameChange { did: string contact: string name: string | null ts: number } export interface NewMessage { id: number did: string contact: string direction: Direction body: string ts: number status?: MessageStatus kind?: MessageKind media?: MediaRef[] sender?: string } type MessageRow = Omit & { media_json: string | null; error: string | null; sender: string | null } const MESSAGE_COLS = 'id, did, contact, direction, body, ts, kind, status, media_json, error, sender' // How far a sent message's voip.ms time can be from the time we stamped it // locally and still count as the same message (sends can queue and retry). const MATCH_BEFORE = 2 * 60_000 const MATCH_AFTER = 15 * 60_000 // A text with its emoji (and their joiners and variation selectors) taken // out, trimmed: what's left of it in voip.ms's listing. export function withoutEmoji(s: string): string { let out = '' for (const ch of s) { const c = ch.codePointAt(0)! const emoji = c >= 0x1f000 || (c >= 0x2600 && c <= 0x27bf) || (c >= 0x2300 && c <= 0x23ff) || (c >= 0x2b00 && c <= 0x2bff) || (c >= 0xfe00 && c <= 0xfe0f) || c === 0x200d || c === 0x20e3 if (!emoji) out += ch } return out.trim() } function fromRow({ media_json, error, sender, ...m }: MessageRow): Message { return { ...m, ...(media_json ? { media: JSON.parse(media_json) as MediaRef[] } : {}), ...(error ? { error } : {}), ...(sender ? { sender } : {}) } } export class MessageDb { readonly db: Database.Database constructor(private file: string) { fs.mkdirSync(path.dirname(file), { recursive: true }) this.db = new Database(file) this.db.pragma('journal_mode = WAL') this.db.pragma('synchronous = NORMAL') this.db.pragma('foreign_keys = ON') this.migrate() } private migrate(): void { const version = this.db.pragma('user_version', { simple: true }) as number if (version >= SCHEMA_VERSION) return if (version < 1) this.db.exec(` CREATE TABLE IF NOT EXISTS messages ( id INTEGER PRIMARY KEY, -- voip.ms id (negative = local pending) did TEXT NOT NULL, contact TEXT NOT NULL, direction TEXT NOT NULL CHECK (direction IN ('in','out')), body TEXT NOT NULL, ts INTEGER NOT NULL, -- epoch ms UTC kind TEXT NOT NULL DEFAULT 'sms', -- 'sms' | 'mms' status TEXT NOT NULL DEFAULT 'delivered', media_json TEXT -- MMS attachments: MediaRef[] ); CREATE INDEX IF NOT EXISTS messages_conv ON messages (did, contact, ts); CREATE INDEX IF NOT EXISTS messages_ts ON messages (ts); CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5 ( body, content='messages', content_rowid='id', tokenize='unicode61 remove_diacritics 2' ); CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(rowid, body) VALUES (new.id, new.body); END; CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, body) VALUES ('delete', old.id, old.body); END; CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE OF id, body ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, body) VALUES ('delete', old.id, old.body); INSERT INTO messages_fts(rowid, body) VALUES (new.id, new.body); END; CREATE TABLE IF NOT EXISTS read_state ( did TEXT NOT NULL, contact TEXT NOT NULL, last_read_ts INTEGER NOT NULL, PRIMARY KEY (did, contact) ); CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY, value TEXT NOT NULL); `) // v2: sync switched from getSMS to getMMS (which also returns MMS). Redo the // history import once so past MMS show up; existing rows dedupe by id. if (version === 1) this.db.exec(`DELETE FROM kv WHERE key IN ('backfill_cursor', 'backfill_empty', 'backfill_done')`) // v3: per-conversation archive + manual "mark as unread". if (version < 3) this.db.exec(` ALTER TABLE read_state ADD COLUMN archived_at INTEGER; -- epoch ms; NULL = not archived ALTER TABLE read_state ADD COLUMN marked_unread INTEGER NOT NULL DEFAULT 0; `) // v4: times from voip.ms were an hour late during US daylight time, and // every getMMS row was labelled 'mms'. Times from voip.ms are whole seconds; // ones stamped locally on send carry milliseconds and were right. if (version >= 1 && version < 4) { const rows = this.db.prepare('SELECT id, ts FROM messages WHERE ts % 1000 = 0').all() as Array<{ id: number; ts: number }> const update = this.db.prepare('UPDATE messages SET ts = ? WHERE id = ?') this.db.transaction(() => { for (const r of rows) { const fixed = fixVoipDst(r.ts) if (fixed !== r.ts) update.run(fixed, r.id) } this.db.exec(`UPDATE messages SET kind = 'sms' WHERE kind = 'mms' AND media_json IS NULL`) })() } // v5: texts too long for one SMS came in as text-only MMS but were labelled // 'sms' (only rows with media counted as MMS). if (version >= 1 && version < 5) this.db.exec(`UPDATE messages SET kind = 'mms' WHERE kind = 'sms' AND length(body) > 160`) // v6: why a send failed, shown on the message. const cols = this.db.pragma('table_info(messages)') as Array<{ name: string }> if (!cols.some((c) => c.name === 'error')) this.db.exec(`ALTER TABLE messages ADD COLUMN error TEXT`) // v7: Google Voice. Who sent each incoming group message, and which GV // thread each conversation is (group threads have opaque ids). if (!cols.some((c) => c.name === 'sender')) this.db.exec(`ALTER TABLE messages ADD COLUMN sender TEXT`) this.db.exec(` CREATE TABLE IF NOT EXISTS gv_threads ( did TEXT NOT NULL, contact TEXT NOT NULL, thread_id TEXT NOT NULL, PRIMARY KEY (did, contact) ); `) // v8: a name of your own for a conversation (group chats), kept locally. const rsCols = this.db.pragma('table_info(read_state)') as Array<{ name: string }> if (!rsCols.some((c) => c.name === 'name')) this.db.exec(`ALTER TABLE read_state ADD COLUMN name TEXT`) // v9: when a name was set, so renames on the desktop and the Rabbit merge (newest wins). if (!rsCols.some((c) => c.name === 'name_ts')) this.db.exec(`ALTER TABLE read_state ADD COLUMN name_ts INTEGER`) // v10: when read/unread and archived last changed, so they merge with the Rabbit too. if (!rsCols.some((c) => c.name === 'unread_ts')) this.db.exec(`ALTER TABLE read_state ADD COLUMN unread_ts INTEGER`) if (!rsCols.some((c) => c.name === 'archived_ts')) this.db.exec(`ALTER TABLE read_state ADD COLUMN archived_ts INTEGER`) this.db.pragma(`user_version = ${SCHEMA_VERSION}`) } // ---- key/value state ---- get(key: string): string | undefined { const r = this.db.prepare('SELECT value FROM kv WHERE key = ?').get(key) as { value: string } | undefined return r?.value } set(key: string, value: string): void { this.db.prepare('INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value) } // ---- unsent drafts, one per contact ---- getDrafts(): Record { const rows = this.db.prepare("SELECT key, value FROM kv WHERE key LIKE 'draft:%'").all() as { key: string; value: string }[] return Object.fromEntries(rows.map((r) => [r.key.slice('draft:'.length), r.value])) } setDraft(contact: string, text: string): void { if (text.trim()) this.set(`draft:${contact}`, text) else this.db.prepare('DELETE FROM kv WHERE key = ?').run(`draft:${contact}`) } // ---- messages ---- // Inserts messages we haven't seen; returns the ones that were new. upsertMessages(msgs: NewMessage[]): NewMessage[] { const insert = this.db.prepare( `INSERT OR IGNORE INTO messages (id, did, contact, direction, body, ts, status, kind, media_json, sender) VALUES (@id, @did, @contact, @direction, @body, @ts, @status, @kind, @media_json, @sender)` ) // Any message newer than the archiving brings a conversation back (older // ones arriving via the history import don't). const unarchive = this.db.prepare( `UPDATE read_state SET archived_at = NULL WHERE did = ? AND contact = ? AND archived_at IS NOT NULL AND archived_at < ?` ) const added: NewMessage[] = [] this.db.transaction(() => { for (const m of msgs) { const row = { id: m.id, did: m.did, contact: m.contact, direction: m.direction, body: m.body, ts: m.ts, status: m.status ?? 'delivered', kind: m.kind ?? 'sms', media_json: m.media?.length ? JSON.stringify(m.media) : null, sender: m.sender ?? null } if (insert.run(row).changes) { added.push(m) unarchive.run(m.did, m.contact, m.ts) } } })() return added } insertPending(m: Omit): number { const row = this.db.prepare('SELECT MIN(id) AS min FROM messages').get() as { min: number | null } const id = Math.min(-1, (row.min ?? 0) - 1) this.upsertMessages([{ ...m, id, status: 'pending' }]) return id } // Replace a local pending row with the id voip.ms assigned. confirmPending(tempId: number, realId: number): void { this.db.transaction(() => { const exists = this.getMessage(realId) if (exists) { // Poll beat us to it: keep our local copies of any attachments so they // don't need downloading again, and our kind (a text-only MMS of 160 // characters or less looks like an SMS in the listing). const pending = this.getMessage(tempId) if (pending) this.db.prepare('UPDATE messages SET kind = ? WHERE id = ?').run(pending.kind, realId) const local = pending?.media if (local && exists.media) { const merged = exists.media.map((ref, i) => (local[i]?.file ? { ...ref, ...local[i] } : ref)) this.db.prepare('UPDATE messages SET media_json = ? WHERE id = ?').run(JSON.stringify(merged), realId) } this.db.prepare('DELETE FROM messages WHERE id = ?').run(tempId) } else { this.db.prepare(`UPDATE messages SET id = ?, status = 'delivered' WHERE id = ?`).run(realId, tempId) } })() } markFailed(tempId: number, error?: string): void { this.db.prepare(`UPDATE messages SET status = 'failed', error = ? WHERE id = ?`).run(error ?? null, tempId) } // Sent messages that turned up in a poll take the place of our local // pending/failed copies (same text, sent around the same time). A send whose // reply never came back may still have gone out; this is how we find out. // Returns the local ids that were replaced. reconcileSent(added: NewMessage[]): number[] { const find = this.db.prepare( `SELECT id, body FROM messages WHERE id < 0 AND status IN ('pending', 'failed') AND direction = 'out' AND did = ? AND contact = ? AND ts BETWEEN ? AND ? ORDER BY ts, id DESC` ) const claimed: number[] = [] this.db.transaction(() => { for (const m of [...added].sort((a, b) => a.ts - b.ts || a.id - b.id)) { if (m.direction !== 'out' || m.id < 0) continue const rows = find.all(m.did, m.contact, m.ts - MATCH_AFTER, m.ts + MATCH_BEFORE) as { id: number; body: string }[] // voip.ms's listing drops emoji ("…everything ❤️" comes back as // "…everything "), so an exact match first, then one without them. const key = withoutEmoji(m.body) const row = rows.find((r) => r.body === m.body) ?? rows.find((r) => withoutEmoji(r.body) === key) if (!row) continue this.confirmPending(row.id, m.id) claimed.push(row.id) } })() return claimed } deleteMessage(id: number): void { this.db.prepare('DELETE FROM messages WHERE id = ?').run(id) } getMessage(id: number): Message | undefined { const row = this.db.prepare(`SELECT ${MESSAGE_COLS} FROM messages WHERE id = ?`).get(id) as MessageRow | undefined return row && fromRow(row) } getConversationMessages(did: string, contact: string): Message[] { const rows = this.db .prepare(`SELECT ${MESSAGE_COLS} FROM messages WHERE did = ? AND contact = ? ORDER BY ts, id`) .all(did, contact) as MessageRow[] return rows.map(fromRow) } // Record that attachment `index` of message `id` is now cached locally. setMediaFile(id: number, index: number, patch: MediaRef): void { this.db.transaction(() => { const media = this.getMessage(id)?.media if (!media?.[index]) return media[index] = { ...media[index], ...patch } this.db.prepare('UPDATE messages SET media_json = ? WHERE id = ?').run(JSON.stringify(media), id) })() } count(): number { return (this.db.prepare('SELECT COUNT(*) AS n FROM messages').get() as { n: number }).n } // ---- conversations ---- listConversations(): Conversation[] { // Messages from before the initial history import never count as unread. const floor = Number(this.get('unread_floor') ?? 0) const rows = this.db .prepare( `WITH latest AS ( SELECT did, contact, MAX(ts) AS mts FROM messages GROUP BY did, contact ) SELECT l.did, l.contact, l.mts AS lastTs, last.body AS lastBody, last.direction AS lastDirection, last.kind AS lastKind, last.sender AS lastSender, json_extract(last.media_json, '$[0]') AS lastMedia, (SELECT COUNT(*) FROM messages m WHERE m.did = l.did AND m.contact = l.contact AND m.direction = 'in' AND m.ts > MAX(COALESCE(r.last_read_ts, 0), @floor)) AS unread, r.archived_at IS NOT NULL AS archived, COALESCE(r.marked_unread, 0) AS markedUnread, r.name AS name FROM latest l JOIN messages last ON last.id = (SELECT id FROM messages m WHERE m.did = l.did AND m.contact = l.contact ORDER BY ts DESC, id DESC LIMIT 1) LEFT JOIN read_state r ON r.did = l.did AND r.contact = l.contact ORDER BY l.mts DESC` ) .all({ floor }) as Array< Omit & { lastMedia: string | null archived: number markedUnread: number lastSender: string | null name: string | null } > return rows.map(({ lastMedia, archived, markedUnread, lastSender, name, ...c }) => ({ ...c, ...(name ? { name } : {}), ...(lastSender ? { lastSender } : {}), archived: !!archived, markedUnread: !!markedUnread, ...(lastMedia ? { lastMedia: JSON.parse(lastMedia) as MediaRef } : {}) })) } // Whether anything has arrived since the conversation was last read. hasUnread(did: string, contact: string): boolean { const floor = Number(this.get('unread_floor') ?? 0) const r = this.db .prepare( `SELECT EXISTS ( SELECT 1 FROM messages m LEFT JOIN read_state r ON r.did = m.did AND r.contact = m.contact WHERE m.did = @did AND m.contact = @contact AND m.direction = 'in' AND m.ts > MAX(COALESCE(r.last_read_ts, 0), @floor) ) AS any` ) .get({ did, contact, floor }) as { any: number } return !!r.any } markRead(did: string, contact: string): void { this.db .prepare( `INSERT INTO read_state (did, contact, last_read_ts, unread_ts) SELECT @did, @contact, COALESCE(MAX(ts), 0), @now FROM messages WHERE did = @did AND contact = @contact ON CONFLICT(did, contact) DO UPDATE SET last_read_ts = MAX(last_read_ts, excluded.last_read_ts), marked_unread = 0, unread_ts = excluded.unread_ts` ) .run({ did, contact, now: Date.now() }) } markUnread(did: string, contact: string): void { this.db .prepare( `INSERT INTO read_state (did, contact, last_read_ts, marked_unread, unread_ts) VALUES (?, ?, 0, 1, ?) ON CONFLICT(did, contact) DO UPDATE SET marked_unread = 1, unread_ts = excluded.unread_ts` ) .run(did, contact, Date.now()) } // Name a conversation (null/empty clears it, back to the members' names). setName(did: string, contact: string, name: string | null, ts = Date.now()): void { this.db .prepare( `INSERT INTO read_state (did, contact, last_read_ts, name, name_ts) VALUES (?, ?, 0, ?, ?) ON CONFLICT(did, contact) DO UPDATE SET name = excluded.name, name_ts = excluded.name_ts` ) .run(did, contact, name?.trim() || null, ts) } // Every rename (including clearing one), for merging with the Rabbit. namesExport(): NameChange[] { return this.db.prepare('SELECT did, contact, name, name_ts AS ts FROM read_state WHERE name_ts IS NOT NULL').all() as NameChange[] } // Apply renames made elsewhere, keeping whichever is newer. Returns how many changed. namesImport(changes: NameChange[]): number { let n = 0 const mine = this.db.prepare('SELECT COALESCE(name_ts, 0) AS ts FROM read_state WHERE did = ? AND contact = ?') this.db.transaction(() => { for (const c of changes) { const cur = (mine.get(c.did, c.contact) as { ts: number } | undefined)?.ts ?? 0 if (c.ts > cur) { this.setName(c.did, c.contact, c.name, c.ts) n++ } } })() return n } getName(did: string, contact: string): string | undefined { const r = this.db.prepare('SELECT name FROM read_state WHERE did = ? AND contact = ?').get(did, contact) as { name: string | null } | undefined return r?.name ?? undefined } setArchived(did: string, contact: string, archived: boolean, now = Date.now()): void { this.db .prepare( `INSERT INTO read_state (did, contact, last_read_ts, archived_at, archived_ts) VALUES (?, ?, 0, ?, ?) ON CONFLICT(did, contact) DO UPDATE SET archived_at = excluded.archived_at, archived_ts = excluded.archived_ts` ) .run(did, contact, archived ? now : null, now) } // Read/unread and archived state of every conversation, for merging with the Rabbit. readStateExport(): ReadState[] { return ( this.db .prepare( `SELECT did, contact, last_read_ts AS lastRead, marked_unread AS markedUnread, unread_ts AS unreadTs, archived_at AS archivedAt, archived_ts AS archivedTs FROM read_state` ) .all() as Array & { markedUnread: number }> ).map((r) => ({ ...r, markedUnread: !!r.markedUnread })) } // Merge the other device's state: "read up to" takes the later of the two; // marked-unread and archived take whichever changed more recently. // Returns how many conversations changed here. readStateImport(rows: ReadState[]): number { let n = 0 const mine = this.db.prepare( `SELECT last_read_ts AS lastRead, marked_unread AS markedUnread, COALESCE(unread_ts, 0) AS unreadTs, archived_at AS archivedAt, COALESCE(archived_ts, 0) AS archivedTs FROM read_state WHERE did = ? AND contact = ?` ) const upsert = this.db.prepare( `INSERT INTO read_state (did, contact, last_read_ts, marked_unread, unread_ts, archived_at, archived_ts) VALUES (@did, @contact, @lastRead, @markedUnread, @unreadTs, @archivedAt, @archivedTs) ON CONFLICT(did, contact) DO UPDATE SET last_read_ts = excluded.last_read_ts, marked_unread = excluded.marked_unread, unread_ts = excluded.unread_ts, archived_at = excluded.archived_at, archived_ts = excluded.archived_ts` ) this.db.transaction(() => { for (const t of rows) { const m = mine.get(t.did, t.contact) as | { lastRead: number; markedUnread: number; unreadTs: number; archivedAt: number | null; archivedTs: number } | undefined const cur = m ?? { lastRead: 0, markedUnread: 0, unreadTs: 0, archivedAt: null, archivedTs: 0 } const theirsUnread = (t.unreadTs ?? 0) > cur.unreadTs const theirsArchived = (t.archivedTs ?? 0) > cur.archivedTs const next = { did: t.did, contact: t.contact, lastRead: Math.max(cur.lastRead, t.lastRead ?? 0), markedUnread: theirsUnread ? (t.markedUnread ? 1 : 0) : cur.markedUnread, unreadTs: theirsUnread ? t.unreadTs : cur.unreadTs || null, archivedAt: theirsArchived ? (t.archivedAt ?? null) : cur.archivedAt, archivedTs: theirsArchived ? t.archivedTs : cur.archivedTs || null } if (next.lastRead !== cur.lastRead || theirsUnread || theirsArchived) { upsert.run(next) n++ } } })() return n } // ---- Google Voice threads ---- setGvThread(did: string, contact: string, threadId: string): void { this.db .prepare( `INSERT INTO gv_threads (did, contact, thread_id) VALUES (?, ?, ?) ON CONFLICT(did, contact) DO UPDATE SET thread_id = excluded.thread_id` ) .run(did, contact, threadId) } getGvThread(did: string, contact: string): string | undefined { const r = this.db.prepare('SELECT thread_id FROM gv_threads WHERE did = ? AND contact = ?').get(did, contact) as { thread_id: string } | undefined return r?.thread_id } // ---- search ---- search(query: string, scope?: { did: string; contact: string }, limit = 300): SearchHit[] { const match = toFtsQuery(query) if (!match) return [] const where = scope ? 'AND m.did = @did AND m.contact = @contact' : '' return this.db .prepare( `SELECT m.id, m.did, m.contact, m.ts, m.direction, snippet(messages_fts, 0, @s, @e, '…', 14) AS snippet FROM messages_fts JOIN messages m ON m.id = messages_fts.rowid WHERE messages_fts MATCH @match ${where} ORDER BY m.ts DESC LIMIT @limit` ) .all({ match, s: SNIPPET_START, e: SNIPPET_END, limit, did: scope?.did, contact: scope?.contact }) as SearchHit[] } // ---- backups ---- // Write a consistent copy to dir/messages-YYYY-MM-DD.db and prune old ones. async snapshot(dir: string, keep = 7): Promise { fs.mkdirSync(dir, { recursive: true }) const dest = path.join(dir, `messages-${localDay()}.db`) const tmp = dest + '.tmp' await this.db.backup(tmp) fs.renameSync(tmp, dest) const old = fs .readdirSync(dir) .filter((f) => /^messages-\d{4}-\d{2}-\d{2}\.db$/.test(f)) .sort() .reverse() .slice(keep) for (const f of old) fs.rmSync(path.join(dir, f), { force: true }) return dest } close(): void { this.db.close() } } // Today's date as YYYY-MM-DD in local time (snapshot file names). export function localDay(d = new Date()): string { return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` } // Turn free text into a safe FTS5 query: every word must appear (prefix match). export function toFtsQuery(input: string): string { const words = input .normalize('NFKC') .split(/[^\p{L}\p{N}]+/u) .filter(Boolean) return words.map((w) => `"${w.replace(/"/g, '""')}"*`).join(' ') }