// History backfill + incremental polling against voip.ms. import { EventEmitter } from 'node:events' import type { MessageDb, NewMessage } from './db' import { parseVoipDate, type RawSms, type VoipMs } from './voipms' import { charCount, normalizeNumber, SMS_TEXT_LIMIT } from '@shared/phone' import type { SyncStatus } from '@shared/types' const WINDOW_DAYS = 90 const PAGE_LIMIT = 10000 // Stop walking back after this many consecutive empty 90-day windows (~2 years). const MAX_EMPTY_WINDOWS = 8 const DAY = 86_400_000 export function ymd(ms: number): string { return new Date(ms).toISOString().slice(0, 10) } function parseYmd(s: string): number { return Date.parse(s + 'T00:00:00Z') } export const isMmsPlaceholder = (body: string): boolean => body.trim().toLowerCase() === 'mms received' export function toNewMessage(r: RawSms): NewMessage { const m: NewMessage = { id: Number(r.id), did: normalizeNumber(r.did), contact: normalizeNumber(r.contact), direction: r.type === '1' ? 'in' : 'out', body: r.message ?? '', ts: parseVoipDate(r.date) } // With all_messages=1 every row carries the media columns, SMS included, so // only rows that actually have media count as MMS -- or text too long for // one SMS, which voip.ms carries as a text-only MMS. const urls = (r.media?.length ? r.media : [r.col_media1, r.col_media2, r.col_media3]).filter((u): u is string => !!u) if (urls.length) { m.kind = 'mms' m.media = urls.map((url) => ({ url })) // voip.ms's own stand-in text for a picture with no words, not the sender's. if (isMmsPlaceholder(m.body)) m.body = '' } else if (charCount(m.body) > SMS_TEXT_LIMIT) { m.kind = 'mms' } return m } // Fetch every message in [from, to] (inclusive days), halving the range if a // single call comes back full so nothing gets truncated. export async function fetchRange(api: Pick, from: number, to: number, limit = PAGE_LIMIT): Promise { const rows = await api.getMessages({ from: ymd(from), to: ymd(to), limit }) if (rows.length < limit || to - from < DAY) return rows const mid = from + Math.floor((to - from) / DAY / 2) * DAY const a = await fetchRange(api, from, mid, limit) const b = await fetchRange(api, mid + DAY, to, limit) return [...a, ...b] } export interface SyncOptions { pollSeconds: number dids: string[] // empty = all backfillFloor: string } export class SyncEngine extends EventEmitter { private pollTimer?: NodeJS.Timeout private burstUntil = 0 private running = false private backfilling = false private polling?: Promise private status: SyncStatus constructor( private db: MessageDb, private api: VoipMs, private opts: SyncOptions ) { super() this.status = { phase: 'idle', backfillDone: db.get('backfill_done') === '1', backfillCursor: db.get('backfill_cursor'), total: db.count() } } getStatus(): SyncStatus { return this.status } private setStatus(patch: Partial): void { this.status = { ...this.status, ...patch, total: this.db.count() } this.emit('status', this.status) } private store(raw: RawSms[]): void { const wanted = new Set(this.opts.dids.map(normalizeNumber)) const msgs = raw.map(toNewMessage).filter((m) => wanted.size === 0 || wanted.has(m.did)) const added = this.db.upsertMessages(msgs) this.db.reconcileSent(added) if (added.length) this.emit('messages', added) } start(): void { if (this.running) return this.running = true if (!this.db.get('unread_floor')) this.db.set('unread_floor', String(Date.now())) void this.poll().finally(() => this.scheduleNextPoll()) if (!this.status.backfillDone) void this.backfill() } stop(): void { this.running = false clearTimeout(this.pollTimer) } // Poll faster for a couple of minutes (e.g. right after sending). burst(): void { this.burstUntil = Date.now() + 2 * 60_000 clearTimeout(this.pollTimer) this.scheduleNextPoll(3000) } private scheduleNextPoll(delay?: number): void { if (!this.running) return const d = delay ?? (Date.now() < this.burstUntil ? 5000 : this.opts.pollSeconds * 1000) this.pollTimer = setTimeout(() => { void this.poll().finally(() => this.scheduleNextPoll()) }, d) } // Manual refresh. Joins a check that's already under way rather than // returning early, so the caller knows when fresh results are in. // Resolves false when the listing couldn't be fetched. pollNow(): Promise { return this.poll() } private poll(): Promise { this.polling ??= this.fetchRecent().finally(() => (this.polling = undefined)) return this.polling } private async fetchRecent(): Promise { try { const now = Date.now() const rows = await this.api.getMessages({ from: ymd(now - DAY), to: ymd(now + DAY), limit: PAGE_LIMIT }) this.store(rows) this.setStatus({ lastPoll: Date.now(), error: undefined, phase: this.backfilling ? 'backfill' : 'polling' }) } catch (e) { this.setStatus({ phase: 'error', error: (e as Error).message }) return false } return true } private async backfill(): Promise { if (this.backfilling) return this.backfilling = true this.setStatus({ phase: 'backfill' }) const floor = parseYmd(this.opts.backfillFloor) let end = this.db.get('backfill_cursor') ? parseYmd(this.db.get('backfill_cursor')!) : parseYmd(ymd(Date.now() + DAY)) let empty = Number(this.db.get('backfill_empty') ?? 0) try { while (this.running && end >= floor && empty < MAX_EMPTY_WINDOWS) { const start = Math.max(floor, end - (WINDOW_DAYS - 1) * DAY) const rows = await fetchRange(this.api, start, end) this.store(rows) empty = rows.length ? 0 : empty + 1 end = start - DAY this.db.set('backfill_cursor', ymd(end)) this.db.set('backfill_empty', String(empty)) this.setStatus({ phase: 'backfill', backfillCursor: ymd(end), error: undefined }) await new Promise((r) => setTimeout(r, 250)) } if (this.running) { this.db.set('backfill_done', '1') this.setStatus({ phase: 'polling', backfillDone: true }) } } catch (e) { this.setStatus({ phase: 'error', error: (e as Error).message }) // Try again shortly; progress is saved so it resumes where it left off. setTimeout(() => this.running && void this.backfill(), 30_000) } finally { this.backfilling = false } } // Start the history import over (e.g. after changing the floor date). resetBackfill(): void { for (const k of ['backfill_cursor', 'backfill_empty', 'backfill_done']) this.db.db.prepare('DELETE FROM kv WHERE key = ?').run(k) this.status.backfillDone = false if (this.running) void this.backfill() } }