diff --git a/nexusai-migration-consolidation.patch b/nexusai-migration-consolidation.patch new file mode 100644 index 0000000..f0f2d11 --- /dev/null +++ b/nexusai-migration-consolidation.patch @@ -0,0 +1,409 @@ +diff -ruN nexusai-baseline/packages/memory-service/src/db/index.js nexusai/packages/memory-service/src/db/index.js +--- nexusai-baseline/packages/memory-service/src/db/index.js 2026-08-17 10:23:40.095197576 +0000 ++++ nexusai/packages/memory-service/src/db/index.js 2026-08-17 11:39:45.973925342 +0000 +@@ -1,6 +1,7 @@ + const Database = require('better-sqlite3'); + const schema = require('./schema'); +-const {getEnv, SQLITE, logger } = require('@nexusai/shared'); ++const { migrate } = require('./migrations'); ++const { getEnv, SQLITE, logger } = require('@nexusai/shared'); + + let db; // Declare db variable in a scope accessible to all functions + +@@ -12,60 +13,29 @@ + db.pragma('journal_mode = WAL'); + db.pragma('foreign_keys = ON'); + +- db.exec(schema); +- +- try{ +- db.exec(`ALTER TABLE sessions ADD COLUMN name TEXT`) +- } catch {} +- +- try { +- db.exec(`ALTER TABLE sessions ADD COLUMN project_id INTEGER REFERENCES projects(id)`); +- } catch {} +- +- try { +- db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id)`); +- } catch {} +- +- try { +- db.exec(`ALTER TABLE projects ADD COLUMN isolated INTEGER NOT NULL DEFAULT 0`); +- } catch {} +- +- try { +- db.exec(`ALTER TABLE projects ADD COLUMN notes TEXT`); // ← add this +- } catch {} +- +- try { +- db.exec(`ALTER TABLE projects ADD COLUMN system_prompt TEXT`); +- } catch {} +- +- try { +- db.exec(`ALTER TABLE summaries ADD COLUMN project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE`); +- } catch {} +- +- try { +- db.exec(`ALTER TABLE summaries ADD COLUMN token_count INTEGER`); +- } catch {} +- +- try { +- db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id)`); +- } catch {} +- +- try { +- db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id)`); +- } catch {} +- +- try { db.exec(`ALTER TABLE entities ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {} +- try { db.exec(`ALTER TABLE entities ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0`) } catch {} +- try { db.exec(`ALTER TABLE entities ADD COLUMN source TEXT NOT NULL DEFAULT 'extraction'`) } catch {} +- try { db.exec(`ALTER TABLE entities ADD COLUMN last_seen_at INTEGER`) } catch {} +- +- try { db.exec(`ALTER TABLE relationships ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {} +- try { db.exec(`ALTER TABLE relationships ADD COLUMN notes TEXT`) } catch {} +- +- +- // Sync FTS index with any existing episodes data +- db.exec(`INSERT OR REPLACE INTO episodes_fts(rowid, user_message, ai_response) +- SELECT id, user_message, ai_response FROM episodes`); ++ // Was the FTS index absent before this boot? (True for a fresh DB, and for ++ // an older DB from before FTS existed.) Checked BEFORE schema runs so we can ++ // decide whether a one-time backfill is needed below. ++ const ftsExisted = db.prepare( ++ `SELECT 1 FROM sqlite_master WHERE type='table' AND name='episodes_fts'` ++ ).get() !== undefined; ++ ++ db.exec(schema); // complete current shape — fresh DBs get everything ++ migrate(db); // carry an older DB forward; no-op on fresh/current DBs ++ ++ // One-time FTS backfill: only when the index was just created on a DB that ++ // already holds episodes (i.e. episodes predate FTS). During normal ++ // operation the insert/delete/update triggers keep it in sync, so this no ++ // longer rebuilds the whole index on every boot. NOTE: COUNT(*) on an ++ // external-content FTS5 table proxies the content table, so it can't detect ++ // a desync — the "was it just created" check is what makes this correct. ++ if (!ftsExisted) { ++ const epCount = db.prepare('SELECT COUNT(*) AS c FROM episodes').get().c; ++ if (epCount > 0) { ++ db.exec(`INSERT INTO episodes_fts(episodes_fts) VALUES('rebuild')`); ++ logger.info(`[db] Backfilled FTS index for ${epCount} pre-existing episodes`); ++ } ++ } + + logger.info(`Connected to SQLite database at ${path}`); + } +@@ -74,4 +44,4 @@ + + module.exports = { + getDB +-}; +\ No newline at end of file ++}; +diff -ruN nexusai-baseline/packages/memory-service/src/db/migrations.js nexusai/packages/memory-service/src/db/migrations.js +--- nexusai-baseline/packages/memory-service/src/db/migrations.js 1970-01-01 00:00:00.000000000 +0000 ++++ nexusai/packages/memory-service/src/db/migrations.js 2026-08-17 11:40:33.900163765 +0000 +@@ -0,0 +1,43 @@ ++const { logger } = require('@nexusai/shared'); ++ ++// Forward-only schema migrations. Entry i takes the database from user_version i ++// to i+1, so migrations[0] is the v0→v1 step, migrations[1] the v1→v2 step, etc. ++// ++// schema.js already holds the COMPLETE current shape, so a fresh database is ++// created whole and stamped straight to LATEST_VERSION — these run only to carry ++// an OLDER database forward. Add a new schema change by appending a function here ++// (which bumps LATEST_VERSION by one); never edit an existing entry once shipped, ++// since databases already stamped past it will not re-run it. ++// ++// Each migration receives the better-sqlite3 db handle and runs inside a ++// transaction together with its version bump, so a failure rolls back cleanly. ++const migrations = [ ++ // v0 → v1: baseline. The historical ALTER TABLE / CREATE INDEX statements that ++ // used to run (wrapped in try/catch) on every boot are folded into schema.js. ++ // Nothing to do here — this entry exists to mark v1 as the consolidated baseline. ++ (_db) => {}, ++]; ++ ++const LATEST_VERSION = migrations.length; ++ ++// Applies any migrations newer than the database's current user_version, one at a ++// time, each with its version bump, inside a transaction. Returns the resulting ++// version. A no-op when the database is already current. The migrations list and ++// target version are injectable for testing; production callers pass just `db`. ++function migrate(db, migs = migrations, latest = migs.length) { ++ const current = db.pragma('user_version', { simple: true }); ++ if (current >= latest) return current; ++ ++ for (let v = current; v < latest; v++) { ++ const step = db.transaction(() => { ++ migs[v](db); ++ db.pragma(`user_version = ${v + 1}`); ++ }); ++ step(); ++ } ++ ++ logger.info(`[db] schema migrated ${current} -> ${latest}`); ++ return latest; ++} ++ ++module.exports = { migrate, migrations, LATEST_VERSION }; +diff -ruN nexusai-baseline/packages/memory-service/src/db/schema.js nexusai/packages/memory-service/src/db/schema.js +--- nexusai-baseline/packages/memory-service/src/db/schema.js 2026-08-17 10:23:40.095507736 +0000 ++++ nexusai/packages/memory-service/src/db/schema.js 2026-08-17 11:39:21.421925904 +0000 +@@ -1,10 +1,17 @@ ++// Complete current schema — the single source of truth for a fresh database. ++// Historical ALTER TABLE statements that used to run on every boot are folded in ++// here. Any change that must reach EXISTING databases goes in migrations.js as a ++// new numbered migration, NOT by editing a table below (CREATE ... IF NOT EXISTS ++// silently skips tables that already exist, so column edits here never reach them). + const schema = ` + CREATE TABLE IF NOT EXISTS sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_id TEXT UNIQUE NOT NULL, + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + updated_at INTEGER NOT NULL DEFAULT (unixepoch()), +- metadata TEXT ++ metadata TEXT, ++ name TEXT, ++ project_id INTEGER REFERENCES projects(id) + ); + + CREATE TABLE IF NOT EXISTS episodes ( +@@ -18,23 +25,29 @@ + ); + + CREATE TABLE IF NOT EXISTS entities ( +- id INTEGER PRIMARY KEY AUTOINCREMENT, +- name TEXT NOT NULL, +- type TEXT NOT NULL, +- notes TEXT, +- created_at INTEGER NOT NULL DEFAULT (unixepoch()), +- updated_at INTEGER NOT NULL DEFAULT (unixepoch()), +- metadata TEXT, ++ id INTEGER PRIMARY KEY AUTOINCREMENT, ++ name TEXT NOT NULL, ++ type TEXT NOT NULL, ++ notes TEXT, ++ created_at INTEGER NOT NULL DEFAULT (unixepoch()), ++ updated_at INTEGER NOT NULL DEFAULT (unixepoch()), ++ metadata TEXT, ++ mention_count INTEGER NOT NULL DEFAULT 1, ++ confidence REAL NOT NULL DEFAULT 1.0, ++ source TEXT NOT NULL DEFAULT 'extraction', ++ last_seen_at INTEGER, + UNIQUE(name, type) + ); + + CREATE TABLE IF NOT EXISTS relationships ( +- id INTEGER PRIMARY KEY AUTOINCREMENT, +- from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, +- to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, +- label TEXT NOT NULL, +- created_at INTEGER NOT NULL DEFAULT (unixepoch()), +- metadata TEXT, ++ id INTEGER PRIMARY KEY AUTOINCREMENT, ++ from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, ++ to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, ++ label TEXT NOT NULL, ++ created_at INTEGER NOT NULL DEFAULT (unixepoch()), ++ metadata TEXT, ++ mention_count INTEGER NOT NULL DEFAULT 1, ++ notes TEXT, + UNIQUE(from_id, to_id, label) + ); + +@@ -49,16 +62,17 @@ + + CREATE INDEX IF NOT EXISTS idx_entity_episodes_entity ON entity_episodes(entity_id); + CREATE INDEX IF NOT EXISTS idx_entity_episodes_episode ON entity_episodes(episode_id); +- +- + + CREATE TABLE IF NOT EXISTS projects ( +- id INTEGER PRIMARY KEY AUTOINCREMENT, +- name TEXT NOT NULL, +- description TEXT, +- colour TEXT, +- icon TEXT, +- created_at INTEGER NOT NULL DEFAULT (unixepoch()) ++ id INTEGER PRIMARY KEY AUTOINCREMENT, ++ name TEXT NOT NULL, ++ description TEXT, ++ colour TEXT, ++ icon TEXT, ++ created_at INTEGER NOT NULL DEFAULT (unixepoch()), ++ isolated INTEGER NOT NULL DEFAULT 0, ++ notes TEXT, ++ system_prompt TEXT + ); + + CREATE TABLE IF NOT EXISTS summaries ( +@@ -72,17 +86,17 @@ + metadata TEXT + ); + +- CREATE INDEX IF NOT EXISTS idx_episodes_session +- ON episodes(session_id); +- CREATE INDEX IF NOT EXISTS idx_episodes_created +- ON episodes(created_at); +- CREATE INDEX IF NOT EXISTS idx_entities_type +- ON entities(type); ++ CREATE INDEX IF NOT EXISTS idx_episodes_session ON episodes(session_id); ++ CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at); ++ CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type); ++ CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id); ++ CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id); ++ CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id); + +- CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts ++ CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts + USING fts5(user_message, ai_response, content=episodes, content_rowid=id); + +- CREATE TRIGGER IF NOT EXISTS episodes_fts_insert ++ CREATE TRIGGER IF NOT EXISTS episodes_fts_insert + AFTER INSERT ON episodes BEGIN + INSERT INTO episodes_fts(rowid, user_message, ai_response) + VALUES (new.id, new.user_message, new.ai_response); +@@ -101,8 +115,6 @@ + INSERT INTO episodes_fts(rowid, user_message, ai_response) + VALUES (new.id, new.user_message, new.ai_response); + END; +- +- + `; + +-module.exports = schema; +\ No newline at end of file ++module.exports = schema; +diff -ruN nexusai-baseline/test/migrations.test.js nexusai/test/migrations.test.js +--- nexusai-baseline/test/migrations.test.js 1970-01-01 00:00:00.000000000 +0000 ++++ nexusai/test/migrations.test.js 2026-08-17 11:41:02.541923591 +0000 +@@ -0,0 +1,65 @@ ++// Migration runner — tests the REAL migrate() version-stepping logic against a ++// minimal fake db (emulating the better-sqlite3 pragma getter/setter + transaction ++// API), so it runs with zero dependencies. Guards that migrations apply in order, ++// advance user_version correctly, and no-op when the DB is already current. ++const { test } = require('node:test'); ++const assert = require('node:assert'); ++const { migrate, LATEST_VERSION } = require('../packages/memory-service/src/db/migrations'); ++ ++// Emulates just the slice of the better-sqlite3 API that migrate() uses. ++function fakeDb(startVersion = 0) { ++ let version = startVersion; ++ return { ++ ran: [], ++ setCalls: [], ++ pragma(str, opts) { ++ const set = str.match(/^user_version\s*=\s*(\d+)$/); ++ if (set) { version = Number(set[1]); this.setCalls.push(version); return; } ++ if (str === 'user_version') return opts && opts.simple ? version : [{ user_version: version }]; ++ throw new Error('unexpected pragma: ' + str); ++ }, ++ transaction(fn) { return (...args) => fn(...args); }, // execute immediately ++ get version() { return version; }, ++ }; ++} ++ ++test('a fresh DB (user_version 0) is stamped to LATEST_VERSION', () => { ++ const db = fakeDb(0); ++ const result = migrate(db); ++ assert.strictEqual(result, LATEST_VERSION); ++ assert.strictEqual(db.version, LATEST_VERSION); ++}); ++ ++test('an already-current DB is a no-op (no version writes)', () => { ++ const db = fakeDb(LATEST_VERSION); ++ const result = migrate(db); ++ assert.strictEqual(result, LATEST_VERSION); ++ assert.deepStrictEqual(db.setCalls, [], 'should not touch user_version when current'); ++}); ++ ++test('injected migrations run in order, each bumping the version by one', () => { ++ const db = fakeDb(0); ++ const order = []; ++ const migs = [ ++ () => order.push('v1'), ++ () => order.push('v2'), ++ () => order.push('v3'), ++ ]; ++ const result = migrate(db, migs); ++ assert.strictEqual(result, 3); ++ assert.deepStrictEqual(order, ['v1', 'v2', 'v3'], 'migrations apply in sequence'); ++ assert.deepStrictEqual(db.setCalls, [1, 2, 3], 'user_version advances one step at a time'); ++}); ++ ++test('only migrations newer than the current version run', () => { ++ const db = fakeDb(1); // already at v1 ++ const order = []; ++ const migs = [ ++ () => order.push('v1'), // should be skipped ++ () => order.push('v2'), ++ () => order.push('v3'), ++ ]; ++ migrate(db, migs); ++ assert.deepStrictEqual(order, ['v2', 'v3'], 'v1 is not re-run'); ++ assert.deepStrictEqual(db.setCalls, [2, 3]); ++}); +diff -ruN nexusai-baseline/test/schema.test.js nexusai/test/schema.test.js +--- nexusai-baseline/test/schema.test.js 1970-01-01 00:00:00.000000000 +0000 ++++ nexusai/test/schema.test.js 2026-08-17 11:40:49.545923888 +0000 +@@ -0,0 +1,58 @@ ++// Schema completeness — execs the REAL schema.js into a throwaway SQLite DB (via ++// built-in node:sqlite) and asserts a fresh database gets every column the old ++// per-boot ALTER statements used to add. This is the guard for the migration ++// consolidation: if someone drops a column from schema.js, a fresh install would ++// silently lose it, and this test goes red. ++const { test, before } = require('node:test'); ++const assert = require('node:assert'); ++const schema = require('../packages/memory-service/src/db/schema'); ++ ++let DatabaseSync; ++try { ({ DatabaseSync } = require('node:sqlite')); } catch { DatabaseSync = null; } ++ ++// The complete intended shape = original base tables + every historical ALTER. ++const EXPECTED = { ++ sessions: ['id','external_id','created_at','updated_at','metadata','name','project_id'], ++ episodes: ['id','session_id','user_message','ai_response','created_at','token_count','metadata'], ++ entities: ['id','name','type','notes','created_at','updated_at','metadata','mention_count','confidence','source','last_seen_at'], ++ relationships: ['id','from_id','to_id','label','created_at','metadata','mention_count','notes'], ++ entity_episodes: ['entity_id','episode_id'], ++ projects: ['id','name','description','colour','icon','created_at','isolated','notes','system_prompt'], ++ summaries: ['id','session_id','project_id','content','token_count','episode_range','created_at','metadata'], ++}; ++ ++const EXPECTED_OBJECTS = [ ++ 'idx_relationships_from','idx_relationships_to','idx_entity_episodes_entity','idx_entity_episodes_episode', ++ 'idx_episodes_session','idx_episodes_created','idx_entities_type','idx_sessions_project', ++ 'idx_summaries_project','idx_summaries_session', ++ 'episodes_fts','episodes_fts_insert','episodes_fts_delete','episodes_fts_update', ++]; ++ ++let db; ++before(() => { ++ if (!DatabaseSync) return; ++ db = new DatabaseSync(':memory:'); ++ db.exec('PRAGMA foreign_keys = ON'); ++ db.exec(schema); // throws here if schema.js has a syntax/ordering error ++}); ++ ++for (const [table, cols] of Object.entries(EXPECTED)) { ++ test(`${table} has exactly its full column set`, { skip: !DatabaseSync }, () => { ++ const got = db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name); ++ assert.deepStrictEqual([...got].sort(), [...cols].sort(), `${table} columns differ from the intended shape`); ++ }); ++} ++ ++test('all indexes, FTS table, and triggers are present', { skip: !DatabaseSync }, () => { ++ const names = db.prepare(`SELECT name FROM sqlite_master WHERE name LIKE 'idx_%' OR name LIKE 'episodes_fts%'`) ++ .all().map(r => r.name); ++ const missing = EXPECTED_OBJECTS.filter(o => !names.includes(o)); ++ assert.deepStrictEqual(missing, [], `missing schema objects: ${missing}`); ++}); ++ ++test('FTS trigger populates the index on episode insert', { skip: !DatabaseSync }, () => { ++ db.prepare(`INSERT INTO sessions(external_id) VALUES ('s-test')`).run(); ++ db.prepare(`INSERT INTO episodes(session_id, user_message, ai_response) VALUES (1,'find me qdrant','ok')`).run(); ++ const hit = db.prepare(`SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH 'qdrant'`).all(); ++ assert.strictEqual(hit.length, 1); ++}); diff --git a/packages/memory-service/src/db/index.js b/packages/memory-service/src/db/index.js index 20ac118..68b497d 100644 --- a/packages/memory-service/src/db/index.js +++ b/packages/memory-service/src/db/index.js @@ -1,6 +1,7 @@ const Database = require('better-sqlite3'); const schema = require('./schema'); -const {getEnv, SQLITE, logger } = require('@nexusai/shared'); +const { migrate } = require('./migrations'); +const { getEnv, SQLITE, logger } = require('@nexusai/shared'); let db; // Declare db variable in a scope accessible to all functions @@ -12,60 +13,29 @@ function getDB() { db.pragma('journal_mode = WAL'); db.pragma('foreign_keys = ON'); - db.exec(schema); + // Was the FTS index absent before this boot? (True for a fresh DB, and for + // an older DB from before FTS existed.) Checked BEFORE schema runs so we can + // decide whether a one-time backfill is needed below. + const ftsExisted = db.prepare( + `SELECT 1 FROM sqlite_master WHERE type='table' AND name='episodes_fts'` + ).get() !== undefined; - try{ - db.exec(`ALTER TABLE sessions ADD COLUMN name TEXT`) - } catch {} + db.exec(schema); // complete current shape — fresh DBs get everything + migrate(db); // carry an older DB forward; no-op on fresh/current DBs - try { - db.exec(`ALTER TABLE sessions ADD COLUMN project_id INTEGER REFERENCES projects(id)`); - } catch {} - - try { - db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id)`); - } catch {} - - try { - db.exec(`ALTER TABLE projects ADD COLUMN isolated INTEGER NOT NULL DEFAULT 0`); - } catch {} - - try { - db.exec(`ALTER TABLE projects ADD COLUMN notes TEXT`); // ← add this - } catch {} - - try { - db.exec(`ALTER TABLE projects ADD COLUMN system_prompt TEXT`); - } catch {} - - try { - db.exec(`ALTER TABLE summaries ADD COLUMN project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE`); - } catch {} - - try { - db.exec(`ALTER TABLE summaries ADD COLUMN token_count INTEGER`); - } catch {} - - try { - db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id)`); - } catch {} - - try { - db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id)`); - } catch {} - - try { db.exec(`ALTER TABLE entities ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {} - try { db.exec(`ALTER TABLE entities ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0`) } catch {} - try { db.exec(`ALTER TABLE entities ADD COLUMN source TEXT NOT NULL DEFAULT 'extraction'`) } catch {} - try { db.exec(`ALTER TABLE entities ADD COLUMN last_seen_at INTEGER`) } catch {} - - try { db.exec(`ALTER TABLE relationships ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {} - try { db.exec(`ALTER TABLE relationships ADD COLUMN notes TEXT`) } catch {} - - - // Sync FTS index with any existing episodes data - db.exec(`INSERT OR REPLACE INTO episodes_fts(rowid, user_message, ai_response) - SELECT id, user_message, ai_response FROM episodes`); + // One-time FTS backfill: only when the index was just created on a DB that + // already holds episodes (i.e. episodes predate FTS). During normal + // operation the insert/delete/update triggers keep it in sync, so this no + // longer rebuilds the whole index on every boot. NOTE: COUNT(*) on an + // external-content FTS5 table proxies the content table, so it can't detect + // a desync — the "was it just created" check is what makes this correct. + if (!ftsExisted) { + const epCount = db.prepare('SELECT COUNT(*) AS c FROM episodes').get().c; + if (epCount > 0) { + db.exec(`INSERT INTO episodes_fts(episodes_fts) VALUES('rebuild')`); + logger.info(`[db] Backfilled FTS index for ${epCount} pre-existing episodes`); + } + } logger.info(`Connected to SQLite database at ${path}`); } @@ -74,4 +44,4 @@ function getDB() { module.exports = { getDB -}; \ No newline at end of file +}; diff --git a/packages/memory-service/src/db/migrations.js b/packages/memory-service/src/db/migrations.js new file mode 100644 index 0000000..1fbe97c --- /dev/null +++ b/packages/memory-service/src/db/migrations.js @@ -0,0 +1,43 @@ +const { logger } = require('@nexusai/shared'); + +// Forward-only schema migrations. Entry i takes the database from user_version i +// to i+1, so migrations[0] is the v0→v1 step, migrations[1] the v1→v2 step, etc. +// +// schema.js already holds the COMPLETE current shape, so a fresh database is +// created whole and stamped straight to LATEST_VERSION — these run only to carry +// an OLDER database forward. Add a new schema change by appending a function here +// (which bumps LATEST_VERSION by one); never edit an existing entry once shipped, +// since databases already stamped past it will not re-run it. +// +// Each migration receives the better-sqlite3 db handle and runs inside a +// transaction together with its version bump, so a failure rolls back cleanly. +const migrations = [ + // v0 → v1: baseline. The historical ALTER TABLE / CREATE INDEX statements that + // used to run (wrapped in try/catch) on every boot are folded into schema.js. + // Nothing to do here — this entry exists to mark v1 as the consolidated baseline. + (_db) => {}, +]; + +const LATEST_VERSION = migrations.length; + +// Applies any migrations newer than the database's current user_version, one at a +// time, each with its version bump, inside a transaction. Returns the resulting +// version. A no-op when the database is already current. The migrations list and +// target version are injectable for testing; production callers pass just `db`. +function migrate(db, migs = migrations, latest = migs.length) { + const current = db.pragma('user_version', { simple: true }); + if (current >= latest) return current; + + for (let v = current; v < latest; v++) { + const step = db.transaction(() => { + migs[v](db); + db.pragma(`user_version = ${v + 1}`); + }); + step(); + } + + logger.info(`[db] schema migrated ${current} -> ${latest}`); + return latest; +} + +module.exports = { migrate, migrations, LATEST_VERSION }; diff --git a/packages/memory-service/src/db/schema.js b/packages/memory-service/src/db/schema.js index e96e688..44973a2 100644 --- a/packages/memory-service/src/db/schema.js +++ b/packages/memory-service/src/db/schema.js @@ -1,10 +1,17 @@ +// Complete current schema — the single source of truth for a fresh database. +// Historical ALTER TABLE statements that used to run on every boot are folded in +// here. Any change that must reach EXISTING databases goes in migrations.js as a +// new numbered migration, NOT by editing a table below (CREATE ... IF NOT EXISTS +// silently skips tables that already exist, so column edits here never reach them). const schema = ` CREATE TABLE IF NOT EXISTS sessions ( id INTEGER PRIMARY KEY AUTOINCREMENT, external_id TEXT UNIQUE NOT NULL, created_at INTEGER NOT NULL DEFAULT (unixepoch()), updated_at INTEGER NOT NULL DEFAULT (unixepoch()), - metadata TEXT + metadata TEXT, + name TEXT, + project_id INTEGER REFERENCES projects(id) ); CREATE TABLE IF NOT EXISTS episodes ( @@ -18,23 +25,29 @@ const schema = ` ); CREATE TABLE IF NOT EXISTS entities ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - type TEXT NOT NULL, - notes TEXT, - created_at INTEGER NOT NULL DEFAULT (unixepoch()), - updated_at INTEGER NOT NULL DEFAULT (unixepoch()), - metadata TEXT, + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + type TEXT NOT NULL, + notes TEXT, + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + updated_at INTEGER NOT NULL DEFAULT (unixepoch()), + metadata TEXT, + mention_count INTEGER NOT NULL DEFAULT 1, + confidence REAL NOT NULL DEFAULT 1.0, + source TEXT NOT NULL DEFAULT 'extraction', + last_seen_at INTEGER, UNIQUE(name, type) ); CREATE TABLE IF NOT EXISTS relationships ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, - to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, - label TEXT NOT NULL, - created_at INTEGER NOT NULL DEFAULT (unixepoch()), - metadata TEXT, + id INTEGER PRIMARY KEY AUTOINCREMENT, + from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + label TEXT NOT NULL, + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + metadata TEXT, + mention_count INTEGER NOT NULL DEFAULT 1, + notes TEXT, UNIQUE(from_id, to_id, label) ); @@ -49,16 +62,17 @@ const schema = ` CREATE INDEX IF NOT EXISTS idx_entity_episodes_entity ON entity_episodes(entity_id); CREATE INDEX IF NOT EXISTS idx_entity_episodes_episode ON entity_episodes(episode_id); - - CREATE TABLE IF NOT EXISTS projects ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - description TEXT, - colour TEXT, - icon TEXT, - created_at INTEGER NOT NULL DEFAULT (unixepoch()) + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT, + colour TEXT, + icon TEXT, + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + isolated INTEGER NOT NULL DEFAULT 0, + notes TEXT, + system_prompt TEXT ); CREATE TABLE IF NOT EXISTS summaries ( @@ -72,17 +86,17 @@ const schema = ` metadata TEXT ); - CREATE INDEX IF NOT EXISTS idx_episodes_session - ON episodes(session_id); - CREATE INDEX IF NOT EXISTS idx_episodes_created - ON episodes(created_at); - CREATE INDEX IF NOT EXISTS idx_entities_type - ON entities(type); + CREATE INDEX IF NOT EXISTS idx_episodes_session ON episodes(session_id); + CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at); + CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type); + CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id); + CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id); + CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id); - CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts + CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts USING fts5(user_message, ai_response, content=episodes, content_rowid=id); - CREATE TRIGGER IF NOT EXISTS episodes_fts_insert + CREATE TRIGGER IF NOT EXISTS episodes_fts_insert AFTER INSERT ON episodes BEGIN INSERT INTO episodes_fts(rowid, user_message, ai_response) VALUES (new.id, new.user_message, new.ai_response); @@ -101,8 +115,6 @@ const schema = ` INSERT INTO episodes_fts(rowid, user_message, ai_response) VALUES (new.id, new.user_message, new.ai_response); END; - - `; -module.exports = schema; \ No newline at end of file +module.exports = schema; diff --git a/test/migrations.test.js b/test/migrations.test.js new file mode 100644 index 0000000..653d878 --- /dev/null +++ b/test/migrations.test.js @@ -0,0 +1,65 @@ +// Migration runner — tests the REAL migrate() version-stepping logic against a +// minimal fake db (emulating the better-sqlite3 pragma getter/setter + transaction +// API), so it runs with zero dependencies. Guards that migrations apply in order, +// advance user_version correctly, and no-op when the DB is already current. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { migrate, LATEST_VERSION } = require('../packages/memory-service/src/db/migrations'); + +// Emulates just the slice of the better-sqlite3 API that migrate() uses. +function fakeDb(startVersion = 0) { + let version = startVersion; + return { + ran: [], + setCalls: [], + pragma(str, opts) { + const set = str.match(/^user_version\s*=\s*(\d+)$/); + if (set) { version = Number(set[1]); this.setCalls.push(version); return; } + if (str === 'user_version') return opts && opts.simple ? version : [{ user_version: version }]; + throw new Error('unexpected pragma: ' + str); + }, + transaction(fn) { return (...args) => fn(...args); }, // execute immediately + get version() { return version; }, + }; +} + +test('a fresh DB (user_version 0) is stamped to LATEST_VERSION', () => { + const db = fakeDb(0); + const result = migrate(db); + assert.strictEqual(result, LATEST_VERSION); + assert.strictEqual(db.version, LATEST_VERSION); +}); + +test('an already-current DB is a no-op (no version writes)', () => { + const db = fakeDb(LATEST_VERSION); + const result = migrate(db); + assert.strictEqual(result, LATEST_VERSION); + assert.deepStrictEqual(db.setCalls, [], 'should not touch user_version when current'); +}); + +test('injected migrations run in order, each bumping the version by one', () => { + const db = fakeDb(0); + const order = []; + const migs = [ + () => order.push('v1'), + () => order.push('v2'), + () => order.push('v3'), + ]; + const result = migrate(db, migs); + assert.strictEqual(result, 3); + assert.deepStrictEqual(order, ['v1', 'v2', 'v3'], 'migrations apply in sequence'); + assert.deepStrictEqual(db.setCalls, [1, 2, 3], 'user_version advances one step at a time'); +}); + +test('only migrations newer than the current version run', () => { + const db = fakeDb(1); // already at v1 + const order = []; + const migs = [ + () => order.push('v1'), // should be skipped + () => order.push('v2'), + () => order.push('v3'), + ]; + migrate(db, migs); + assert.deepStrictEqual(order, ['v2', 'v3'], 'v1 is not re-run'); + assert.deepStrictEqual(db.setCalls, [2, 3]); +}); diff --git a/test/schema.test.js b/test/schema.test.js new file mode 100644 index 0000000..79744e7 --- /dev/null +++ b/test/schema.test.js @@ -0,0 +1,58 @@ +// Schema completeness — execs the REAL schema.js into a throwaway SQLite DB (via +// built-in node:sqlite) and asserts a fresh database gets every column the old +// per-boot ALTER statements used to add. This is the guard for the migration +// consolidation: if someone drops a column from schema.js, a fresh install would +// silently lose it, and this test goes red. +const { test, before } = require('node:test'); +const assert = require('node:assert'); +const schema = require('../packages/memory-service/src/db/schema'); + +let DatabaseSync; +try { ({ DatabaseSync } = require('node:sqlite')); } catch { DatabaseSync = null; } + +// The complete intended shape = original base tables + every historical ALTER. +const EXPECTED = { + sessions: ['id','external_id','created_at','updated_at','metadata','name','project_id'], + episodes: ['id','session_id','user_message','ai_response','created_at','token_count','metadata'], + entities: ['id','name','type','notes','created_at','updated_at','metadata','mention_count','confidence','source','last_seen_at'], + relationships: ['id','from_id','to_id','label','created_at','metadata','mention_count','notes'], + entity_episodes: ['entity_id','episode_id'], + projects: ['id','name','description','colour','icon','created_at','isolated','notes','system_prompt'], + summaries: ['id','session_id','project_id','content','token_count','episode_range','created_at','metadata'], +}; + +const EXPECTED_OBJECTS = [ + 'idx_relationships_from','idx_relationships_to','idx_entity_episodes_entity','idx_entity_episodes_episode', + 'idx_episodes_session','idx_episodes_created','idx_entities_type','idx_sessions_project', + 'idx_summaries_project','idx_summaries_session', + 'episodes_fts','episodes_fts_insert','episodes_fts_delete','episodes_fts_update', +]; + +let db; +before(() => { + if (!DatabaseSync) return; + db = new DatabaseSync(':memory:'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec(schema); // throws here if schema.js has a syntax/ordering error +}); + +for (const [table, cols] of Object.entries(EXPECTED)) { + test(`${table} has exactly its full column set`, { skip: !DatabaseSync }, () => { + const got = db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name); + assert.deepStrictEqual([...got].sort(), [...cols].sort(), `${table} columns differ from the intended shape`); + }); +} + +test('all indexes, FTS table, and triggers are present', { skip: !DatabaseSync }, () => { + const names = db.prepare(`SELECT name FROM sqlite_master WHERE name LIKE 'idx_%' OR name LIKE 'episodes_fts%'`) + .all().map(r => r.name); + const missing = EXPECTED_OBJECTS.filter(o => !names.includes(o)); + assert.deepStrictEqual(missing, [], `missing schema objects: ${missing}`); +}); + +test('FTS trigger populates the index on episode insert', { skip: !DatabaseSync }, () => { + db.prepare(`INSERT INTO sessions(external_id) VALUES ('s-test')`).run(); + db.prepare(`INSERT INTO episodes(session_id, user_message, ai_response) VALUES (1,'find me qdrant','ok')`).run(); + const hit = db.prepare(`SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH 'qdrant'`).all(); + assert.strictEqual(hit.length, 1); +});