migration consolidation patch

2026-08-17 04:53:12 -07:00
parent e570bbecf2
commit 0353ce96e0
6 changed files with 644 additions and 87 deletions
@@ -0,0 +1,409 @@
+diff -ruN nexusai-baseline/packages/memory-service/src/db/index.js nexusai/packages/memory-service/src/db/index.js
+--- nexusai-baseline/packages/memory-service/src/db/index.js	2026-08-17 10:23:40.095197576 +0000
+++ nexusai/packages/memory-service/src/db/index.js	2026-08-17 11:39:45.973925342 +0000
+@@ -1,6 +1,7 @@
+ const Database = require('better-sqlite3');
+ const schema = require('./schema');
+-const {getEnv, SQLITE, logger } = require('@nexusai/shared');
+const { migrate } = require('./migrations');
+const { getEnv, SQLITE, logger } = require('@nexusai/shared');
+ 
+ let db;  // Declare db variable in a scope accessible to all functions
+ 
+@@ -12,60 +13,29 @@
+         db.pragma('journal_mode = WAL');
+         db.pragma('foreign_keys = ON');
+ 
+-        db.exec(schema);
+-
+-        try{
+-            db.exec(`ALTER TABLE sessions ADD COLUMN name TEXT`)
+-        } catch {}
+-
+-        try {
+-            db.exec(`ALTER TABLE sessions ADD COLUMN project_id INTEGER REFERENCES projects(id)`);
+-        } catch {}
+-
+-        try {
+-            db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id)`);
+-        } catch {}  
+-
+-        try {
+-            db.exec(`ALTER TABLE projects ADD COLUMN isolated INTEGER NOT NULL DEFAULT 0`);
+-        } catch {}
+-        
+-        try {
+-            db.exec(`ALTER TABLE projects ADD COLUMN notes TEXT`);  // ← add this
+-        } catch {}
+-
+-        try { 
+-            db.exec(`ALTER TABLE projects ADD COLUMN system_prompt TEXT`); 
+-        } catch {}
+-
+-        try {
+-            db.exec(`ALTER TABLE summaries ADD COLUMN project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE`);
+-        } catch {}
+-
+-        try {
+-            db.exec(`ALTER TABLE summaries ADD COLUMN token_count INTEGER`);
+-        } catch {}
+-
+-        try {
+-            db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id)`);
+-        } catch {}
+-
+-        try {
+-            db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id)`);
+-        } catch {}
+-
+-        try { db.exec(`ALTER TABLE entities ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {}
+-        try { db.exec(`ALTER TABLE entities ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0`) } catch {}
+-        try { db.exec(`ALTER TABLE entities ADD COLUMN source TEXT NOT NULL DEFAULT 'extraction'`) } catch {}
+-        try { db.exec(`ALTER TABLE entities ADD COLUMN last_seen_at INTEGER`) } catch {}
+-
+-        try { db.exec(`ALTER TABLE relationships ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {}
+-        try { db.exec(`ALTER TABLE relationships ADD COLUMN notes TEXT`) } catch {}
+-
+-        
+-        // Sync FTS index with any existing episodes data
+-        db.exec(`INSERT OR REPLACE INTO episodes_fts(rowid, user_message, ai_response) 
+-            SELECT id, user_message, ai_response FROM episodes`);
+        // Was the FTS index absent before this boot? (True for a fresh DB, and for
+        // an older DB from before FTS existed.) Checked BEFORE schema runs so we can
+        // decide whether a one-time backfill is needed below.
+        const ftsExisted = db.prepare(
+            `SELECT 1 FROM sqlite_master WHERE type='table' AND name='episodes_fts'`
+        ).get() !== undefined;
+
+        db.exec(schema);   // complete current shape — fresh DBs get everything
+        migrate(db);       // carry an older DB forward; no-op on fresh/current DBs
+
+        // One-time FTS backfill: only when the index was just created on a DB that
+        // already holds episodes (i.e. episodes predate FTS). During normal
+        // operation the insert/delete/update triggers keep it in sync, so this no
+        // longer rebuilds the whole index on every boot. NOTE: COUNT(*) on an
+        // external-content FTS5 table proxies the content table, so it can't detect
+        // a desync — the "was it just created" check is what makes this correct.
+        if (!ftsExisted) {
+            const epCount = db.prepare('SELECT COUNT(*) AS c FROM episodes').get().c;
+            if (epCount > 0) {
+                db.exec(`INSERT INTO episodes_fts(episodes_fts) VALUES('rebuild')`);
+                logger.info(`[db] Backfilled FTS index for ${epCount} pre-existing episodes`);
+            }
+        }
+ 
+         logger.info(`Connected to SQLite database at ${path}`);
+     }
+@@ -74,4 +44,4 @@
+ 
+ module.exports = {
+     getDB
+-};
+\ No newline at end of file
+};
+diff -ruN nexusai-baseline/packages/memory-service/src/db/migrations.js nexusai/packages/memory-service/src/db/migrations.js
+--- nexusai-baseline/packages/memory-service/src/db/migrations.js	1970-01-01 00:00:00.000000000 +0000
+++ nexusai/packages/memory-service/src/db/migrations.js	2026-08-17 11:40:33.900163765 +0000
+@@ -0,0 +1,43 @@
+const { logger } = require('@nexusai/shared');
+
+// Forward-only schema migrations. Entry i takes the database from user_version i
+// to i+1, so migrations[0] is the v0→v1 step, migrations[1] the v1→v2 step, etc.
+//
+// schema.js already holds the COMPLETE current shape, so a fresh database is
+// created whole and stamped straight to LATEST_VERSION — these run only to carry
+// an OLDER database forward. Add a new schema change by appending a function here
+// (which bumps LATEST_VERSION by one); never edit an existing entry once shipped,
+// since databases already stamped past it will not re-run it.
+//
+// Each migration receives the better-sqlite3 db handle and runs inside a
+// transaction together with its version bump, so a failure rolls back cleanly.
+const migrations = [
+  // v0 → v1: baseline. The historical ALTER TABLE / CREATE INDEX statements that
+  // used to run (wrapped in try/catch) on every boot are folded into schema.js.
+  // Nothing to do here — this entry exists to mark v1 as the consolidated baseline.
+  (_db) => {},
+];
+
+const LATEST_VERSION = migrations.length;
+
+// Applies any migrations newer than the database's current user_version, one at a
+// time, each with its version bump, inside a transaction. Returns the resulting
+// version. A no-op when the database is already current. The migrations list and
+// target version are injectable for testing; production callers pass just `db`.
+function migrate(db, migs = migrations, latest = migs.length) {
+  const current = db.pragma('user_version', { simple: true });
+  if (current >= latest) return current;
+
+  for (let v = current; v < latest; v++) {
+    const step = db.transaction(() => {
+      migs[v](db);
+      db.pragma(`user_version = ${v + 1}`);
+    });
+    step();
+  }
+
+  logger.info(`[db] schema migrated ${current} -> ${latest}`);
+  return latest;
+}
+
+module.exports = { migrate, migrations, LATEST_VERSION };
+diff -ruN nexusai-baseline/packages/memory-service/src/db/schema.js nexusai/packages/memory-service/src/db/schema.js
+--- nexusai-baseline/packages/memory-service/src/db/schema.js	2026-08-17 10:23:40.095507736 +0000
+++ nexusai/packages/memory-service/src/db/schema.js	2026-08-17 11:39:21.421925904 +0000
+@@ -1,10 +1,17 @@
+// Complete current schema — the single source of truth for a fresh database.
+// Historical ALTER TABLE statements that used to run on every boot are folded in
+// here. Any change that must reach EXISTING databases goes in migrations.js as a
+// new numbered migration, NOT by editing a table below (CREATE ... IF NOT EXISTS
+// silently skips tables that already exist, so column edits here never reach them).
+ const schema = `
+   CREATE TABLE IF NOT EXISTS sessions (
+     id          INTEGER PRIMARY KEY AUTOINCREMENT,
+     external_id TEXT    UNIQUE NOT NULL,
+     created_at  INTEGER NOT NULL DEFAULT (unixepoch()),
+     updated_at  INTEGER NOT NULL DEFAULT (unixepoch()),
+-    metadata    TEXT
+    metadata    TEXT,
+    name        TEXT,
+    project_id  INTEGER REFERENCES projects(id)
+   );
+ 
+   CREATE TABLE IF NOT EXISTS episodes (
+@@ -18,23 +25,29 @@
+   );
+ 
+   CREATE TABLE IF NOT EXISTS entities (
+-    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+-    name       TEXT    NOT NULL,
+-    type       TEXT    NOT NULL,
+-    notes      TEXT,
+-    created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+-    updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
+-    metadata   TEXT,
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    name          TEXT    NOT NULL,
+    type          TEXT    NOT NULL,
+    notes         TEXT,
+    created_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    updated_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    metadata      TEXT,
+    mention_count INTEGER NOT NULL DEFAULT 1,
+    confidence    REAL    NOT NULL DEFAULT 1.0,
+    source        TEXT    NOT NULL DEFAULT 'extraction',
+    last_seen_at  INTEGER,
+     UNIQUE(name, type)
+   );
+ 
+   CREATE TABLE IF NOT EXISTS relationships (
+-    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+-    from_id    INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+-    to_id      INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+-    label      TEXT    NOT NULL,
+-    created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+-    metadata   TEXT,
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    from_id       INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+    to_id         INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+    label         TEXT    NOT NULL,
+    created_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    metadata      TEXT,
+    mention_count INTEGER NOT NULL DEFAULT 1,
+    notes         TEXT,
+     UNIQUE(from_id, to_id, label)
+   );
+ 
+@@ -49,16 +62,17 @@
+ 
+   CREATE INDEX IF NOT EXISTS idx_entity_episodes_entity  ON entity_episodes(entity_id);
+   CREATE INDEX IF NOT EXISTS idx_entity_episodes_episode ON entity_episodes(episode_id);
+-  
+-
+ 
+   CREATE TABLE IF NOT EXISTS projects (
+-    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+-    name        TEXT NOT NULL,
+-    description TEXT,
+-    colour      TEXT,
+-    icon        TEXT,
+-    created_at  INTEGER NOT NULL DEFAULT (unixepoch())
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    name          TEXT NOT NULL,
+    description   TEXT,
+    colour        TEXT,
+    icon          TEXT,
+    created_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    isolated      INTEGER NOT NULL DEFAULT 0,
+    notes         TEXT,
+    system_prompt TEXT
+   );
+ 
+   CREATE TABLE IF NOT EXISTS summaries (
+@@ -72,17 +86,17 @@
+     metadata      TEXT
+   );
+ 
+-  CREATE INDEX IF NOT EXISTS idx_episodes_session 
+-    ON episodes(session_id);
+-  CREATE INDEX IF NOT EXISTS idx_episodes_created 
+-    ON episodes(created_at);
+-  CREATE INDEX IF NOT EXISTS idx_entities_type 
+-    ON entities(type);
+  CREATE INDEX IF NOT EXISTS idx_episodes_session   ON episodes(session_id);
+  CREATE INDEX IF NOT EXISTS idx_episodes_created   ON episodes(created_at);
+  CREATE INDEX IF NOT EXISTS idx_entities_type      ON entities(type);
+  CREATE INDEX IF NOT EXISTS idx_sessions_project   ON sessions(project_id);
+  CREATE INDEX IF NOT EXISTS idx_summaries_project  ON summaries(project_id);
+  CREATE INDEX IF NOT EXISTS idx_summaries_session  ON summaries(session_id);
+ 
+-  CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts 
+  CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts
+     USING fts5(user_message, ai_response, content=episodes, content_rowid=id);
+ 
+-    CREATE TRIGGER IF NOT EXISTS episodes_fts_insert
+  CREATE TRIGGER IF NOT EXISTS episodes_fts_insert
+     AFTER INSERT ON episodes BEGIN
+       INSERT INTO episodes_fts(rowid, user_message, ai_response)
+       VALUES (new.id, new.user_message, new.ai_response);
+@@ -101,8 +115,6 @@
+       INSERT INTO episodes_fts(rowid, user_message, ai_response)
+       VALUES (new.id, new.user_message, new.ai_response);
+     END;
+-  
+-  
+ `;
+ 
+-module.exports = schema;
+\ No newline at end of file
+module.exports = schema;
+diff -ruN nexusai-baseline/test/migrations.test.js nexusai/test/migrations.test.js
+--- nexusai-baseline/test/migrations.test.js	1970-01-01 00:00:00.000000000 +0000
+++ nexusai/test/migrations.test.js	2026-08-17 11:41:02.541923591 +0000
+@@ -0,0 +1,65 @@
+// Migration runner — tests the REAL migrate() version-stepping logic against a
+// minimal fake db (emulating the better-sqlite3 pragma getter/setter + transaction
+// API), so it runs with zero dependencies. Guards that migrations apply in order,
+// advance user_version correctly, and no-op when the DB is already current.
+const { test } = require('node:test');
+const assert = require('node:assert');
+const { migrate, LATEST_VERSION } = require('../packages/memory-service/src/db/migrations');
+
+// Emulates just the slice of the better-sqlite3 API that migrate() uses.
+function fakeDb(startVersion = 0) {
+  let version = startVersion;
+  return {
+    ran: [],
+    setCalls: [],
+    pragma(str, opts) {
+      const set = str.match(/^user_version\s*=\s*(\d+)$/);
+      if (set) { version = Number(set[1]); this.setCalls.push(version); return; }
+      if (str === 'user_version') return opts && opts.simple ? version : [{ user_version: version }];
+      throw new Error('unexpected pragma: ' + str);
+    },
+    transaction(fn) { return (...args) => fn(...args); },  // execute immediately
+    get version() { return version; },
+  };
+}
+
+test('a fresh DB (user_version 0) is stamped to LATEST_VERSION', () => {
+  const db = fakeDb(0);
+  const result = migrate(db);
+  assert.strictEqual(result, LATEST_VERSION);
+  assert.strictEqual(db.version, LATEST_VERSION);
+});
+
+test('an already-current DB is a no-op (no version writes)', () => {
+  const db = fakeDb(LATEST_VERSION);
+  const result = migrate(db);
+  assert.strictEqual(result, LATEST_VERSION);
+  assert.deepStrictEqual(db.setCalls, [], 'should not touch user_version when current');
+});
+
+test('injected migrations run in order, each bumping the version by one', () => {
+  const db = fakeDb(0);
+  const order = [];
+  const migs = [
+    () => order.push('v1'),
+    () => order.push('v2'),
+    () => order.push('v3'),
+  ];
+  const result = migrate(db, migs);
+  assert.strictEqual(result, 3);
+  assert.deepStrictEqual(order, ['v1', 'v2', 'v3'], 'migrations apply in sequence');
+  assert.deepStrictEqual(db.setCalls, [1, 2, 3], 'user_version advances one step at a time');
+});
+
+test('only migrations newer than the current version run', () => {
+  const db = fakeDb(1);   // already at v1
+  const order = [];
+  const migs = [
+    () => order.push('v1'),   // should be skipped
+    () => order.push('v2'),
+    () => order.push('v3'),
+  ];
+  migrate(db, migs);
+  assert.deepStrictEqual(order, ['v2', 'v3'], 'v1 is not re-run');
+  assert.deepStrictEqual(db.setCalls, [2, 3]);
+});
+diff -ruN nexusai-baseline/test/schema.test.js nexusai/test/schema.test.js
+--- nexusai-baseline/test/schema.test.js	1970-01-01 00:00:00.000000000 +0000
+++ nexusai/test/schema.test.js	2026-08-17 11:40:49.545923888 +0000
+@@ -0,0 +1,58 @@
+// Schema completeness — execs the REAL schema.js into a throwaway SQLite DB (via
+// built-in node:sqlite) and asserts a fresh database gets every column the old
+// per-boot ALTER statements used to add. This is the guard for the migration
+// consolidation: if someone drops a column from schema.js, a fresh install would
+// silently lose it, and this test goes red.
+const { test, before } = require('node:test');
+const assert = require('node:assert');
+const schema = require('../packages/memory-service/src/db/schema');
+
+let DatabaseSync;
+try { ({ DatabaseSync } = require('node:sqlite')); } catch { DatabaseSync = null; }
+
+// The complete intended shape = original base tables + every historical ALTER.
+const EXPECTED = {
+  sessions:        ['id','external_id','created_at','updated_at','metadata','name','project_id'],
+  episodes:        ['id','session_id','user_message','ai_response','created_at','token_count','metadata'],
+  entities:        ['id','name','type','notes','created_at','updated_at','metadata','mention_count','confidence','source','last_seen_at'],
+  relationships:   ['id','from_id','to_id','label','created_at','metadata','mention_count','notes'],
+  entity_episodes: ['entity_id','episode_id'],
+  projects:        ['id','name','description','colour','icon','created_at','isolated','notes','system_prompt'],
+  summaries:       ['id','session_id','project_id','content','token_count','episode_range','created_at','metadata'],
+};
+
+const EXPECTED_OBJECTS = [
+  'idx_relationships_from','idx_relationships_to','idx_entity_episodes_entity','idx_entity_episodes_episode',
+  'idx_episodes_session','idx_episodes_created','idx_entities_type','idx_sessions_project',
+  'idx_summaries_project','idx_summaries_session',
+  'episodes_fts','episodes_fts_insert','episodes_fts_delete','episodes_fts_update',
+];
+
+let db;
+before(() => {
+  if (!DatabaseSync) return;
+  db = new DatabaseSync(':memory:');
+  db.exec('PRAGMA foreign_keys = ON');
+  db.exec(schema);   // throws here if schema.js has a syntax/ordering error
+});
+
+for (const [table, cols] of Object.entries(EXPECTED)) {
+  test(`${table} has exactly its full column set`, { skip: !DatabaseSync }, () => {
+    const got = db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name);
+    assert.deepStrictEqual([...got].sort(), [...cols].sort(), `${table} columns differ from the intended shape`);
+  });
+}
+
+test('all indexes, FTS table, and triggers are present', { skip: !DatabaseSync }, () => {
+  const names = db.prepare(`SELECT name FROM sqlite_master WHERE name LIKE 'idx_%' OR name LIKE 'episodes_fts%'`)
+    .all().map(r => r.name);
+  const missing = EXPECTED_OBJECTS.filter(o => !names.includes(o));
+  assert.deepStrictEqual(missing, [], `missing schema objects: ${missing}`);
+});
+
+test('FTS trigger populates the index on episode insert', { skip: !DatabaseSync }, () => {
+  db.prepare(`INSERT INTO sessions(external_id) VALUES ('s-test')`).run();
+  db.prepare(`INSERT INTO episodes(session_id, user_message, ai_response) VALUES (1,'find me qdrant','ok')`).run();
+  const hit = db.prepare(`SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH 'qdrant'`).all();
+  assert.strictEqual(hit.length, 1);
+});
@@ -1,6 +1,7 @@
 const Database = require('better-sqlite3');
 const schema = require('./schema');
-const {getEnv, SQLITE, logger } = require('@nexusai/shared');
+const { migrate } = require('./migrations');
+const { getEnv, SQLITE, logger } = require('@nexusai/shared');

 let db;  // Declare db variable in a scope accessible to all functions

@@ -12,60 +13,29 @@ function getDB() {
        db.pragma('journal_mode = WAL');
        db.pragma('foreign_keys = ON');

-        db.exec(schema);
+        // Was the FTS index absent before this boot? (True for a fresh DB, and for
+        // an older DB from before FTS existed.) Checked BEFORE schema runs so we can
+        // decide whether a one-time backfill is needed below.
+        const ftsExisted = db.prepare(
+            `SELECT 1 FROM sqlite_master WHERE type='table' AND name='episodes_fts'`
+        ).get() !== undefined;

-        try{
-            db.exec(`ALTER TABLE sessions ADD COLUMN name TEXT`)
-        } catch {}
+        db.exec(schema);   // complete current shape — fresh DBs get everything
+        migrate(db);       // carry an older DB forward; no-op on fresh/current DBs

-        try {
-            db.exec(`ALTER TABLE sessions ADD COLUMN project_id INTEGER REFERENCES projects(id)`);
-        } catch {}
-
-        try {
-            db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id)`);
-        } catch {}  
-
-        try {
-            db.exec(`ALTER TABLE projects ADD COLUMN isolated INTEGER NOT NULL DEFAULT 0`);
-        } catch {}
-        
-        try {
-            db.exec(`ALTER TABLE projects ADD COLUMN notes TEXT`);  // ← add this
-        } catch {}
-
-        try { 
-            db.exec(`ALTER TABLE projects ADD COLUMN system_prompt TEXT`); 
-        } catch {}
-
-        try {
-            db.exec(`ALTER TABLE summaries ADD COLUMN project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE`);
-        } catch {}
-
-        try {
-            db.exec(`ALTER TABLE summaries ADD COLUMN token_count INTEGER`);
-        } catch {}
-
-        try {
-            db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id)`);
-        } catch {}
-
-        try {
-            db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id)`);
-        } catch {}
-
-        try { db.exec(`ALTER TABLE entities ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {}
-        try { db.exec(`ALTER TABLE entities ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0`) } catch {}
-        try { db.exec(`ALTER TABLE entities ADD COLUMN source TEXT NOT NULL DEFAULT 'extraction'`) } catch {}
-        try { db.exec(`ALTER TABLE entities ADD COLUMN last_seen_at INTEGER`) } catch {}
-
-        try { db.exec(`ALTER TABLE relationships ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {}
-        try { db.exec(`ALTER TABLE relationships ADD COLUMN notes TEXT`) } catch {}
-
-        
-        // Sync FTS index with any existing episodes data
-        db.exec(`INSERT OR REPLACE INTO episodes_fts(rowid, user_message, ai_response) 
-            SELECT id, user_message, ai_response FROM episodes`);
+        // One-time FTS backfill: only when the index was just created on a DB that
+        // already holds episodes (i.e. episodes predate FTS). During normal
+        // operation the insert/delete/update triggers keep it in sync, so this no
+        // longer rebuilds the whole index on every boot. NOTE: COUNT(*) on an
+        // external-content FTS5 table proxies the content table, so it can't detect
+        // a desync — the "was it just created" check is what makes this correct.
+        if (!ftsExisted) {
+            const epCount = db.prepare('SELECT COUNT(*) AS c FROM episodes').get().c;
+            if (epCount > 0) {
+                db.exec(`INSERT INTO episodes_fts(episodes_fts) VALUES('rebuild')`);
+                logger.info(`[db] Backfilled FTS index for ${epCount} pre-existing episodes`);
+            }
+        }

        logger.info(`Connected to SQLite database at ${path}`);
    }
@@ -74,4 +44,4 @@ function getDB() {

 module.exports = {
    getDB
-};
+};
@@ -0,0 +1,43 @@
+const { logger } = require('@nexusai/shared');
+
+// Forward-only schema migrations. Entry i takes the database from user_version i
+// to i+1, so migrations[0] is the v0→v1 step, migrations[1] the v1→v2 step, etc.
+//
+// schema.js already holds the COMPLETE current shape, so a fresh database is
+// created whole and stamped straight to LATEST_VERSION — these run only to carry
+// an OLDER database forward. Add a new schema change by appending a function here
+// (which bumps LATEST_VERSION by one); never edit an existing entry once shipped,
+// since databases already stamped past it will not re-run it.
+//
+// Each migration receives the better-sqlite3 db handle and runs inside a
+// transaction together with its version bump, so a failure rolls back cleanly.
+const migrations = [
+  // v0 → v1: baseline. The historical ALTER TABLE / CREATE INDEX statements that
+  // used to run (wrapped in try/catch) on every boot are folded into schema.js.
+  // Nothing to do here — this entry exists to mark v1 as the consolidated baseline.
+  (_db) => {},
+];
+
+const LATEST_VERSION = migrations.length;
+
+// Applies any migrations newer than the database's current user_version, one at a
+// time, each with its version bump, inside a transaction. Returns the resulting
+// version. A no-op when the database is already current. The migrations list and
+// target version are injectable for testing; production callers pass just `db`.
+function migrate(db, migs = migrations, latest = migs.length) {
+  const current = db.pragma('user_version', { simple: true });
+  if (current >= latest) return current;
+
+  for (let v = current; v < latest; v++) {
+    const step = db.transaction(() => {
+      migs[v](db);
+      db.pragma(`user_version = ${v + 1}`);
+    });
+    step();
+  }
+
+  logger.info(`[db] schema migrated ${current} -> ${latest}`);
+  return latest;
+}
+
+module.exports = { migrate, migrations, LATEST_VERSION };
@@ -1,10 +1,17 @@
+// Complete current schema — the single source of truth for a fresh database.
+// Historical ALTER TABLE statements that used to run on every boot are folded in
+// here. Any change that must reach EXISTING databases goes in migrations.js as a
+// new numbered migration, NOT by editing a table below (CREATE ... IF NOT EXISTS
+// silently skips tables that already exist, so column edits here never reach them).
 const schema = `
  CREATE TABLE IF NOT EXISTS sessions (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    external_id TEXT    UNIQUE NOT NULL,
    created_at  INTEGER NOT NULL DEFAULT (unixepoch()),
    updated_at  INTEGER NOT NULL DEFAULT (unixepoch()),
-    metadata    TEXT
+    metadata    TEXT,
+    name        TEXT,
+    project_id  INTEGER REFERENCES projects(id)
  );

  CREATE TABLE IF NOT EXISTS episodes (
@@ -18,23 +25,29 @@ const schema = `
  );

  CREATE TABLE IF NOT EXISTS entities (
-    id         INTEGER PRIMARY KEY AUTOINCREMENT,
-    name       TEXT    NOT NULL,
-    type       TEXT    NOT NULL,
-    notes      TEXT,
-    created_at INTEGER NOT NULL DEFAULT (unixepoch()),
-    updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
-    metadata   TEXT,
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    name          TEXT    NOT NULL,
+    type          TEXT    NOT NULL,
+    notes         TEXT,
+    created_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    updated_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    metadata      TEXT,
+    mention_count INTEGER NOT NULL DEFAULT 1,
+    confidence    REAL    NOT NULL DEFAULT 1.0,
+    source        TEXT    NOT NULL DEFAULT 'extraction',
+    last_seen_at  INTEGER,
    UNIQUE(name, type)
  );

  CREATE TABLE IF NOT EXISTS relationships (
-    id         INTEGER PRIMARY KEY AUTOINCREMENT,
-    from_id    INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
-    to_id      INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
-    label      TEXT    NOT NULL,
-    created_at INTEGER NOT NULL DEFAULT (unixepoch()),
-    metadata   TEXT,
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    from_id       INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+    to_id         INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+    label         TEXT    NOT NULL,
+    created_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    metadata      TEXT,
+    mention_count INTEGER NOT NULL DEFAULT 1,
+    notes         TEXT,
    UNIQUE(from_id, to_id, label)
  );

@@ -49,16 +62,17 @@ const schema = `

  CREATE INDEX IF NOT EXISTS idx_entity_episodes_entity  ON entity_episodes(entity_id);
  CREATE INDEX IF NOT EXISTS idx_entity_episodes_episode ON entity_episodes(episode_id);
-  
-

  CREATE TABLE IF NOT EXISTS projects (
-    id          INTEGER PRIMARY KEY AUTOINCREMENT,
-    name        TEXT NOT NULL,
-    description TEXT,
-    colour      TEXT,
-    icon        TEXT,
-    created_at  INTEGER NOT NULL DEFAULT (unixepoch())
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    name          TEXT NOT NULL,
+    description   TEXT,
+    colour        TEXT,
+    icon          TEXT,
+    created_at    INTEGER NOT NULL DEFAULT (unixepoch()),
+    isolated      INTEGER NOT NULL DEFAULT 0,
+    notes         TEXT,
+    system_prompt TEXT
  );

  CREATE TABLE IF NOT EXISTS summaries (
@@ -72,17 +86,17 @@ const schema = `
    metadata      TEXT
  );

-  CREATE INDEX IF NOT EXISTS idx_episodes_session 
-    ON episodes(session_id);
-  CREATE INDEX IF NOT EXISTS idx_episodes_created 
-    ON episodes(created_at);
-  CREATE INDEX IF NOT EXISTS idx_entities_type 
-    ON entities(type);
+  CREATE INDEX IF NOT EXISTS idx_episodes_session   ON episodes(session_id);
+  CREATE INDEX IF NOT EXISTS idx_episodes_created   ON episodes(created_at);
+  CREATE INDEX IF NOT EXISTS idx_entities_type      ON entities(type);
+  CREATE INDEX IF NOT EXISTS idx_sessions_project   ON sessions(project_id);
+  CREATE INDEX IF NOT EXISTS idx_summaries_project  ON summaries(project_id);
+  CREATE INDEX IF NOT EXISTS idx_summaries_session  ON summaries(session_id);

-  CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts 
+  CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts
    USING fts5(user_message, ai_response, content=episodes, content_rowid=id);

-    CREATE TRIGGER IF NOT EXISTS episodes_fts_insert
+  CREATE TRIGGER IF NOT EXISTS episodes_fts_insert
    AFTER INSERT ON episodes BEGIN
      INSERT INTO episodes_fts(rowid, user_message, ai_response)
      VALUES (new.id, new.user_message, new.ai_response);
@@ -101,8 +115,6 @@ const schema = `
      INSERT INTO episodes_fts(rowid, user_message, ai_response)
      VALUES (new.id, new.user_message, new.ai_response);
    END;
-  
-  
 `;

-module.exports = schema;
+module.exports = schema;
@@ -0,0 +1,65 @@
+// Migration runner — tests the REAL migrate() version-stepping logic against a
+// minimal fake db (emulating the better-sqlite3 pragma getter/setter + transaction
+// API), so it runs with zero dependencies. Guards that migrations apply in order,
+// advance user_version correctly, and no-op when the DB is already current.
+const { test } = require('node:test');
+const assert = require('node:assert');
+const { migrate, LATEST_VERSION } = require('../packages/memory-service/src/db/migrations');
+
+// Emulates just the slice of the better-sqlite3 API that migrate() uses.
+function fakeDb(startVersion = 0) {
+  let version = startVersion;
+  return {
+    ran: [],
+    setCalls: [],
+    pragma(str, opts) {
+      const set = str.match(/^user_version\s*=\s*(\d+)$/);
+      if (set) { version = Number(set[1]); this.setCalls.push(version); return; }
+      if (str === 'user_version') return opts && opts.simple ? version : [{ user_version: version }];
+      throw new Error('unexpected pragma: ' + str);
+    },
+    transaction(fn) { return (...args) => fn(...args); },  // execute immediately
+    get version() { return version; },
+  };
+}
+
+test('a fresh DB (user_version 0) is stamped to LATEST_VERSION', () => {
+  const db = fakeDb(0);
+  const result = migrate(db);
+  assert.strictEqual(result, LATEST_VERSION);
+  assert.strictEqual(db.version, LATEST_VERSION);
+});
+
+test('an already-current DB is a no-op (no version writes)', () => {
+  const db = fakeDb(LATEST_VERSION);
+  const result = migrate(db);
+  assert.strictEqual(result, LATEST_VERSION);
+  assert.deepStrictEqual(db.setCalls, [], 'should not touch user_version when current');
+});
+
+test('injected migrations run in order, each bumping the version by one', () => {
+  const db = fakeDb(0);
+  const order = [];
+  const migs = [
+    () => order.push('v1'),
+    () => order.push('v2'),
+    () => order.push('v3'),
+  ];
+  const result = migrate(db, migs);
+  assert.strictEqual(result, 3);
+  assert.deepStrictEqual(order, ['v1', 'v2', 'v3'], 'migrations apply in sequence');
+  assert.deepStrictEqual(db.setCalls, [1, 2, 3], 'user_version advances one step at a time');
+});
+
+test('only migrations newer than the current version run', () => {
+  const db = fakeDb(1);   // already at v1
+  const order = [];
+  const migs = [
+    () => order.push('v1'),   // should be skipped
+    () => order.push('v2'),
+    () => order.push('v3'),
+  ];
+  migrate(db, migs);
+  assert.deepStrictEqual(order, ['v2', 'v3'], 'v1 is not re-run');
+  assert.deepStrictEqual(db.setCalls, [2, 3]);
+});
@@ -0,0 +1,58 @@
+// Schema completeness — execs the REAL schema.js into a throwaway SQLite DB (via
+// built-in node:sqlite) and asserts a fresh database gets every column the old
+// per-boot ALTER statements used to add. This is the guard for the migration
+// consolidation: if someone drops a column from schema.js, a fresh install would
+// silently lose it, and this test goes red.
+const { test, before } = require('node:test');
+const assert = require('node:assert');
+const schema = require('../packages/memory-service/src/db/schema');
+
+let DatabaseSync;
+try { ({ DatabaseSync } = require('node:sqlite')); } catch { DatabaseSync = null; }
+
+// The complete intended shape = original base tables + every historical ALTER.
+const EXPECTED = {
+  sessions:        ['id','external_id','created_at','updated_at','metadata','name','project_id'],
+  episodes:        ['id','session_id','user_message','ai_response','created_at','token_count','metadata'],
+  entities:        ['id','name','type','notes','created_at','updated_at','metadata','mention_count','confidence','source','last_seen_at'],
+  relationships:   ['id','from_id','to_id','label','created_at','metadata','mention_count','notes'],
+  entity_episodes: ['entity_id','episode_id'],
+  projects:        ['id','name','description','colour','icon','created_at','isolated','notes','system_prompt'],
+  summaries:       ['id','session_id','project_id','content','token_count','episode_range','created_at','metadata'],
+};
+
+const EXPECTED_OBJECTS = [
+  'idx_relationships_from','idx_relationships_to','idx_entity_episodes_entity','idx_entity_episodes_episode',
+  'idx_episodes_session','idx_episodes_created','idx_entities_type','idx_sessions_project',
+  'idx_summaries_project','idx_summaries_session',
+  'episodes_fts','episodes_fts_insert','episodes_fts_delete','episodes_fts_update',
+];
+
+let db;
+before(() => {
+  if (!DatabaseSync) return;
+  db = new DatabaseSync(':memory:');
+  db.exec('PRAGMA foreign_keys = ON');
+  db.exec(schema);   // throws here if schema.js has a syntax/ordering error
+});
+
+for (const [table, cols] of Object.entries(EXPECTED)) {
+  test(`${table} has exactly its full column set`, { skip: !DatabaseSync }, () => {
+    const got = db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name);
+    assert.deepStrictEqual([...got].sort(), [...cols].sort(), `${table} columns differ from the intended shape`);
+  });
+}
+
+test('all indexes, FTS table, and triggers are present', { skip: !DatabaseSync }, () => {
+  const names = db.prepare(`SELECT name FROM sqlite_master WHERE name LIKE 'idx_%' OR name LIKE 'episodes_fts%'`)
+    .all().map(r => r.name);
+  const missing = EXPECTED_OBJECTS.filter(o => !names.includes(o));
+  assert.deepStrictEqual(missing, [], `missing schema objects: ${missing}`);
+});
+
+test('FTS trigger populates the index on episode insert', { skip: !DatabaseSync }, () => {
+  db.prepare(`INSERT INTO sessions(external_id) VALUES ('s-test')`).run();
+  db.prepare(`INSERT INTO episodes(session_id, user_message, ai_response) VALUES (1,'find me qdrant','ok')`).run();
+  const hit = db.prepare(`SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH 'qdrant'`).all();
+  assert.strictEqual(hit.length, 1);
+});