From 1df6acc4279c08aee1ae0470a61c53acf09c1920 Mon Sep 17 00:00:00 2001 From: Storme-bit Date: Mon, 17 Aug 2026 04:54:30 -0700 Subject: [PATCH] migration consolidation patch --- nexusai-migration-consolidation.patch | 409 -------------------------- 1 file changed, 409 deletions(-) delete mode 100644 nexusai-migration-consolidation.patch diff --git a/nexusai-migration-consolidation.patch b/nexusai-migration-consolidation.patch deleted file mode 100644 index f0f2d11..0000000 --- a/nexusai-migration-consolidation.patch +++ /dev/null @@ -1,409 +0,0 @@ -diff -ruN nexusai-baseline/packages/memory-service/src/db/index.js nexusai/packages/memory-service/src/db/index.js ---- nexusai-baseline/packages/memory-service/src/db/index.js 2026-08-17 10:23:40.095197576 +0000 -+++ nexusai/packages/memory-service/src/db/index.js 2026-08-17 11:39:45.973925342 +0000 -@@ -1,6 +1,7 @@ - const Database = require('better-sqlite3'); - const schema = require('./schema'); --const {getEnv, SQLITE, logger } = require('@nexusai/shared'); -+const { migrate } = require('./migrations'); -+const { getEnv, SQLITE, logger } = require('@nexusai/shared'); - - let db; // Declare db variable in a scope accessible to all functions - -@@ -12,60 +13,29 @@ - db.pragma('journal_mode = WAL'); - db.pragma('foreign_keys = ON'); - -- db.exec(schema); -- -- try{ -- db.exec(`ALTER TABLE sessions ADD COLUMN name TEXT`) -- } catch {} -- -- try { -- db.exec(`ALTER TABLE sessions ADD COLUMN project_id INTEGER REFERENCES projects(id)`); -- } catch {} -- -- try { -- db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id)`); -- } catch {} -- -- try { -- db.exec(`ALTER TABLE projects ADD COLUMN isolated INTEGER NOT NULL DEFAULT 0`); -- } catch {} -- -- try { -- db.exec(`ALTER TABLE projects ADD COLUMN notes TEXT`); // ← add this -- } catch {} -- -- try { -- db.exec(`ALTER TABLE projects ADD COLUMN system_prompt TEXT`); -- } catch {} -- -- try { -- db.exec(`ALTER TABLE summaries ADD COLUMN project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE`); -- } catch {} -- -- try { -- db.exec(`ALTER TABLE summaries ADD COLUMN token_count INTEGER`); -- } catch {} -- -- try { -- db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id)`); -- } catch {} -- -- try { -- db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id)`); -- } catch {} -- -- try { db.exec(`ALTER TABLE entities ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {} -- try { db.exec(`ALTER TABLE entities ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0`) } catch {} -- try { db.exec(`ALTER TABLE entities ADD COLUMN source TEXT NOT NULL DEFAULT 'extraction'`) } catch {} -- try { db.exec(`ALTER TABLE entities ADD COLUMN last_seen_at INTEGER`) } catch {} -- -- try { db.exec(`ALTER TABLE relationships ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {} -- try { db.exec(`ALTER TABLE relationships ADD COLUMN notes TEXT`) } catch {} -- -- -- // Sync FTS index with any existing episodes data -- db.exec(`INSERT OR REPLACE INTO episodes_fts(rowid, user_message, ai_response) -- SELECT id, user_message, ai_response FROM episodes`); -+ // Was the FTS index absent before this boot? (True for a fresh DB, and for -+ // an older DB from before FTS existed.) Checked BEFORE schema runs so we can -+ // decide whether a one-time backfill is needed below. -+ const ftsExisted = db.prepare( -+ `SELECT 1 FROM sqlite_master WHERE type='table' AND name='episodes_fts'` -+ ).get() !== undefined; -+ -+ db.exec(schema); // complete current shape — fresh DBs get everything -+ migrate(db); // carry an older DB forward; no-op on fresh/current DBs -+ -+ // One-time FTS backfill: only when the index was just created on a DB that -+ // already holds episodes (i.e. episodes predate FTS). During normal -+ // operation the insert/delete/update triggers keep it in sync, so this no -+ // longer rebuilds the whole index on every boot. NOTE: COUNT(*) on an -+ // external-content FTS5 table proxies the content table, so it can't detect -+ // a desync — the "was it just created" check is what makes this correct. -+ if (!ftsExisted) { -+ const epCount = db.prepare('SELECT COUNT(*) AS c FROM episodes').get().c; -+ if (epCount > 0) { -+ db.exec(`INSERT INTO episodes_fts(episodes_fts) VALUES('rebuild')`); -+ logger.info(`[db] Backfilled FTS index for ${epCount} pre-existing episodes`); -+ } -+ } - - logger.info(`Connected to SQLite database at ${path}`); - } -@@ -74,4 +44,4 @@ - - module.exports = { - getDB --}; -\ No newline at end of file -+}; -diff -ruN nexusai-baseline/packages/memory-service/src/db/migrations.js nexusai/packages/memory-service/src/db/migrations.js ---- nexusai-baseline/packages/memory-service/src/db/migrations.js 1970-01-01 00:00:00.000000000 +0000 -+++ nexusai/packages/memory-service/src/db/migrations.js 2026-08-17 11:40:33.900163765 +0000 -@@ -0,0 +1,43 @@ -+const { logger } = require('@nexusai/shared'); -+ -+// Forward-only schema migrations. Entry i takes the database from user_version i -+// to i+1, so migrations[0] is the v0→v1 step, migrations[1] the v1→v2 step, etc. -+// -+// schema.js already holds the COMPLETE current shape, so a fresh database is -+// created whole and stamped straight to LATEST_VERSION — these run only to carry -+// an OLDER database forward. Add a new schema change by appending a function here -+// (which bumps LATEST_VERSION by one); never edit an existing entry once shipped, -+// since databases already stamped past it will not re-run it. -+// -+// Each migration receives the better-sqlite3 db handle and runs inside a -+// transaction together with its version bump, so a failure rolls back cleanly. -+const migrations = [ -+ // v0 → v1: baseline. The historical ALTER TABLE / CREATE INDEX statements that -+ // used to run (wrapped in try/catch) on every boot are folded into schema.js. -+ // Nothing to do here — this entry exists to mark v1 as the consolidated baseline. -+ (_db) => {}, -+]; -+ -+const LATEST_VERSION = migrations.length; -+ -+// Applies any migrations newer than the database's current user_version, one at a -+// time, each with its version bump, inside a transaction. Returns the resulting -+// version. A no-op when the database is already current. The migrations list and -+// target version are injectable for testing; production callers pass just `db`. -+function migrate(db, migs = migrations, latest = migs.length) { -+ const current = db.pragma('user_version', { simple: true }); -+ if (current >= latest) return current; -+ -+ for (let v = current; v < latest; v++) { -+ const step = db.transaction(() => { -+ migs[v](db); -+ db.pragma(`user_version = ${v + 1}`); -+ }); -+ step(); -+ } -+ -+ logger.info(`[db] schema migrated ${current} -> ${latest}`); -+ return latest; -+} -+ -+module.exports = { migrate, migrations, LATEST_VERSION }; -diff -ruN nexusai-baseline/packages/memory-service/src/db/schema.js nexusai/packages/memory-service/src/db/schema.js ---- nexusai-baseline/packages/memory-service/src/db/schema.js 2026-08-17 10:23:40.095507736 +0000 -+++ nexusai/packages/memory-service/src/db/schema.js 2026-08-17 11:39:21.421925904 +0000 -@@ -1,10 +1,17 @@ -+// Complete current schema — the single source of truth for a fresh database. -+// Historical ALTER TABLE statements that used to run on every boot are folded in -+// here. Any change that must reach EXISTING databases goes in migrations.js as a -+// new numbered migration, NOT by editing a table below (CREATE ... IF NOT EXISTS -+// silently skips tables that already exist, so column edits here never reach them). - const schema = ` - CREATE TABLE IF NOT EXISTS sessions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - external_id TEXT UNIQUE NOT NULL, - created_at INTEGER NOT NULL DEFAULT (unixepoch()), - updated_at INTEGER NOT NULL DEFAULT (unixepoch()), -- metadata TEXT -+ metadata TEXT, -+ name TEXT, -+ project_id INTEGER REFERENCES projects(id) - ); - - CREATE TABLE IF NOT EXISTS episodes ( -@@ -18,23 +25,29 @@ - ); - - CREATE TABLE IF NOT EXISTS entities ( -- id INTEGER PRIMARY KEY AUTOINCREMENT, -- name TEXT NOT NULL, -- type TEXT NOT NULL, -- notes TEXT, -- created_at INTEGER NOT NULL DEFAULT (unixepoch()), -- updated_at INTEGER NOT NULL DEFAULT (unixepoch()), -- metadata TEXT, -+ id INTEGER PRIMARY KEY AUTOINCREMENT, -+ name TEXT NOT NULL, -+ type TEXT NOT NULL, -+ notes TEXT, -+ created_at INTEGER NOT NULL DEFAULT (unixepoch()), -+ updated_at INTEGER NOT NULL DEFAULT (unixepoch()), -+ metadata TEXT, -+ mention_count INTEGER NOT NULL DEFAULT 1, -+ confidence REAL NOT NULL DEFAULT 1.0, -+ source TEXT NOT NULL DEFAULT 'extraction', -+ last_seen_at INTEGER, - UNIQUE(name, type) - ); - - CREATE TABLE IF NOT EXISTS relationships ( -- id INTEGER PRIMARY KEY AUTOINCREMENT, -- from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, -- to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, -- label TEXT NOT NULL, -- created_at INTEGER NOT NULL DEFAULT (unixepoch()), -- metadata TEXT, -+ id INTEGER PRIMARY KEY AUTOINCREMENT, -+ from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, -+ to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, -+ label TEXT NOT NULL, -+ created_at INTEGER NOT NULL DEFAULT (unixepoch()), -+ metadata TEXT, -+ mention_count INTEGER NOT NULL DEFAULT 1, -+ notes TEXT, - UNIQUE(from_id, to_id, label) - ); - -@@ -49,16 +62,17 @@ - - CREATE INDEX IF NOT EXISTS idx_entity_episodes_entity ON entity_episodes(entity_id); - CREATE INDEX IF NOT EXISTS idx_entity_episodes_episode ON entity_episodes(episode_id); -- -- - - CREATE TABLE IF NOT EXISTS projects ( -- id INTEGER PRIMARY KEY AUTOINCREMENT, -- name TEXT NOT NULL, -- description TEXT, -- colour TEXT, -- icon TEXT, -- created_at INTEGER NOT NULL DEFAULT (unixepoch()) -+ id INTEGER PRIMARY KEY AUTOINCREMENT, -+ name TEXT NOT NULL, -+ description TEXT, -+ colour TEXT, -+ icon TEXT, -+ created_at INTEGER NOT NULL DEFAULT (unixepoch()), -+ isolated INTEGER NOT NULL DEFAULT 0, -+ notes TEXT, -+ system_prompt TEXT - ); - - CREATE TABLE IF NOT EXISTS summaries ( -@@ -72,17 +86,17 @@ - metadata TEXT - ); - -- CREATE INDEX IF NOT EXISTS idx_episodes_session -- ON episodes(session_id); -- CREATE INDEX IF NOT EXISTS idx_episodes_created -- ON episodes(created_at); -- CREATE INDEX IF NOT EXISTS idx_entities_type -- ON entities(type); -+ CREATE INDEX IF NOT EXISTS idx_episodes_session ON episodes(session_id); -+ CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at); -+ CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type); -+ CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id); -+ CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id); -+ CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id); - -- CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts -+ CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts - USING fts5(user_message, ai_response, content=episodes, content_rowid=id); - -- CREATE TRIGGER IF NOT EXISTS episodes_fts_insert -+ CREATE TRIGGER IF NOT EXISTS episodes_fts_insert - AFTER INSERT ON episodes BEGIN - INSERT INTO episodes_fts(rowid, user_message, ai_response) - VALUES (new.id, new.user_message, new.ai_response); -@@ -101,8 +115,6 @@ - INSERT INTO episodes_fts(rowid, user_message, ai_response) - VALUES (new.id, new.user_message, new.ai_response); - END; -- -- - `; - --module.exports = schema; -\ No newline at end of file -+module.exports = schema; -diff -ruN nexusai-baseline/test/migrations.test.js nexusai/test/migrations.test.js ---- nexusai-baseline/test/migrations.test.js 1970-01-01 00:00:00.000000000 +0000 -+++ nexusai/test/migrations.test.js 2026-08-17 11:41:02.541923591 +0000 -@@ -0,0 +1,65 @@ -+// Migration runner — tests the REAL migrate() version-stepping logic against a -+// minimal fake db (emulating the better-sqlite3 pragma getter/setter + transaction -+// API), so it runs with zero dependencies. Guards that migrations apply in order, -+// advance user_version correctly, and no-op when the DB is already current. -+const { test } = require('node:test'); -+const assert = require('node:assert'); -+const { migrate, LATEST_VERSION } = require('../packages/memory-service/src/db/migrations'); -+ -+// Emulates just the slice of the better-sqlite3 API that migrate() uses. -+function fakeDb(startVersion = 0) { -+ let version = startVersion; -+ return { -+ ran: [], -+ setCalls: [], -+ pragma(str, opts) { -+ const set = str.match(/^user_version\s*=\s*(\d+)$/); -+ if (set) { version = Number(set[1]); this.setCalls.push(version); return; } -+ if (str === 'user_version') return opts && opts.simple ? version : [{ user_version: version }]; -+ throw new Error('unexpected pragma: ' + str); -+ }, -+ transaction(fn) { return (...args) => fn(...args); }, // execute immediately -+ get version() { return version; }, -+ }; -+} -+ -+test('a fresh DB (user_version 0) is stamped to LATEST_VERSION', () => { -+ const db = fakeDb(0); -+ const result = migrate(db); -+ assert.strictEqual(result, LATEST_VERSION); -+ assert.strictEqual(db.version, LATEST_VERSION); -+}); -+ -+test('an already-current DB is a no-op (no version writes)', () => { -+ const db = fakeDb(LATEST_VERSION); -+ const result = migrate(db); -+ assert.strictEqual(result, LATEST_VERSION); -+ assert.deepStrictEqual(db.setCalls, [], 'should not touch user_version when current'); -+}); -+ -+test('injected migrations run in order, each bumping the version by one', () => { -+ const db = fakeDb(0); -+ const order = []; -+ const migs = [ -+ () => order.push('v1'), -+ () => order.push('v2'), -+ () => order.push('v3'), -+ ]; -+ const result = migrate(db, migs); -+ assert.strictEqual(result, 3); -+ assert.deepStrictEqual(order, ['v1', 'v2', 'v3'], 'migrations apply in sequence'); -+ assert.deepStrictEqual(db.setCalls, [1, 2, 3], 'user_version advances one step at a time'); -+}); -+ -+test('only migrations newer than the current version run', () => { -+ const db = fakeDb(1); // already at v1 -+ const order = []; -+ const migs = [ -+ () => order.push('v1'), // should be skipped -+ () => order.push('v2'), -+ () => order.push('v3'), -+ ]; -+ migrate(db, migs); -+ assert.deepStrictEqual(order, ['v2', 'v3'], 'v1 is not re-run'); -+ assert.deepStrictEqual(db.setCalls, [2, 3]); -+}); -diff -ruN nexusai-baseline/test/schema.test.js nexusai/test/schema.test.js ---- nexusai-baseline/test/schema.test.js 1970-01-01 00:00:00.000000000 +0000 -+++ nexusai/test/schema.test.js 2026-08-17 11:40:49.545923888 +0000 -@@ -0,0 +1,58 @@ -+// Schema completeness — execs the REAL schema.js into a throwaway SQLite DB (via -+// built-in node:sqlite) and asserts a fresh database gets every column the old -+// per-boot ALTER statements used to add. This is the guard for the migration -+// consolidation: if someone drops a column from schema.js, a fresh install would -+// silently lose it, and this test goes red. -+const { test, before } = require('node:test'); -+const assert = require('node:assert'); -+const schema = require('../packages/memory-service/src/db/schema'); -+ -+let DatabaseSync; -+try { ({ DatabaseSync } = require('node:sqlite')); } catch { DatabaseSync = null; } -+ -+// The complete intended shape = original base tables + every historical ALTER. -+const EXPECTED = { -+ sessions: ['id','external_id','created_at','updated_at','metadata','name','project_id'], -+ episodes: ['id','session_id','user_message','ai_response','created_at','token_count','metadata'], -+ entities: ['id','name','type','notes','created_at','updated_at','metadata','mention_count','confidence','source','last_seen_at'], -+ relationships: ['id','from_id','to_id','label','created_at','metadata','mention_count','notes'], -+ entity_episodes: ['entity_id','episode_id'], -+ projects: ['id','name','description','colour','icon','created_at','isolated','notes','system_prompt'], -+ summaries: ['id','session_id','project_id','content','token_count','episode_range','created_at','metadata'], -+}; -+ -+const EXPECTED_OBJECTS = [ -+ 'idx_relationships_from','idx_relationships_to','idx_entity_episodes_entity','idx_entity_episodes_episode', -+ 'idx_episodes_session','idx_episodes_created','idx_entities_type','idx_sessions_project', -+ 'idx_summaries_project','idx_summaries_session', -+ 'episodes_fts','episodes_fts_insert','episodes_fts_delete','episodes_fts_update', -+]; -+ -+let db; -+before(() => { -+ if (!DatabaseSync) return; -+ db = new DatabaseSync(':memory:'); -+ db.exec('PRAGMA foreign_keys = ON'); -+ db.exec(schema); // throws here if schema.js has a syntax/ordering error -+}); -+ -+for (const [table, cols] of Object.entries(EXPECTED)) { -+ test(`${table} has exactly its full column set`, { skip: !DatabaseSync }, () => { -+ const got = db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name); -+ assert.deepStrictEqual([...got].sort(), [...cols].sort(), `${table} columns differ from the intended shape`); -+ }); -+} -+ -+test('all indexes, FTS table, and triggers are present', { skip: !DatabaseSync }, () => { -+ const names = db.prepare(`SELECT name FROM sqlite_master WHERE name LIKE 'idx_%' OR name LIKE 'episodes_fts%'`) -+ .all().map(r => r.name); -+ const missing = EXPECTED_OBJECTS.filter(o => !names.includes(o)); -+ assert.deepStrictEqual(missing, [], `missing schema objects: ${missing}`); -+}); -+ -+test('FTS trigger populates the index on episode insert', { skip: !DatabaseSync }, () => { -+ db.prepare(`INSERT INTO sessions(external_id) VALUES ('s-test')`).run(); -+ db.prepare(`INSERT INTO episodes(session_id, user_message, ai_response) VALUES (1,'find me qdrant','ok')`).run(); -+ const hit = db.prepare(`SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH 'qdrant'`).all(); -+ assert.strictEqual(hit.length, 1); -+});