storme/nexusAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migration consolidation patch

Browse Source
This commit is contained in:
Storme-bit
2026-08-17 04:53:12 -07:00
parent e570bbecf2
commit 0353ce96e0
6 changed files with 644 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/memory-service/src/db/index.js
+24 -54
View File
@@ -1,6 +1,7 @@
const Database = require('better-sqlite3');
const schema = require('./schema');
const {getEnv, SQLITE, logger } = require('@nexusai/shared');
const { migrate } = require('./migrations');
const { getEnv, SQLITE, logger } = require('@nexusai/shared');
let db; // Declare db variable in a scope accessible to all functions
@@ -12,60 +13,29 @@ function getDB() {
db.pragma('journal_mode = WAL');
db.pragma('foreign_keys = ON');
db.exec(schema);
// Was the FTS index absent before this boot? (True for a fresh DB, and for
// an older DB from before FTS existed.) Checked BEFORE schema runs so we can
// decide whether a one-time backfill is needed below.
const ftsExisted = db.prepare(
`SELECT 1 FROM sqlite_master WHERE type='table' AND name='episodes_fts'`
).get() !== undefined;
try{
db.exec(`ALTER TABLE sessions ADD COLUMN name TEXT`)
} catch {}
db.exec(schema); // complete current shape — fresh DBs get everything
migrate(db); // carry an older DB forward; no-op on fresh/current DBs
try {
db.exec(`ALTER TABLE sessions ADD COLUMN project_id INTEGER REFERENCES projects(id)`);
} catch {}
try {
db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id)`);
} catch {}
try {
db.exec(`ALTER TABLE projects ADD COLUMN isolated INTEGER NOT NULL DEFAULT 0`);
} catch {}
try {
db.exec(`ALTER TABLE projects ADD COLUMN notes TEXT`); // ← add this
} catch {}
try {
db.exec(`ALTER TABLE projects ADD COLUMN system_prompt TEXT`);
} catch {}
try {
db.exec(`ALTER TABLE summaries ADD COLUMN project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE`);
} catch {}
try {
db.exec(`ALTER TABLE summaries ADD COLUMN token_count INTEGER`);
} catch {}
try {
db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id)`);
} catch {}
try {
db.exec(`CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id)`);
} catch {}
try { db.exec(`ALTER TABLE entities ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {}
try { db.exec(`ALTER TABLE entities ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0`) } catch {}
try { db.exec(`ALTER TABLE entities ADD COLUMN source TEXT NOT NULL DEFAULT 'extraction'`) } catch {}
try { db.exec(`ALTER TABLE entities ADD COLUMN last_seen_at INTEGER`) } catch {}
try { db.exec(`ALTER TABLE relationships ADD COLUMN mention_count INTEGER NOT NULL DEFAULT 1`) } catch {}
try { db.exec(`ALTER TABLE relationships ADD COLUMN notes TEXT`) } catch {}
// Sync FTS index with any existing episodes data
db.exec(`INSERT OR REPLACE INTO episodes_fts(rowid, user_message, ai_response)
SELECT id, user_message, ai_response FROM episodes`);
// One-time FTS backfill: only when the index was just created on a DB that
// already holds episodes (i.e. episodes predate FTS). During normal
// operation the insert/delete/update triggers keep it in sync, so this no
// longer rebuilds the whole index on every boot. NOTE: COUNT(*) on an
// external-content FTS5 table proxies the content table, so it can't detect
// a desync — the "was it just created" check is what makes this correct.
if (!ftsExisted) {
const epCount = db.prepare('SELECT COUNT(*) AS c FROM episodes').get().c;
if (epCount > 0) {
db.exec(`INSERT INTO episodes_fts(episodes_fts) VALUES('rebuild')`);
logger.info(`[db] Backfilled FTS index for ${epCount} pre-existing episodes`);
}
}
logger.info(`Connected to SQLite database at ${path}`);
}
@@ -74,4 +44,4 @@ function getDB() {
module.exports = {
getDB
};
};
packages/memory-service/src/db/migrations.js
+43
View File
@@ -0,0 +1,43 @@
const { logger } = require('@nexusai/shared');
// Forward-only schema migrations. Entry i takes the database from user_version i
// to i+1, so migrations[0] is the v0→v1 step, migrations[1] the v1→v2 step, etc.
//
// schema.js already holds the COMPLETE current shape, so a fresh database is
// created whole and stamped straight to LATEST_VERSION — these run only to carry
// an OLDER database forward. Add a new schema change by appending a function here
// (which bumps LATEST_VERSION by one); never edit an existing entry once shipped,
// since databases already stamped past it will not re-run it.
//
// Each migration receives the better-sqlite3 db handle and runs inside a
// transaction together with its version bump, so a failure rolls back cleanly.
const migrations = [
// v0 → v1: baseline. The historical ALTER TABLE / CREATE INDEX statements that
// used to run (wrapped in try/catch) on every boot are folded into schema.js.
// Nothing to do here — this entry exists to mark v1 as the consolidated baseline.
(_db) => {},
];
const LATEST_VERSION = migrations.length;
// Applies any migrations newer than the database's current user_version, one at a
// time, each with its version bump, inside a transaction. Returns the resulting
// version. A no-op when the database is already current. The migrations list and
// target version are injectable for testing; production callers pass just `db`.
function migrate(db, migs = migrations, latest = migs.length) {
const current = db.pragma('user_version', { simple: true });
if (current >= latest) return current;
for (let v = current; v < latest; v++) {
const step = db.transaction(() => {
migs[v](db);
db.pragma(`user_version = ${v + 1}`);
});
step();
}
logger.info(`[db] schema migrated ${current} -> ${latest}`);
return latest;
}
module.exports = { migrate, migrations, LATEST_VERSION };
packages/memory-service/src/db/schema.js
+45 -33
View File
@@ -1,10 +1,17 @@
// Complete current schema — the single source of truth for a fresh database.
// Historical ALTER TABLE statements that used to run on every boot are folded in
// here. Any change that must reach EXISTING databases goes in migrations.js as a
// new numbered migration, NOT by editing a table below (CREATE ... IF NOT EXISTS
// silently skips tables that already exist, so column edits here never reach them).
const schema = `
CREATE TABLE IF NOT EXISTS sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
external_id TEXT UNIQUE NOT NULL,
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
metadata TEXT
metadata TEXT,
name TEXT,
project_id INTEGER REFERENCES projects(id)
);
CREATE TABLE IF NOT EXISTS episodes (
@@ -18,23 +25,29 @@ const schema = `
);
CREATE TABLE IF NOT EXISTS entities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
type TEXT NOT NULL,
notes TEXT,
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
metadata TEXT,
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
type TEXT NOT NULL,
notes TEXT,
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
metadata TEXT,
mention_count INTEGER NOT NULL DEFAULT 1,
confidence REAL NOT NULL DEFAULT 1.0,
source TEXT NOT NULL DEFAULT 'extraction',
last_seen_at INTEGER,
UNIQUE(name, type)
);
CREATE TABLE IF NOT EXISTS relationships (
id INTEGER PRIMARY KEY AUTOINCREMENT,
from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
label TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
metadata TEXT,
id INTEGER PRIMARY KEY AUTOINCREMENT,
from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
label TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
metadata TEXT,
mention_count INTEGER NOT NULL DEFAULT 1,
notes TEXT,
UNIQUE(from_id, to_id, label)
);
@@ -49,16 +62,17 @@ const schema = `
CREATE INDEX IF NOT EXISTS idx_entity_episodes_entity ON entity_episodes(entity_id);
CREATE INDEX IF NOT EXISTS idx_entity_episodes_episode ON entity_episodes(episode_id);
CREATE TABLE IF NOT EXISTS projects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
colour TEXT,
icon TEXT,
created_at INTEGER NOT NULL DEFAULT (unixepoch())
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
colour TEXT,
icon TEXT,
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
isolated INTEGER NOT NULL DEFAULT 0,
notes TEXT,
system_prompt TEXT
);
CREATE TABLE IF NOT EXISTS summaries (
@@ -72,17 +86,17 @@ const schema = `
metadata TEXT
);
CREATE INDEX IF NOT EXISTS idx_episodes_session
ON episodes(session_id);
CREATE INDEX IF NOT EXISTS idx_episodes_created
ON episodes(created_at);
CREATE INDEX IF NOT EXISTS idx_entities_type
ON entities(type);
CREATE INDEX IF NOT EXISTS idx_episodes_session ON episodes(session_id);
CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at);
CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type);
CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id);
CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_id);
CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id);
CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts
CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts
USING fts5(user_message, ai_response, content=episodes, content_rowid=id);
CREATE TRIGGER IF NOT EXISTS episodes_fts_insert
CREATE TRIGGER IF NOT EXISTS episodes_fts_insert
AFTER INSERT ON episodes BEGIN
INSERT INTO episodes_fts(rowid, user_message, ai_response)
VALUES (new.id, new.user_message, new.ai_response);
@@ -101,8 +115,6 @@ const schema = `
INSERT INTO episodes_fts(rowid, user_message, ai_response)
VALUES (new.id, new.user_message, new.ai_response);
END;
`;
module.exports = schema;
module.exports = schema;