70 lines
2.8 KiB
JavaScript
70 lines
2.8 KiB
JavaScript
// FTS keyword search — tests the REAL buildFtsQuery against a real FTS5 table.
|
|
// Uses Node's built-in node:sqlite (no external deps) to build a throwaway index,
|
|
// so this runs offline with nothing installed beyond Node itself.
|
|
const { test, before } = require('node:test');
|
|
const assert = require('node:assert');
|
|
const { buildFtsQuery } = require('../packages/memory-service/src/episodic/index');
|
|
|
|
let DatabaseSync;
|
|
try {
|
|
({ DatabaseSync } = require('node:sqlite'));
|
|
} catch {
|
|
DatabaseSync = null;
|
|
}
|
|
|
|
const ROWS = [
|
|
'To configure the Qdrant collection you set the vector size and distance metric',
|
|
'The embedding service wraps nomic-embed-text via Ollama',
|
|
'Reciprocal rank fusion merges semantic and keyword results',
|
|
'I had a great morning walk today',
|
|
];
|
|
|
|
let db;
|
|
before(() => {
|
|
if (!DatabaseSync) return;
|
|
db = new DatabaseSync(':memory:');
|
|
db.exec('CREATE VIRTUAL TABLE fts USING fts5(body)');
|
|
const ins = db.prepare('INSERT INTO fts(rowid, body) VALUES (?, ?)');
|
|
ROWS.forEach((r, i) => ins.run(i + 1, r));
|
|
});
|
|
|
|
const matchRows = (match) =>
|
|
db.prepare('SELECT rowid FROM fts WHERE fts MATCH ? ORDER BY rank').all(match).map(r => r.rowid);
|
|
|
|
// --- Pure buildFtsQuery behaviour (runs without sqlite) ---
|
|
|
|
test('tokenizes and OR-joins significant terms, dropping stopwords', () => {
|
|
assert.strictEqual(
|
|
buildFtsQuery('How do I configure the Qdrant collection?'),
|
|
'"configure" OR "qdrant" OR "collection"',
|
|
);
|
|
});
|
|
|
|
test('all-stopword / empty input returns null so caller can skip keyword search', () => {
|
|
assert.strictEqual(buildFtsQuery('how do I do it?'), null);
|
|
assert.strictEqual(buildFtsQuery(' '), null);
|
|
assert.strictEqual(buildFtsQuery('!!! ??? ...'), null);
|
|
});
|
|
|
|
test('FTS-significant tokens in the message are neutralized as literals', () => {
|
|
// "OR" and the stray quote/asterisk must not become operators
|
|
const q = buildFtsQuery('fusion OR results" hack*');
|
|
assert.strictEqual(q, '"fusion" OR "results" OR "hack"');
|
|
});
|
|
|
|
// --- Real FTS5 matching (skips cleanly if node:sqlite is unavailable) ---
|
|
|
|
test('a conversational query matches on shared terms (the whole point of the fix)', { skip: !DatabaseSync }, () => {
|
|
const q = buildFtsQuery('How do I configure the Qdrant collection?');
|
|
assert.deepStrictEqual(matchRows(q), [1], 'should hit the Qdrant-collection episode');
|
|
});
|
|
|
|
test('the OLD whole-phrase approach would NOT have matched (regression guard)', { skip: !DatabaseSync }, () => {
|
|
const oldPhrase = `"${'How do I configure the Qdrant collection?'.replace(/"/g, '""')}"`;
|
|
assert.deepStrictEqual(matchRows(oldPhrase), [], 'verbatim phrase never appears → the bug we fixed');
|
|
});
|
|
|
|
test('injection-y input runs without error and matches real terms', { skip: !DatabaseSync }, () => {
|
|
const q = buildFtsQuery('fusion OR results" hack*');
|
|
assert.deepStrictEqual(matchRows(q), [3]);
|
|
}); |