retrieval fusion

packages/orchestration-service/src/chat/index.js

@@ -2,7 +2,7 @@ const memory = require("../services/memory");
 const inference = require("../services/inference");
 const embedding = require("../services/embedding");
 const qdrant = require("../services/qdrant");
-const { ORCHESTRATION, logger } = require("@nexusai/shared");
+const { ORCHESTRATION, RETRIEVAL, logger } = require("@nexusai/shared");
 const appSettings = require("../config/settings");
 const {triggerSummary} = require('../services/summarization')
 const graph = require('../services/graph');
@@ -143,10 +143,59 @@ async function getRelevantEntities(userMessage, projectId = null) {
     }
 }
 
+async function getFTSResults(userMessage, { limit, sessionIds }) {
+    try {
+        return await memory.searchEpisodes(userMessage, { limit, sessionIds });
+    } catch (err) {
+        logger.warn('[orchestration] FTS search failed, continuing without:', err.message);
+        return [];
+    }
+}
+
+function fuseEpisodeResults(semanticEps, keywordEps, { semanticWeight, keywordWeight, limit }) {
+    const k = RETRIEVAL.RRF_K;
+    const scores = new Map();
+
+    semanticEps.forEach((ep, i) => {
+        scores.set(ep.id, { episode: ep, score: semanticWeight / (k + i + 1) });
+    });
+
+    keywordEps.forEach((ep, i) => {
+        const contrib = keywordWeight / (k + i + 1);
+        if (scores.has(ep.id)) {
+            scores.get(ep.id).score += contrib;
+        } else {
+            scores.set(ep.id, { episode: ep, score: contrib });
+        }
+    });
+
+    return [...scores.values()]
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit)
+        .map(({ episode }) => episode);
+}
+
+async function getFusedEpisodes(userMessage, session, recentIds, projectSessionIds, settings) {
+    const { semanticLimit, scoreThreshold, semanticWeight, keywordWeight } = settings;
+    const ftsSessionIds = projectSessionIds ?? [session.id];
+
+    const ftsPromise = keywordWeight > 0
+        ? getFTSResults(userMessage, { limit: semanticLimit * 2, sessionIds: ftsSessionIds })
+        : Promise.resolve([]);
+
+    const [semanticEps, rawKeywordEps] = await Promise.all([
+        getSemanticEpisodes(userMessage, session.id, recentIds, projectSessionIds, { semanticLimit, scoreThreshold }),
+        ftsPromise,
+    ]);
+
+    const keywordEps = rawKeywordEps.filter(ep => !recentIds.has(ep.id));
+    return fuseEpisodeResults(semanticEps, keywordEps, { semanticWeight, keywordWeight, limit: semanticLimit });
+}
+
 async function assembleContext(externalId, userMessage) {
     const settings = appSettings.load();
     const { recentEpisodeLimit, semanticLimit, scoreThreshold, 
-            temperature, repeatPenalty, topP, topK, systemPrompt } = settings;
+            temperature, repeatPenalty, topP, topK, systemPrompt, semanticWeight, keywordWeight } = settings;
 
     // 1. Resolve or create session
     let session = await memory.getSessionByExternalId(externalId);
@@ -174,9 +223,7 @@ async function assembleContext(externalId, userMessage) {
     const recentIds = new Set(recentEpisodes.map(e => e.id));
 
     // 4. Semantic + entity search
-    const semanticEpisodes = await getSemanticEpisodes(
-        userMessage, session.id, recentIds, projectSessionIds, { semanticLimit, scoreThreshold }
-    );
+    const fusedEpisodes = await getFusedEpisodes(userMessage, session, recentIds, projectSessionIds, { semanticLimit, scoreThreshold, semanticWeight, keywordWeight });
     const entityResults = await getRelevantEntities(userMessage, session.project_id ?? null);
 
     // 5. Expand matched entities into 1-hop graph neighborhood
@@ -192,7 +239,7 @@ async function assembleContext(externalId, userMessage) {
     }
 
     // 6. Assemble prompt
-    const prompt = buildPrompt(recentEpisodes, semanticEpisodes, neighborhood, userMessage, activeSystemPrompt);
+    const prompt = buildPrompt(recentEpisodes, fusedEpisodes, neighborhood, userMessage, activeSystemPrompt);
 
     return {
         session,

packages/orchestration-service/src/config/settings.js

@@ -1,6 +1,6 @@
 const fs = require('fs');
 const path = require('path');
-const { getEnv, ORCHESTRATION, INFERENCE_DEFAULTS } = require('@nexusai/shared');
+const { getEnv, ORCHESTRATION, INFERENCE_DEFAULTS, RETRIEVAL } = require('@nexusai/shared');
 
 const SETTINGS_PATH = path.join(__dirname, '../../data/settings.json');
 
@@ -14,6 +14,8 @@ const DEFAULTS = {
   topP:                 INFERENCE_DEFAULTS.TOP_P,
   topK:                 INFERENCE_DEFAULTS.TOP_K,
   systemPrompt:         ORCHESTRATION.SYSTEM_PROMPT,
+  semanticWeight:       RETRIEVAL.SEMANTIC_WEIGHT,
+  keywordWeight:        RETRIEVAL.KEYWORD_WEIGHT,
 };
 
 function load() {

packages/orchestration-service/src/routes/settings.js

@@ -80,6 +80,20 @@ if (req.body.systemPrompt !== undefined) {
   updates.systemPrompt = val.trim() || null; // null reverts to default
 }
 
+  if (req.body.semanticWeight !== undefined) {
+    const val = Number(req.body.semanticWeight);
+    if (isNaN(val) || val < 0 || val > 5)
+      return res.status(400).json({ error: 'semanticWeight must be 0–5' });
+    updates.semanticWeight = val;
+  }
+
+  if (req.body.keywordWeight !== undefined) {
+    const val = Number(req.body.keywordWeight);
+    if (isNaN(val) || val < 0 || val > 5)
+      return res.status(400).json({ error: 'keywordWeight must be 0–5' });
+    updates.keywordWeight = val;
+  }
+
   res.json(settings.save(updates));
 });
 

packages/orchestration-service/src/services/memory.js

@@ -196,6 +196,16 @@ async function getProjectOverviewSummary(projectId) {
     return res.json(); // null if none exists yet
 }
 
+async function searchEpisodes(query, { limit = 10, sessionIds = null } = {}) {
+    const url = new URL(`${BASE_URL}/episodes/search`);
+    url.searchParams.set('q', query);
+    url.searchParams.set('limit', limit);
+    if (sessionIds?.length) url.searchParams.set('sessionIds', sessionIds.join(','));
+    const res = await fetch(url.toString());
+    if (!res.ok) throw new Error(`FTS search error: ${res.status}`);
+    return res.json();
+}
+
 module.exports = {
     getSessionByExternalId,
     createSession,
@@ -220,4 +230,5 @@ module.exports = {
     getSummariesByProject,
     generateProjectSummary,
     getProjectOverviewSummary,
+    searchEpisodes,
 }