From fab8f323956bf03ec8d4e503267490a61781a470 Mon Sep 17 00:00:00 2001 From: Storme-bit Date: Mon, 17 Aug 2026 07:18:51 -0700 Subject: [PATCH] utilityInference --- packages/inference-service/src/index.js | 2 + .../inference-service/src/routes/utility.js | 21 +++++++++ packages/inference-service/src/utility.js | 44 +++++++++++++++++++ packages/shared/src/config/constants.js | 9 ++++ packages/shared/src/index.js | 18 +++++++- 5 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 packages/inference-service/src/routes/utility.js create mode 100644 packages/inference-service/src/utility.js diff --git a/packages/inference-service/src/index.js b/packages/inference-service/src/index.js index 55169cc..60116a0 100644 --- a/packages/inference-service/src/index.js +++ b/packages/inference-service/src/index.js @@ -2,6 +2,7 @@ require ('dotenv').config(); const express = require('express'); const {getEnv, PORTS, OLLAMA, logger} = require('@nexusai/shared'); const inferenceRouter = require('./routes/inference'); +const utilityRouter = require('./routes/utility') const app = express(); app.use(express.json({ limit: '8mb' })); // prompts include full context window @@ -20,6 +21,7 @@ app.get('/health', (req, res) => { }); }); +app.use('/', utilityRouter) app.use('/', inferenceRouter); // Start the server diff --git a/packages/inference-service/src/routes/utility.js b/packages/inference-service/src/routes/utility.js new file mode 100644 index 0000000..92d68b8 --- /dev/null +++ b/packages/inference-service/src/routes/utility.js @@ -0,0 +1,21 @@ +const { Router } = require('express'); +const { logger } = require('@nexusai/shared'); +const { utilityComplete } = require('../utility'); + +const router = Router(); + +router.post('/utility/complete', async (req, res) => { + const { system, user, json, temperature, maxTokens } = req.body; + + if (!user) return res.status(400).json({ error: 'user message is required' }); + + try { + const result = await utilityComplete({ system, user, json, temperature, maxTokens }); + res.json(result); + } catch (error) { + logger.error('[Utility] Completion error:', error.message); + res.status(500).json({ error: 'Utility inference failed', detail: error.message }); + } +}); + +module.exports = router; \ No newline at end of file diff --git a/packages/inference-service/src/utility.js b/packages/inference-service/src/utility.js new file mode 100644 index 0000000..f3799fc --- /dev/null +++ b/packages/inference-service/src/utility.js @@ -0,0 +1,44 @@ +const { getEnv, UTILITY } = require ('@nexusai/shared') + +const UTILITY_URL = getEnv('UTILITY_URL', UTILITY.DEFAULT_URL); +const UTILITY_MODEL = getEnv('UTILITY_MODEL', UTILITY.DEFAULT_MODEL); + +// Background task inference (extraction/summarization). Uses ollama's /api/chat +// so the model's own prompt template is server-side, no need for ChatML +async function utilityComplete({ + system, + user, + json = false, + temperature, + maxTokens +}) { + const messages = []; + if(system) messages.push({ role: 'system', content: system}); + messages.push ({role: 'user', content: user}); + + const res = await fetch (`${UTILITY_URL}/api/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json'}, + body: JSON.stringify({ + model: UTILITY_MODEL, + messages, + stream: false, + ...UTILITY(json && {format: 'json' }), + options: { + temperature: temperature ?? UTILITY.TEMPERATURE, + num_predict: maxTokens ?? UTILITY.MAX_TOKENS, + }, + }), + signal: AbortSignal.timeout(UTILITY.TIMEOUT_MS), + }); + + if (!res.ok) throw new Error(`Utility backend responded ${res.status}`); + const data = await res.json(); + + return { + text: (data.message?.content ?? '').trim(), + model: data.model, + } +} + +module.exports = { utilityComplete}; \ No newline at end of file diff --git a/packages/shared/src/config/constants.js b/packages/shared/src/config/constants.js index a599406..f63f7b0 100644 --- a/packages/shared/src/config/constants.js +++ b/packages/shared/src/config/constants.js @@ -105,6 +105,14 @@ const RETRIEVAL = { KEYWORD_WEIGHT: 0.5, // Weight applied to keyword (SQLite) results, 0 = disables, set >0 to enable and tune balance between semantic vs keyword matches } +const UTILITY = { + DEFAULT_URL: 'http://localhost:11434', // Ollama host for background/utility tasks + DEFAULT_MODEL: 'qwen2.5:3b', + TEMPERATURE: 0.2, // precise, low-creativity default for extraction/summaries + MAX_TOKENS: 1200, + TIMEOUT_MS: 120_000, // extraction previously used 60s; summaries had none — standardized +}; + module.exports = { QDRANT, COLLECTIONS, @@ -119,4 +127,5 @@ module.exports = { SUMMARIES, ENTITIES, RETRIEVAL, + UTILITY, }; \ No newline at end of file diff --git a/packages/shared/src/index.js b/packages/shared/src/index.js index 896dc3c..2ee5ba4 100644 --- a/packages/shared/src/index.js +++ b/packages/shared/src/index.js @@ -1,5 +1,20 @@ const {getEnv} = require('./config/env'); -const {QDRANT, COLLECTIONS, EPISODIC, SERVICES, OLLAMA, PORTS, LLAMACPP, INFERENCE_DEFAULTS, SQLITE, ORCHESTRATION, SUMMARIES, ENTITIES, RETRIEVAL } = require('./config/constants'); +const { + QDRANT, + COLLECTIONS, + EPISODIC, + SERVICES, + OLLAMA, + PORTS, + LLAMACPP, + INFERENCE_DEFAULTS, + SQLITE, + ORCHESTRATION, + SUMMARIES, + ENTITIES, + RETRIEVAL, + UTILITY + } = require('./config/constants'); const {parseRow, formatEpisodeText, isTrivialTurn} = require('./utils') const logger = require('./utils/logger'); @@ -22,4 +37,5 @@ module.exports = { ENTITIES, logger, RETRIEVAL, + UTILITY, }; \ No newline at end of file