utilityInference

packages/inference-service/src/index.js

@@ -2,6 +2,7 @@ require ('dotenv').config();
 const express = require('express');
 const {getEnv, PORTS, OLLAMA, logger} = require('@nexusai/shared');
 const inferenceRouter = require('./routes/inference');
+const utilityRouter = require('./routes/utility')
 
 const app = express();
 app.use(express.json({ limit: '8mb' }));  // prompts include full context window
@@ -20,6 +21,7 @@ app.get('/health', (req, res) => {
     });
 });
 
+app.use('/', utilityRouter)
 app.use('/', inferenceRouter);
 
 // Start the server

packages/inference-service/src/routes/utility.js

@@ -0,0 +1,21 @@
+const { Router } = require('express');
+const { logger } = require('@nexusai/shared');
+const { utilityComplete } = require('../utility');
+
+const router = Router();
+
+router.post('/utility/complete', async (req, res) => {
+    const { system, user, json, temperature, maxTokens } = req.body;
+
+    if (!user) return res.status(400).json({ error: 'user message is required' });
+
+    try {
+        const result = await utilityComplete({ system, user, json, temperature, maxTokens });
+        res.json(result);
+    } catch (error) {
+        logger.error('[Utility] Completion error:', error.message);
+        res.status(500).json({ error: 'Utility inference failed', detail: error.message });
+    }
+});
+
+module.exports = router;

packages/inference-service/src/utility.js

@@ -0,0 +1,44 @@
+const { getEnv, UTILITY } = require ('@nexusai/shared')
+
+const UTILITY_URL = getEnv('UTILITY_URL', UTILITY.DEFAULT_URL);
+const UTILITY_MODEL = getEnv('UTILITY_MODEL', UTILITY.DEFAULT_MODEL);
+
+// Background task inference (extraction/summarization).  Uses ollama's /api/chat
+// so the model's own prompt template is server-side, no need for ChatML
+async function utilityComplete({
+    system,
+    user,
+    json = false,
+    temperature,
+    maxTokens
+}) {
+    const messages = [];
+    if(system) messages.push({ role: 'system', content: system});
+    messages.push ({role: 'user', content: user});
+    
+    const res = await fetch (`${UTILITY_URL}/api/chat`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json'},
+        body: JSON.stringify({
+            model: UTILITY_MODEL,
+            messages,
+            stream: false,
+            ...UTILITY(json && {format: 'json' }),
+            options: {
+                temperature: temperature ?? UTILITY.TEMPERATURE,
+                num_predict: maxTokens ?? UTILITY.MAX_TOKENS,
+            },
+        }),
+        signal: AbortSignal.timeout(UTILITY.TIMEOUT_MS),
+    });
+
+    if (!res.ok) throw new Error(`Utility backend responded ${res.status}`);
+    const data = await res.json();
+
+    return {
+        text: (data.message?.content ?? '').trim(),
+        model: data.model,
+    }
+}
+
+module.exports = { utilityComplete};

packages/shared/src/config/constants.js

@@ -105,6 +105,14 @@ const RETRIEVAL = {
     KEYWORD_WEIGHT:     0.5,    // Weight applied to keyword (SQLite) results, 0 = disables, set >0 to enable and tune balance between semantic vs keyword matches
 }
 
+const UTILITY = {
+    DEFAULT_URL:   'http://localhost:11434', // Ollama host for background/utility tasks
+    DEFAULT_MODEL: 'qwen2.5:3b',
+    TEMPERATURE:   0.2,      // precise, low-creativity default for extraction/summaries
+    MAX_TOKENS:    1200,
+    TIMEOUT_MS:    120_000,  // extraction previously used 60s; summaries had none — standardized
+};
+
 module.exports = {
     QDRANT,
     COLLECTIONS,
@@ -119,4 +127,5 @@ module.exports = {
     SUMMARIES,
     ENTITIES,
     RETRIEVAL,
+    UTILITY,
 };

packages/shared/src/index.js

@@ -1,5 +1,20 @@
 const {getEnv} = require('./config/env');
-const {QDRANT, COLLECTIONS, EPISODIC, SERVICES, OLLAMA, PORTS, LLAMACPP, INFERENCE_DEFAULTS, SQLITE, ORCHESTRATION, SUMMARIES, ENTITIES, RETRIEVAL } = require('./config/constants');
+const { 
+        QDRANT, 
+        COLLECTIONS, 
+        EPISODIC, 
+        SERVICES, 
+        OLLAMA, 
+        PORTS, 
+        LLAMACPP, 
+        INFERENCE_DEFAULTS, 
+        SQLITE, 
+        ORCHESTRATION, 
+        SUMMARIES, 
+        ENTITIES, 
+        RETRIEVAL,
+        UTILITY
+     } = require('./config/constants');
 const {parseRow, formatEpisodeText, isTrivialTurn} = require('./utils')
 const logger = require('./utils/logger');
 
@@ -22,4 +37,5 @@ module.exports = {
     ENTITIES,
     logger,
     RETRIEVAL,
+    UTILITY,
 };