utilityInference

packages/inference-service/src/index.js

@@ -2,6 +2,7 @@ require ('dotenv').config();
 const express = require('express');
 const {getEnv, PORTS, OLLAMA, logger} = require('@nexusai/shared');
 const inferenceRouter = require('./routes/inference');
+const utilityRouter = require('./routes/utility')
 
 const app = express();
 app.use(express.json({ limit: '8mb' }));  // prompts include full context window
@@ -20,6 +21,7 @@ app.get('/health', (req, res) => {
     });
 });
 
+app.use('/', utilityRouter)
 app.use('/', inferenceRouter);
 
 // Start the server

packages/inference-service/src/routes/utility.js

@@ -0,0 +1,21 @@
+const { Router } = require('express');
+const { logger } = require('@nexusai/shared');
+const { utilityComplete } = require('../utility');
+
+const router = Router();
+
+router.post('/utility/complete', async (req, res) => {
+    const { system, user, json, temperature, maxTokens } = req.body;
+
+    if (!user) return res.status(400).json({ error: 'user message is required' });
+
+    try {
+        const result = await utilityComplete({ system, user, json, temperature, maxTokens });
+        res.json(result);
+    } catch (error) {
+        logger.error('[Utility] Completion error:', error.message);
+        res.status(500).json({ error: 'Utility inference failed', detail: error.message });
+    }
+});
+
+module.exports = router;

packages/inference-service/src/utility.js

@@ -0,0 +1,44 @@
+const { getEnv, UTILITY } = require ('@nexusai/shared')
+
+const UTILITY_URL = getEnv('UTILITY_URL', UTILITY.DEFAULT_URL);
+const UTILITY_MODEL = getEnv('UTILITY_MODEL', UTILITY.DEFAULT_MODEL);
+
+// Background task inference (extraction/summarization).  Uses ollama's /api/chat
+// so the model's own prompt template is server-side, no need for ChatML
+async function utilityComplete({
+    system,
+    user,
+    json = false,
+    temperature,
+    maxTokens
+}) {
+    const messages = [];
+    if(system) messages.push({ role: 'system', content: system});
+    messages.push ({role: 'user', content: user});
+    
+    const res = await fetch (`${UTILITY_URL}/api/chat`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json'},
+        body: JSON.stringify({
+            model: UTILITY_MODEL,
+            messages,
+            stream: false,
+            ...UTILITY(json && {format: 'json' }),
+            options: {
+                temperature: temperature ?? UTILITY.TEMPERATURE,
+                num_predict: maxTokens ?? UTILITY.MAX_TOKENS,
+            },
+        }),
+        signal: AbortSignal.timeout(UTILITY.TIMEOUT_MS),
+    });
+
+    if (!res.ok) throw new Error(`Utility backend responded ${res.status}`);
+    const data = await res.json();
+
+    return {
+        text: (data.message?.content ?? '').trim(),
+        model: data.model,
+    }
+}
+
+module.exports = { utilityComplete};