storme/nexusAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chat client clean up and switch to llama.cpp with models folder network sharing

Browse Source
This commit is contained in:
Storme-bit
2026-04-09 04:13:21 -07:00
parent 541e664da1
commit 5c6e027fc1
15 changed files with 305 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/chat-client/src/App.jsx
View File

@@ -10,7 +10,7 @@ import { DEFAULT_MODEL } from './config/constants';
export default function App() {
const [leftOpen, setLeftOpen] = useState(true);
const [rightOpen, setRightOpen] = useState(true);
const [selectedModel, setSelectedModel] = useState(DEFAULT_MODEL);
const { models, selectedModel, setSelectedModel } = useModels();
const {
sessions,
@@ -64,10 +64,12 @@ export default function App() {
isOpen={rightOpen}
onToggle={() => setRightOpen(o => !o)}
activeSession={activeSession}
models={models}
selectedModel={selectedModel}
onModelChange={setSelectedModel}
lastModel={lastModel}
lastTokenCount={lastTokenCount}
/>
</div>
);

6
packages/chat-client/src/api/orchestration.js
View File

@@ -80,3 +80,9 @@ export function streamMessage(sessionId, message, model, { onChunk, onDone, onEr
return () => controller.abort();
}
export async function fetchModels() {
const res = await fetch(`{BASE_URL}/models`);
if(!res.ok) throw new Error(`Failted to fetch models: ${res.status}`);
return res.json();
}

81
packages/chat-client/src/components/ChatWindow.jsx
View File

@@ -6,7 +6,6 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
const inputRef = useRef(null);
const [input, setInput] = React.useState('');
// Auto-scroll to bottom when messages change
useEffect(() => {
bottomRef.current?.scrollIntoView({ behavior: 'smooth' });
}, [messages]);
@@ -26,58 +25,30 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
}
return (
<div style={{
flex: 1,
display: 'flex',
flexDirection: 'column',
overflow: 'hidden',
background: 'var(--bg-base)',
}}>
<div className="flex-col flex-1 overflow-hidden" style={{ background: 'var(--bg-base)' }}>
{/* Header */}
<div style={{
height: 'var(--header-height)',
borderBottom: '1px solid var(--border)',
display: 'flex',
alignItems: 'center',
padding: '0 20px',
background: 'var(--bg-surface)',
flexShrink: 0,
}}>
<span style={{ color: 'var(--text-secondary)', fontSize: '13px' }}>
<div className="panel-header" style={{ padding: '0 20px' }}>
<span className="text-base text-secondary">
{activeSession ? activeSession.external_id : 'No session selected'}
</span>
</div>
{/* Message thread */}
<div style={{
flex: 1,
overflowY: 'auto',
padding: '20px 0',
}}>
<div className="flex-1 scroll-y" style={{ padding: '20px 0' }}>
{!activeSession && (
<div style={{
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
justifyContent: 'center',
<div className="flex-col items-center justify-center" style={{
height: '100%',
color: 'var(--text-muted)',
gap: '12px',
}}>
<div style={{ fontSize: '32px', opacity: 0.4 }}>✦</div>
<p style={{ fontSize: '14px' }}>Select a session or start a new chat</p>
<p className="text-base">Select a session or start a new chat</p>
</div>
)}
{loadingHistory && (
<div style={{
display: 'flex',
justifyContent: 'center',
padding: '40px',
color: 'var(--text-muted)',
fontSize: '13px',
}}>
<div className="flex justify-center text-muted" style={{ padding: '40px', fontSize: '13px' }}>
Loading history...
</div>
)}
@@ -96,13 +67,11 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
background: 'var(--bg-surface)',
flexShrink: 0,
}}>
<div style={{
display: 'flex',
<div className="flex items-end" style={{
gap: '10px',
alignItems: 'flex-end',
background: 'var(--bg-elevated)',
border: '1px solid var(--border)',
borderRadius: '12px',
borderRadius: 'var(--radius-lg)',
padding: '8px 12px',
}}>
<textarea
@@ -127,23 +96,17 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
overflowY: 'auto',
}}
onInput={e => {
// Auto-grow textarea
e.target.style.height = 'auto';
e.target.style.height = `${e.target.scrollHeight}px`;
}}
/>
{streaming ? (
<button onClick={onCancel} style={{
<button onClick={onCancel} className="btn-reset" style={{
background: 'var(--text-muted)',
border: 'none',
borderRadius: '8px',
borderRadius: 'var(--radius-md)',
width: '32px',
height: '32px',
cursor: 'pointer',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
flexShrink: 0,
color: 'white',
fontSize: '12px',
@@ -152,29 +115,19 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
<button
onClick={handleSend}
disabled={!activeSession || !input.trim()}
className="btn-primary"
style={{
background: activeSession && input.trim() ? 'var(--accent)' : 'var(--bg-elevated)',
border: '1px solid var(--border)',
borderRadius: '8px',
width: '32px',
height: '32px',
cursor: activeSession && input.trim() ? 'pointer' : 'default',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
flexShrink: 0,
color: activeSession && input.trim() ? 'white' : 'var(--text-muted)',
fontSize: '16px',
transition: 'background 0.15s',
}}>↑</button>
border: '1px solid var(--border)',
}}
>↑</button>
)}
</div>
<p style={{
fontSize: '11px',
color: 'var(--text-muted)',
textAlign: 'center',
marginTop: '8px',
}}>
<p className="text-xs text-muted" style={{ textAlign: 'center', marginTop: '8px' }}>
Enter to send · Shift+Enter for new line
</p>
</div>

85
packages/chat-client/src/components/InfoPanel.jsx
View File

@@ -1,54 +1,28 @@
import React from 'react';
import { MODELS } from '../config/constants';
export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel, lastTokenCount, selectedModel, onModelChange }) {
export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel, lastTokenCount, selectedModel, onModelChange, models }) {
return (
<div style={{
<div className="flex-col" style={{
width: isOpen ? 'var(--panel-width)' : '56px',
flexShrink: 0,
background: 'var(--bg-surface)',
borderLeft: '1px solid var(--border)',
display: 'flex',
flexDirection: 'column',
transition: 'width 0.2s ease',
overflow: 'hidden',
}}>
{/* Header */}
<div style={{
height: 'var(--header-height)',
display: 'flex',
alignItems: 'center',
<div className="panel-header" style={{
justifyContent: isOpen ? 'space-between' : 'center',
padding: isOpen ? '0 16px 0 12px' : '0',
borderBottom: '1px solid var(--border)',
flexShrink: 0,
}}>
<button onClick={onToggle} style={{
background: 'none',
border: 'none',
color: 'var(--text-muted)',
cursor: 'pointer',
padding: '6px',
borderRadius: '6px',
fontSize: '16px',
lineHeight: 1,
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
}}>
{isOpen ? '▶' : '◀'}
</button>
{isOpen && (
<span style={{ fontSize: '13px', fontWeight: 500, color: 'var(--text-secondary)' }}>
Session Info
</span>
)}
<button className="btn-icon" onClick={onToggle}>{isOpen ? '▶' : '◀'}</button>
{isOpen && <span className="text-base" style={{ fontWeight: 500, color: 'var(--text-secondary)' }}>Session Info</span>}
</div>
{isOpen && (
<div style={{ flex: 1, overflowY: 'auto', padding: '16px' }}>
<div className="flex-1 scroll-y" style={{ padding: '16px' }}>
{/* Model selector */}
<Section title="Model">
@@ -60,14 +34,14 @@ export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel,
padding: '8px 10px',
background: 'var(--bg-elevated)',
border: '1px solid var(--border)',
borderRadius: '8px',
borderRadius: 'var(--radius-md)',
color: 'var(--text-primary)',
fontSize: '13px',
cursor: 'pointer',
outline: 'none',
}}
>
{MODELS.map(m => (
{models.map(m => (
<option key={m.value} value={m.value}>{m.label}</option>
))}
</select>
@@ -76,44 +50,32 @@ export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel,
{/* Session details */}
<Section title="Session">
{activeSession ? (
<div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
<div className="flex-col" style={{ gap: '8px' }}>
<InfoRow label="ID" value={activeSession.external_id} mono truncate />
<InfoRow
label="Status"
value={activeSession.isNew ? 'Unsaved' : 'Active'}
accent={activeSession.isNew}
/>
<InfoRow label="Status" value={activeSession.isNew ? 'Unsaved' : 'Active'} accent={activeSession.isNew} />
</div>
) : (
<p style={{ fontSize: '12px', color: 'var(--text-muted)' }}>No session selected</p>
<p className="text-sm text-muted">No session selected</p>
)}
</Section>
{/* Last response stats */}
<Section title="Last Response">
{lastModel ? (
<div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
<div className="flex-col" style={{ gap: '8px' }}>
<InfoRow label="Model" value={lastModel} />
<InfoRow label="Tokens" value={lastTokenCount > 0 ? lastTokenCount.toLocaleString() : '—'} />
</div>
) : (
<p style={{ fontSize: '12px', color: 'var(--text-muted)' }}>No response yet</p>
<p className="text-sm text-muted">No response yet</p>
)}
</Section>
</div>
)}
{/* Collapsed — show icon indicators */}
{!isOpen && (
<div style={{
flex: 1,
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
paddingTop: '16px',
gap: '16px',
}}>
<div className="flex-col items-center" style={{ flex: 1, paddingTop: '16px', gap: '16px' }}>
<IconHint title="Model">M</IconHint>
<IconHint title="Session">S</IconHint>
</div>
@@ -122,21 +84,10 @@ export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel,
);
}
// ── Internal sub-components ──────────────────────────────────
function Section({ title, children }) {
return (
<div style={{ marginBottom: '24px' }}>
<p style={{
fontSize: '11px',
fontWeight: 500,
color: 'var(--text-muted)',
textTransform: 'uppercase',
letterSpacing: '0.08em',
marginBottom: '10px',
}}>
{title}
</p>
<p className="label-upper" style={{ marginBottom: '10px' }}>{title}</p>
{children}
</div>
);
@@ -144,8 +95,8 @@ function Section({ title, children }) {
function InfoRow({ label, value, mono, truncate, accent }) {
return (
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: '8px' }}>
<span style={{ fontSize: '12px', color: 'var(--text-muted)', flexShrink: 0 }}>{label}</span>
<div className="flex items-center" style={{ justifyContent: 'space-between', gap: '8px' }}>
<span className="text-sm text-muted flex-shrink">{label}</span>
<span style={{
fontSize: '12px',
color: accent ? 'var(--accent)' : 'var(--text-secondary)',
@@ -167,7 +118,7 @@ function IconHint({ title, children }) {
<div title={title} style={{
width: '32px',
height: '32px',
borderRadius: '8px',
borderRadius: 'var(--radius-md)',
background: 'var(--bg-elevated)',
border: '1px solid var(--border)',
display: 'flex',

17
packages/chat-client/src/components/MessageBubble.jsx
View File

@@ -4,25 +4,20 @@ export default function MessageBubble({ message }) {
const isUser = message.role === 'user';
return (
<div style={{
display: 'flex',
<div className="flex" style={{
justifyContent: isUser ? 'flex-end' : 'flex-start',
marginBottom: '12px',
padding: '0 16px',
}}>
{!isUser && (
<div style={{
<div className="flex items-center justify-center flex-shrink" style={{
width: '28px',
height: '28px',
borderRadius: '50%',
background: 'var(--accent)',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
fontSize: '12px',
fontWeight: 600,
marginRight: '8px',
flexShrink: 0,
alignSelf: 'flex-end',
}}>N</div>
)}
@@ -47,16 +42,12 @@ export default function MessageBubble({ message }) {
height: '14px',
background: 'var(--text-secondary)',
marginLeft: '2px',
borderRadius: '2px',
borderRadius: 'var(--radius-sm)',
animation: 'blink 1s step-end infinite',
}} />
)}
{message.error && (
<div style={{
marginTop: '6px',
fontSize: '12px',
color: '#ff6b6b',
}}>
<div className="text-xs" style={{ marginTop: '6px', color: '#ff6b6b' }}>
⚠ Failed to complete response
</div>
)}

84
packages/chat-client/src/components/SessionList.jsx
View File

@@ -23,89 +23,51 @@ export default function SessionList({ sessions, activeSession, onSelectSession,
flexShrink: 0,
background: 'var(--bg-surface)',
borderRight: '1px solid var(--border)',
display: 'flex',
flexDirection: 'column',
transition: 'width 0.2s ease',
overflow: 'hidden',
}}>
}} className="flex-col">
{/* Header */}
<div style={{
height: 'var(--header-height)',
display: 'flex',
alignItems: 'center',
<div className="panel-header" style={{
justifyContent: isOpen ? 'space-between' : 'center',
padding: isOpen ? '0 12px 0 16px' : '0',
borderBottom: '1px solid var(--border)',
flexShrink: 0,
}}>
{isOpen && (
<span style={{ fontSize: '13px', fontWeight: 500, color: 'var(--text-secondary)' }}>
Conversations
</span>
)}
<button onClick={onToggle} style={{
background: 'none',
border: 'none',
color: 'var(--text-muted)',
cursor: 'pointer',
padding: '6px',
borderRadius: '6px',
fontSize: '16px',
lineHeight: 1,
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
}}>
{isOpen ? '◀' : '▶'}
</button>
{isOpen && <span className="text-base" style={{ fontWeight: 500, color: 'var(--text-secondary)' }}>Conversations</span>}
<button className="btn-icon" onClick={onToggle}>{isOpen ? '◀' : '▶'}</button>
</div>
{/* New chat button */}
<div style={{ padding: isOpen ? '12px' : '12px 8px', flexShrink: 0 }}>
<button onClick={onNewChat} style={{
<button className="btn-primary" onClick={onNewChat} style={{
width: '100%',
padding: isOpen ? '8px 12px' : '8px',
background: 'var(--accent)',
border: 'none',
borderRadius: '8px',
color: 'white',
fontSize: '13px',
fontWeight: 500,
cursor: 'pointer',
display: 'flex',
alignItems: 'center',
justifyContent: isOpen ? 'flex-start' : 'center',
gap: '8px',
transition: 'background 0.15s',
whiteSpace: 'nowrap',
overflow: 'hidden',
}}
onMouseEnter={e => e.currentTarget.style.background = 'var(--accent-hover)'}
onMouseLeave={e => e.currentTarget.style.background = 'var(--accent)'}
>
}}>
<span style={{ fontSize: '18px', lineHeight: 1, flexShrink: 0 }}>+</span>
{isOpen && <span>New Chat</span>}
</button>
</div>
{/* Session list */}
<div style={{ flex: 1, overflowY: 'auto', overflowX: 'hidden' }}>
<div className="flex-1 scroll-y">
{isOpen && sessions.map(session => {
const isActive = activeSession?.external_id === session.external_id;
return (
<button
key={session.external_id}
onClick={() => onSelectSession(session)}
className="btn-reset"
style={{
width: '100%',
padding: '10px 16px',
background: isActive ? 'var(--bg-elevated)' : 'transparent',
border: 'none',
borderLeft: isActive ? '2px solid var(--accent)' : '2px solid transparent',
textAlign: 'left',
cursor: 'pointer',
display: 'flex',
flexDirection: 'column',
gap: '3px',
transition: 'background 0.1s',
@@ -113,45 +75,25 @@ export default function SessionList({ sessions, activeSession, onSelectSession,
onMouseEnter={e => { if (!isActive) e.currentTarget.style.background = 'var(--bg-elevated)'; }}
onMouseLeave={e => { if (!isActive) e.currentTarget.style.background = 'transparent'; }}
>
<div style={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
gap: '8px',
}}>
<span style={{
fontSize: '13px',
<div className="flex truncate" style={{ justifyContent: 'space-between', gap: '8px' }}>
<span className="text-base truncate" style={{
color: isActive ? 'var(--text-primary)' : 'var(--text-secondary)',
fontWeight: isActive ? 500 : 400,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
flex: 1,
}}>
{getPreview(session)}
</span>
<span style={{ fontSize: '11px', color: 'var(--text-muted)', flexShrink: 0 }}>
{formatDate(session.updated_at)}
</span>
<span className="text-xs text-muted flex-shrink">{formatDate(session.updated_at)}</span>
</div>
{session.isNew && (
<span style={{
fontSize: '11px',
color: 'var(--accent)',
fontStyle: 'italic',
}}>Unsaved</span>
<span className="text-xs text-accent" style={{ fontStyle: 'italic' }}>Unsaved</span>
)}
</button>
);
})}
{isOpen && sessions.length === 0 && (
<div style={{
padding: '24px 16px',
color: 'var(--text-muted)',
fontSize: '13px',
textAlign: 'center',
}}>
<div className="text-base text-muted" style={{ padding: '24px 16px', textAlign: 'center' }}>
No conversations yet
</div>
)}

2
packages/chat-client/src/config/constants.js
View File

@@ -1,4 +1,4 @@
export const MODELS = [
export const FALLBACK_MODELS = [
{ value: 'companion:latest', label: 'Companion' },
{ value: 'mistral-nemo:latest', label: 'Mistral Nemo' },
{ value: 'coder:latest', label: 'Coder' },

24
packages/chat-client/src/hooks/useModels.js Normal file
View File

@@ -0,0 +1,24 @@
// hooks/useModels.js
import { useState, useEffect } from 'react';
import { fetchModels } from '../api/orchestration';
import { FALLBACK_MODELS, DEFAULT_MODEL } from '../config/constants';
export function useModels() {
const [models, setModels] = useState(FALLBACK_MODELS);
const [selectedModel, setSelectedModel] = useState(DEFAULT_MODEL);
const [loading, setLoading] = useState(true);
useEffect(() => {
fetchModels()
.then(data => {
setModels(data);
setSelectedModel(data[0]?.value ?? DEFAULT_MODEL);
})
.catch(err => {
console.warn('[useModels] Falling back to static list:', err.message);
})
.finally(() => setLoading(false));
}, []);
return { models, selectedModel, setSelectedModel, loading };
}

82
packages/chat-client/src/index.css
View File

@@ -15,6 +15,9 @@
--sidebar-width: 280px;
--panel-width: 260px;
--header-height: 56px;
--radius-sm: 6px;
--radius-md: 8px;
--radius-lg: 12px;
}
html, body, #root {
@@ -26,6 +29,81 @@ html, body, #root {
}
@keyframes blink {
0%, 100% { opacity: 1; }
50% { opacity: 0; }
0%, 100% { opacity: 1; }
50% { opacity: 0; }
}
/* ── Layout ─────────────────────────────────────────── */
.flex { display: flex; }
.flex-col { display: flex; flex-direction: column; }
.flex-1 { flex: 1; }
.flex-shrink { flex-shrink: 0; }
.items-center { align-items: center; }
.justify-center { justify-content: center; }
.justify-between { justify-content: space-between; }
.overflow-hidden { overflow: hidden; }
.scroll-y { overflow-y: auto; overflow-x: hidden; }
/* ── Panel header — shared by all three sidebars ────── */
.panel-header {
height: var(--header-height);
display: flex;
align-items: center;
border-bottom: 1px solid var(--border);
flex-shrink: 0;
background: var(--bg-surface);
}
/* ── Button resets ──────────────────────────────────── */
.btn-reset {
background: none;
border: none;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
}
.btn-icon {
background: none;
border: none;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
padding: 6px;
border-radius: var(--radius-sm);
color: var(--text-muted);
font-size: 16px;
line-height: 1;
}
.btn-icon:hover { background: var(--bg-elevated); }
.btn-primary {
background: var(--accent);
border: none;
border-radius: var(--radius-md);
color: white;
cursor: pointer;
font-size: 13px;
font-weight: 500;
transition: background 0.15s;
}
.btn-primary:hover { background: var(--accent-hover); }
.btn-primary:disabled { background: var(--bg-elevated); color: var(--text-muted); cursor: default; }
/* ── Typography helpers ─────────────────────────────── */
.text-xs { font-size: 11px; }
.text-sm { font-size: 12px; }
.text-base { font-size: 13px; }
.text-muted { color: var(--text-muted); }
.text-secondary { color: var(--text-secondary); }
.text-accent { color: var(--accent); }
.label-upper { font-size: 11px; font-weight: 500; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.08em; }
.truncate { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }

128
packages/inference-service/src/providers/llamacpp.js
View File

@@ -1,80 +1,90 @@
const { getEnv, LLAMACPP, INFERENCE_DEFAULTS } = require('@nexusai/shared');
const { getEnv, LLAMACPP, INFERENCE_DEFAULTS } = require("@nexusai/shared");
const BASE_URL = getEnv('INFERENCE_URL', LLAMACPP.DEFAULT_URL);
const DEFAULT_MODEL = getEnv('DEFAULT_MODEL', LLAMACPP.DEFAULT_MODEL);
const BASE_URL = getEnv("INFERENCE_URL", LLAMACPP.DEFAULT_URL);
const DEFAULT_MODEL = getEnv("DEFAULT_MODEL", LLAMACPP.DEFAULT_MODEL);
function resolveOptions(options) {
return {
temperature: options.temperature ?? INFERENCE_DEFAULTS.TEMPERATURE,
maxTokens: options.maxTokens ?? INFERENCE_DEFAULTS.MAX_TOKENS,
topP: options.topP ?? INFERENCE_DEFAULTS.TOP_P,
topK: options.topK ?? INFERENCE_DEFAULTS.TOP_K,
repeatPenalty: options.repeatPenalty ?? INFERENCE_DEFAULTS.REPEAT_PENALTY,
seed: options.seed ?? INFERENCE_DEFAULTS.SEED,
};
return {
temperature: options.temperature ?? INFERENCE_DEFAULTS.TEMPERATURE,
maxTokens: options.maxTokens ?? INFERENCE_DEFAULTS.MAX_TOKENS,
topP: options.topP ?? INFERENCE_DEFAULTS.TOP_P,
topK: options.topK ?? INFERENCE_DEFAULTS.TOP_K,
repeatPenalty: options.repeatPenalty ?? INFERENCE_DEFAULTS.REPEAT_PENALTY,
seed: options.seed ?? INFERENCE_DEFAULTS.SEED,
};
}
function buildPayload(prompt, options, stream = false){
const opts = resolveOptions(options);
function buildPayload(prompt, options, stream = false) {
const opts = resolveOptions(options);
return {
model: options.model || DEFAULT_MODEL,
messages: [{ role: 'user', content: prompt }],
temperature: opts.temperature,
max_tokens: opts.maxTokens,
top_p: opts.topP,
top_k: opts.topK,
repeat_penalty: opts.repeatPenalty,
stream,
...(opts.seed !== null && { seed: opts.seed }),
};
return {
model: options.model || DEFAULT_MODEL,
messages: [{ role: "user", content: prompt }],
temperature: opts.temperature,
max_tokens: opts.maxTokens,
top_p: opts.topP,
top_k: opts.topK,
repeat_penalty: opts.repeatPenalty,
stream,
...(opts.seed !== null && { seed: opts.seed }),
};
}
async function complete(prompt, options = {} ) {
const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(buildPayload(prompt, options, false))
})
async function complete(prompt, options = {}) {
const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(buildPayload(prompt, options, false)),
});
if (!res.ok) throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
if (!res.ok)
throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
const data = await res.json();
const choice = data.choices[0];
const data = await res.json();
const choice = data.choices[0];
return {
text: choice.message.content,
model: data.model,
done: choice.finish_reason === 'stop',
evalCount: data.usage?.completion_tokens,
promptEvalCount: data.usage?.prompt_tokens,
}
return {
text: choice.message.content,
model: data.model,
done: choice.finish_reason === "stop",
evalCount: data.usage?.completion_tokens,
promptEvalCount: data.usage?.prompt_tokens,
};
}
async function* completeStream(prompt, options = {}) {
const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(buildPayload(prompt, options, true))
});
let finalModel = DEFAULT_MODEL;
let finalTokenCount = 0;
if (!res.ok) throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(buildPayload(prompt, options, true)),
});
//OpenAI streaming sends newline-delimited JSON (NDJSON) with "data: " prefix for each chunk
//Example chunk: data: {"choices":[{"delta":{"content":"Hello"},"finish_reason":null,"index":0}]}
//we parse each chunk as it arrives
for await (const chunk of res.body){
const lines = Buffer.from(chunk).toString('utf8')
.split('\n')
.filter(l => l.startsWith('data: ') && l !== 'data: [DONE]');
if (!res.ok)
throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
for (const line of lines) {
const json = JSON.parse(line.slice(6)); //remove 'data: ' prefix
const delta = json.choices?.[0]?.delta?.content;
if (delta) yield {response: delta, done: false};
}
for await (const chunk of res.body) {
const lines = Buffer.from(chunk)
.toString("utf8")
.split("\n")
.filter((l) => l.startsWith("data: ") && l !== "data: [DONE]");
for (const line of lines) {
const json = JSON.parse(line.slice(6));
const delta = json.choices?.[0]?.delta?.content;
// Capture final metadata from the stop chunk
if (json.choices?.[0]?.finish_reason === "stop") {
finalModel = json.model ?? finalModel;
finalTokenCount = json.usage?.completion_tokens ?? finalTokenCount;
}
if (delta) yield { response: delta, done: false };
}
yield { response: '', done: true}; //signal completion at the end of the stream
}
yield { response: '', done: true, model: finalModel, tokenCount: finalTokenCount };
}
module.exports = { complete, completeStream };

28
packages/inference-service/src/routes/inference.js
View File

@@ -24,22 +24,34 @@ router.post('/complete', async (req, res) => {
router.post('/complete/stream', async (req, res) => {
const { prompt, model, temperature } = req.body;
if (!prompt) {
return res.status(400).json({error: 'prompt is required'});
}
if (!prompt) return res.status(400).json({ error: 'prompt is required' });
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
try {
for await (const chunk of completeStream(prompt, {model, temperature})) {
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
let lastModel = model;
let tokenCount = 0;
for await (const chunk of completeStream(prompt, { model, temperature })) {
if (chunk.response) {
res.write(`data: ${JSON.stringify({ response: chunk.response })}\n\n`);
}
if (chunk.done) {
// capture final metadata from the done signal
lastModel = chunk.model ?? lastModel;
tokenCount = chunk.tokenCount ?? tokenCount;
}
}
// Send a single done event with metadata after stream closes
res.write(`data: ${JSON.stringify({ done: true, model: lastModel, tokenCount })}\n\n`);
res.write('data: [DONE]\n\n');
} catch (error) {
console.error('[Inference] Streaming error:', error.message);
res.write(`data: ${JSON.stringify({ error: error.message })}\n\n`);
} catch (err) {
console.error('[Inference] Streaming error:', err.message);
res.write(`data: ${JSON.stringify({ error: err.message })}\n\n`);
} finally {
res.end();
}

24
packages/orchestration-service/src/chat/index.js
View File

@@ -109,31 +109,35 @@ async function chatStream(externalId, userMessage, onChunk, options = {} ) {
let tokenCount = 0;
// 5. Parse SSE chunks
for await (const chunk of res.body){
// Replace the current SSE parsing block in chatStream:
for await (const chunk of res.body) {
const lines = chunk.toString().split('\n');
for (const line of lines) {
if (!line.startsWith('data: ')) continue;
const raw = line.slice(6).trim();
if (raw === '[DONE]') continue //stream closed sentinel
if (raw === '[DONE]') continue;
try {
const data = JSON.parse(raw);
if (data.model) model = data.model
// llama.cpp provider shape: { response, done }
if (data.response) {
fullText += data.response;
onChunk(data.response);
}
if (data.done && data.eval_count !== undefined) {
tokenCount = (data.eval_count || 0) + (data.prompt_eval_count || 0)
}
} catch {
//partial chunk
//skip and wait for next
}
// model comes through on done chunk from inference route
if (data.model) model = data.model;
// token count — inference.js route sends this on the done chunk
if (data.done && data.tokenCount !== undefined) {
tokenCount = data.tokenCount;
}
} catch {
// partial chunk — skip
}
}
}

2
packages/orchestration-service/src/index.js
View File

@@ -3,6 +3,7 @@ const express = require('express');
const {getEnv, PORTS, SERVICES, ORCHESTRATION} = require('@nexusai/shared');
const chatRouter = require('./routes/chat');
const sessionsRouter = require('./routes/sessions');
const modelsRouter = require('./routes/models')
const cors = require('cors');
const app = express();
@@ -35,6 +36,7 @@ app.get('/health', (req, res) => {
app.use('/chat', chatRouter);
app.use('/sessions', sessionsRouter);
app.use('/models', modelsRouter);
/******* Start the server ************/
app.listen(PORT, () => {

12
packages/orchestration-service/src/routes/chat.js
View File

@@ -36,10 +36,14 @@ router.post('/stream', async (req, res) => {
res.flushHeaders();
try {
await chatStream(sessionId, message, (delta) => {
res.write(`data: ${JSON.stringify({ text: delta})}\n\n`)
})
res.write(`data: ${JSON.stringify({done: true})}\n\n`);
const { model, tokenCount } = await chatStream(
sessionId,
message,
(delta) => { res.write(`data: ${JSON.stringify({ text: delta })}\n\n`) },
{ model: req.body.model, temperature: req.body.temperature }
);
res.write(`data: ${JSON.stringify({ done: true, model, tokenCount })}\n\n`);
} catch (err) {
res.write(`data: ${JSON.stringify({error: err.message})}\n\n`);
} finally {

21
packages/orchestration-service/src/routes/models.js Normal file
View File

@@ -0,0 +1,21 @@
// routes/models.js
const express = require('express');
const router = express.Router();
const fs = require('fs');
const path = require('path');
const {getEnv} = require('@nexusai/shared');
const MODELS_PATH = getEnv('MODELS_MANIFEST_PATH', path.join(__dirname, '../models.json'));
router.get('/', (req, res) => {
try {
const raw = fs.readFileSync(MODELS_PATH, 'utf8');
const models = JSON.parse(raw);
res.json(models);
} catch (err) {
console.error('[models] Failed to read manifest:', err.message);
res.status(500).json({ error: 'Could not load models manifest' });
}
});
module.exports = router;