chat client clean up and switch to llama.cpp with models folder network sharing

packages/chat-client/src/App.jsx

@@ -10,7 +10,7 @@ import { DEFAULT_MODEL } from './config/constants';
 export default function App() {
   const [leftOpen, setLeftOpen] = useState(true);
   const [rightOpen, setRightOpen] = useState(true);
-  const [selectedModel, setSelectedModel] = useState(DEFAULT_MODEL);
+  const { models, selectedModel, setSelectedModel } = useModels();
 
   const {
     sessions,
@@ -64,10 +64,12 @@ export default function App() {
         isOpen={rightOpen}
         onToggle={() => setRightOpen(o => !o)}
         activeSession={activeSession}
+        models={models}
         selectedModel={selectedModel}
         onModelChange={setSelectedModel}
         lastModel={lastModel}
         lastTokenCount={lastTokenCount}
+
       />
     </div>
   );

packages/chat-client/src/api/orchestration.js

@@ -80,3 +80,9 @@ export function streamMessage(sessionId, message, model, { onChunk, onDone, onEr
 
   return () => controller.abort();
 }
+
+export async function fetchModels() {
+  const res = await fetch(`{BASE_URL}/models`);
+  if(!res.ok) throw new Error(`Failted to fetch models: ${res.status}`);
+  return res.json();
+}

packages/chat-client/src/components/ChatWindow.jsx

@@ -6,7 +6,6 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
   const inputRef = useRef(null);
   const [input, setInput] = React.useState('');
 
-  // Auto-scroll to bottom when messages change
   useEffect(() => {
     bottomRef.current?.scrollIntoView({ behavior: 'smooth' });
   }, [messages]);
@@ -26,58 +25,30 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
   }
 
   return (
-    <div style={{
+    <div className="flex-col flex-1 overflow-hidden" style={{ background: 'var(--bg-base)' }}>
-      flex: 1,
-      display: 'flex',
-      flexDirection: 'column',
-      overflow: 'hidden',
-      background: 'var(--bg-base)',
-    }}>
 
       {/* Header */}
-      <div style={{
+      <div className="panel-header" style={{ padding: '0 20px' }}>
-        height: 'var(--header-height)',
+        <span className="text-base text-secondary">
-        borderBottom: '1px solid var(--border)',
-        display: 'flex',
-        alignItems: 'center',
-        padding: '0 20px',
-        background: 'var(--bg-surface)',
-        flexShrink: 0,
-      }}>
-        <span style={{ color: 'var(--text-secondary)', fontSize: '13px' }}>
           {activeSession ? activeSession.external_id : 'No session selected'}
         </span>
       </div>
 
       {/* Message thread */}
-      <div style={{
+      <div className="flex-1 scroll-y" style={{ padding: '20px 0' }}>
-        flex: 1,
-        overflowY: 'auto',
-        padding: '20px 0',
-      }}>
         {!activeSession && (
-          <div style={{
+          <div className="flex-col items-center justify-center" style={{
-            display: 'flex',
-            flexDirection: 'column',
-            alignItems: 'center',
-            justifyContent: 'center',
             height: '100%',
             color: 'var(--text-muted)',
             gap: '12px',
           }}>
             <div style={{ fontSize: '32px', opacity: 0.4 }}>✦</div>
-            <p style={{ fontSize: '14px' }}>Select a session or start a new chat</p>
+            <p className="text-base">Select a session or start a new chat</p>
           </div>
         )}
 
         {loadingHistory && (
-          <div style={{
+          <div className="flex justify-center text-muted" style={{ padding: '40px', fontSize: '13px' }}>
-            display: 'flex',
-            justifyContent: 'center',
-            padding: '40px',
-            color: 'var(--text-muted)',
-            fontSize: '13px',
-          }}>
             Loading history...
           </div>
         )}
@@ -96,13 +67,11 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
         background: 'var(--bg-surface)',
         flexShrink: 0,
       }}>
-        <div style={{
+        <div className="flex items-end" style={{
-          display: 'flex',
           gap: '10px',
-          alignItems: 'flex-end',
           background: 'var(--bg-elevated)',
           border: '1px solid var(--border)',
-          borderRadius: '12px',
+          borderRadius: 'var(--radius-lg)',
           padding: '8px 12px',
         }}>
           <textarea
@@ -127,23 +96,17 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
               overflowY: 'auto',
             }}
             onInput={e => {
-              // Auto-grow textarea
               e.target.style.height = 'auto';
               e.target.style.height = `${e.target.scrollHeight}px`;
             }}
           />
 
           {streaming ? (
-            <button onClick={onCancel} style={{
+            <button onClick={onCancel} className="btn-reset" style={{
               background: 'var(--text-muted)',
-              border: 'none',
+              borderRadius: 'var(--radius-md)',
-              borderRadius: '8px',
               width: '32px',
               height: '32px',
-              cursor: 'pointer',
-              display: 'flex',
-              alignItems: 'center',
-              justifyContent: 'center',
               flexShrink: 0,
               color: 'white',
               fontSize: '12px',
@@ -152,29 +115,19 @@ export default function ChatWindow({ messages, loadingHistory, streaming, onSend
             <button
               onClick={handleSend}
               disabled={!activeSession || !input.trim()}
+              className="btn-primary"
               style={{
-                background: activeSession && input.trim() ? 'var(--accent)' : 'var(--bg-elevated)',
-                border: '1px solid var(--border)',
-                borderRadius: '8px',
                 width: '32px',
                 height: '32px',
-                cursor: activeSession && input.trim() ? 'pointer' : 'default',
-                display: 'flex',
-                alignItems: 'center',
-                justifyContent: 'center',
                 flexShrink: 0,
-                color: activeSession && input.trim() ? 'white' : 'var(--text-muted)',
                 fontSize: '16px',
-                transition: 'background 0.15s',
+                border: '1px solid var(--border)',
-              }}>↑</button>
+              }}
+            >↑</button>
           )}
         </div>
-        <p style={{
+
-          fontSize: '11px',
+        <p className="text-xs text-muted" style={{ textAlign: 'center', marginTop: '8px' }}>
-          color: 'var(--text-muted)',
-          textAlign: 'center',
-          marginTop: '8px',
-        }}>
           Enter to send · Shift+Enter for new line
         </p>
       </div>

packages/chat-client/src/components/InfoPanel.jsx

@@ -1,54 +1,28 @@
 import React from 'react';
 import { MODELS } from '../config/constants';
 
-export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel, lastTokenCount, selectedModel, onModelChange }) {
+export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel, lastTokenCount, selectedModel, onModelChange, models }) {
-
   return (
-    <div style={{
+    <div className="flex-col" style={{
       width: isOpen ? 'var(--panel-width)' : '56px',
       flexShrink: 0,
       background: 'var(--bg-surface)',
       borderLeft: '1px solid var(--border)',
-      display: 'flex',
-      flexDirection: 'column',
       transition: 'width 0.2s ease',
       overflow: 'hidden',
     }}>
 
       {/* Header */}
-      <div style={{
+      <div className="panel-header" style={{
-        height: 'var(--header-height)',
-        display: 'flex',
-        alignItems: 'center',
         justifyContent: isOpen ? 'space-between' : 'center',
         padding: isOpen ? '0 16px 0 12px' : '0',
-        borderBottom: '1px solid var(--border)',
-        flexShrink: 0,
       }}>
-        <button onClick={onToggle} style={{
+        <button className="btn-icon" onClick={onToggle}>{isOpen ? '▶' : '◀'}</button>
-          background: 'none',
+        {isOpen && <span className="text-base" style={{ fontWeight: 500, color: 'var(--text-secondary)' }}>Session Info</span>}
-          border: 'none',
-          color: 'var(--text-muted)',
-          cursor: 'pointer',
-          padding: '6px',
-          borderRadius: '6px',
-          fontSize: '16px',
-          lineHeight: 1,
-          display: 'flex',
-          alignItems: 'center',
-          justifyContent: 'center',
-        }}>
-          {isOpen ? '▶' : '◀'}
-        </button>
-        {isOpen && (
-          <span style={{ fontSize: '13px', fontWeight: 500, color: 'var(--text-secondary)' }}>
-            Session Info
-          </span>
-        )}
       </div>
 
       {isOpen && (
-        <div style={{ flex: 1, overflowY: 'auto', padding: '16px' }}>
+        <div className="flex-1 scroll-y" style={{ padding: '16px' }}>
 
           {/* Model selector */}
           <Section title="Model">
@@ -60,14 +34,14 @@ export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel,
                 padding: '8px 10px',
                 background: 'var(--bg-elevated)',
                 border: '1px solid var(--border)',
-                borderRadius: '8px',
+                borderRadius: 'var(--radius-md)',
                 color: 'var(--text-primary)',
                 fontSize: '13px',
                 cursor: 'pointer',
                 outline: 'none',
               }}
             >
-              {MODELS.map(m => (
+              {models.map(m => (
                 <option key={m.value} value={m.value}>{m.label}</option>
               ))}
             </select>
@@ -76,44 +50,32 @@ export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel,
           {/* Session details */}
           <Section title="Session">
             {activeSession ? (
-              <div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
+              <div className="flex-col" style={{ gap: '8px' }}>
                 <InfoRow label="ID" value={activeSession.external_id} mono truncate />
-                <InfoRow
+                <InfoRow label="Status" value={activeSession.isNew ? 'Unsaved' : 'Active'} accent={activeSession.isNew} />
-                  label="Status"
-                  value={activeSession.isNew ? 'Unsaved' : 'Active'}
-                  accent={activeSession.isNew}
-                />
               </div>
             ) : (
-              <p style={{ fontSize: '12px', color: 'var(--text-muted)' }}>No session selected</p>
+              <p className="text-sm text-muted">No session selected</p>
             )}
           </Section>
 
           {/* Last response stats */}
           <Section title="Last Response">
             {lastModel ? (
-              <div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
+              <div className="flex-col" style={{ gap: '8px' }}>
                 <InfoRow label="Model"  value={lastModel} />
                 <InfoRow label="Tokens" value={lastTokenCount > 0 ? lastTokenCount.toLocaleString() : '—'} />
               </div>
             ) : (
-              <p style={{ fontSize: '12px', color: 'var(--text-muted)' }}>No response yet</p>
+              <p className="text-sm text-muted">No response yet</p>
             )}
           </Section>
 
         </div>
       )}
 
-      {/* Collapsed — show icon indicators */}
       {!isOpen && (
-        <div style={{
+        <div className="flex-col items-center" style={{ flex: 1, paddingTop: '16px', gap: '16px' }}>
-          flex: 1,
-          display: 'flex',
-          flexDirection: 'column',
-          alignItems: 'center',
-          paddingTop: '16px',
-          gap: '16px',
-        }}>
           <IconHint title="Model">M</IconHint>
           <IconHint title="Session">S</IconHint>
         </div>
@@ -122,21 +84,10 @@ export default function InfoPanel({ isOpen, onToggle, activeSession, lastModel,
   );
 }
 
-// ── Internal sub-components ──────────────────────────────────
-
 function Section({ title, children }) {
   return (
     <div style={{ marginBottom: '24px' }}>
-      <p style={{
+      <p className="label-upper" style={{ marginBottom: '10px' }}>{title}</p>
-        fontSize: '11px',
-        fontWeight: 500,
-        color: 'var(--text-muted)',
-        textTransform: 'uppercase',
-        letterSpacing: '0.08em',
-        marginBottom: '10px',
-      }}>
-        {title}
-      </p>
       {children}
     </div>
   );
@@ -144,8 +95,8 @@ function Section({ title, children }) {
 
 function InfoRow({ label, value, mono, truncate, accent }) {
   return (
-    <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: '8px' }}>
+    <div className="flex items-center" style={{ justifyContent: 'space-between', gap: '8px' }}>
-      <span style={{ fontSize: '12px', color: 'var(--text-muted)', flexShrink: 0 }}>{label}</span>
+      <span className="text-sm text-muted flex-shrink">{label}</span>
       <span style={{
         fontSize: '12px',
         color: accent ? 'var(--accent)' : 'var(--text-secondary)',
@@ -167,7 +118,7 @@ function IconHint({ title, children }) {
     <div title={title} style={{
       width: '32px',
       height: '32px',
-      borderRadius: '8px',
+      borderRadius: 'var(--radius-md)',
       background: 'var(--bg-elevated)',
       border: '1px solid var(--border)',
       display: 'flex',

packages/chat-client/src/components/MessageBubble.jsx

@@ -4,25 +4,20 @@ export default function MessageBubble({ message }) {
   const isUser = message.role === 'user';
 
   return (
-    <div style={{
+    <div className="flex" style={{
-      display: 'flex',
       justifyContent: isUser ? 'flex-end' : 'flex-start',
       marginBottom: '12px',
       padding: '0 16px',
     }}>
       {!isUser && (
-        <div style={{
+        <div className="flex items-center justify-center flex-shrink" style={{
           width: '28px',
           height: '28px',
           borderRadius: '50%',
           background: 'var(--accent)',
-          display: 'flex',
-          alignItems: 'center',
-          justifyContent: 'center',
           fontSize: '12px',
           fontWeight: 600,
           marginRight: '8px',
-          flexShrink: 0,
           alignSelf: 'flex-end',
         }}>N</div>
       )}
@@ -47,16 +42,12 @@ export default function MessageBubble({ message }) {
             height: '14px',
             background: 'var(--text-secondary)',
             marginLeft: '2px',
-            borderRadius: '2px',
+            borderRadius: 'var(--radius-sm)',
             animation: 'blink 1s step-end infinite',
           }} />
         )}
         {message.error && (
-          <div style={{
+          <div className="text-xs" style={{ marginTop: '6px', color: '#ff6b6b' }}>
-            marginTop: '6px',
-            fontSize: '12px',
-            color: '#ff6b6b',
-          }}>
             ⚠ Failed to complete response
           </div>
         )}

packages/chat-client/src/components/SessionList.jsx

@@ -23,89 +23,51 @@ export default function SessionList({ sessions, activeSession, onSelectSession,
       flexShrink: 0,
       background: 'var(--bg-surface)',
       borderRight: '1px solid var(--border)',
-      display: 'flex',
-      flexDirection: 'column',
       transition: 'width 0.2s ease',
       overflow: 'hidden',
-    }}>
+    }} className="flex-col">
 
       {/* Header */}
-      <div style={{
+      <div className="panel-header" style={{
-        height: 'var(--header-height)',
-        display: 'flex',
-        alignItems: 'center',
         justifyContent: isOpen ? 'space-between' : 'center',
         padding: isOpen ? '0 12px 0 16px' : '0',
-        borderBottom: '1px solid var(--border)',
-        flexShrink: 0,
       }}>
-        {isOpen && (
+        {isOpen && <span className="text-base" style={{ fontWeight: 500, color: 'var(--text-secondary)' }}>Conversations</span>}
-          <span style={{ fontSize: '13px', fontWeight: 500, color: 'var(--text-secondary)' }}>
+        <button className="btn-icon" onClick={onToggle}>{isOpen ? '◀' : '▶'}</button>
-            Conversations
-          </span>
-        )}
-        <button onClick={onToggle} style={{
-          background: 'none',
-          border: 'none',
-          color: 'var(--text-muted)',
-          cursor: 'pointer',
-          padding: '6px',
-          borderRadius: '6px',
-          fontSize: '16px',
-          lineHeight: 1,
-          display: 'flex',
-          alignItems: 'center',
-          justifyContent: 'center',
-        }}>
-          {isOpen ? '◀' : '▶'}
-        </button>
       </div>
 
       {/* New chat button */}
       <div style={{ padding: isOpen ? '12px' : '12px 8px', flexShrink: 0 }}>
-        <button onClick={onNewChat} style={{
+        <button className="btn-primary" onClick={onNewChat} style={{
           width: '100%',
           padding: isOpen ? '8px 12px' : '8px',
-          background: 'var(--accent)',
-          border: 'none',
-          borderRadius: '8px',
-          color: 'white',
-          fontSize: '13px',
-          fontWeight: 500,
-          cursor: 'pointer',
           display: 'flex',
           alignItems: 'center',
           justifyContent: isOpen ? 'flex-start' : 'center',
           gap: '8px',
-          transition: 'background 0.15s',
           whiteSpace: 'nowrap',
           overflow: 'hidden',
-        }}
+        }}>
-          onMouseEnter={e => e.currentTarget.style.background = 'var(--accent-hover)'}
-          onMouseLeave={e => e.currentTarget.style.background = 'var(--accent)'}
-        >
           <span style={{ fontSize: '18px', lineHeight: 1, flexShrink: 0 }}>+</span>
           {isOpen && <span>New Chat</span>}
         </button>
       </div>
 
       {/* Session list */}
-      <div style={{ flex: 1, overflowY: 'auto', overflowX: 'hidden' }}>
+      <div className="flex-1 scroll-y">
         {isOpen && sessions.map(session => {
           const isActive = activeSession?.external_id === session.external_id;
           return (
             <button
               key={session.external_id}
               onClick={() => onSelectSession(session)}
+              className="btn-reset"
               style={{
                 width: '100%',
                 padding: '10px 16px',
                 background: isActive ? 'var(--bg-elevated)' : 'transparent',
-                border: 'none',
                 borderLeft: isActive ? '2px solid var(--accent)' : '2px solid transparent',
                 textAlign: 'left',
-                cursor: 'pointer',
-                display: 'flex',
                 flexDirection: 'column',
                 gap: '3px',
                 transition: 'background 0.1s',
@@ -113,45 +75,25 @@ export default function SessionList({ sessions, activeSession, onSelectSession,
               onMouseEnter={e => { if (!isActive) e.currentTarget.style.background = 'var(--bg-elevated)'; }}
               onMouseLeave={e => { if (!isActive) e.currentTarget.style.background = 'transparent'; }}
             >
-              <div style={{
+              <div className="flex truncate" style={{ justifyContent: 'space-between', gap: '8px' }}>
-                display: 'flex',
+                <span className="text-base truncate" style={{
-                justifyContent: 'space-between',
-                alignItems: 'center',
-                gap: '8px',
-              }}>
-                <span style={{
-                  fontSize: '13px',
                   color: isActive ? 'var(--text-primary)' : 'var(--text-secondary)',
                   fontWeight: isActive ? 500 : 400,
-                  overflow: 'hidden',
-                  textOverflow: 'ellipsis',
-                  whiteSpace: 'nowrap',
                   flex: 1,
                 }}>
                   {getPreview(session)}
                 </span>
-                <span style={{ fontSize: '11px', color: 'var(--text-muted)', flexShrink: 0 }}>
+                <span className="text-xs text-muted flex-shrink">{formatDate(session.updated_at)}</span>
-                  {formatDate(session.updated_at)}
-                </span>
               </div>
               {session.isNew && (
-                <span style={{
+                <span className="text-xs text-accent" style={{ fontStyle: 'italic' }}>Unsaved</span>
-                  fontSize: '11px',
-                  color: 'var(--accent)',
-                  fontStyle: 'italic',
-                }}>Unsaved</span>
               )}
             </button>
           );
         })}
 
         {isOpen && sessions.length === 0 && (
-          <div style={{
+          <div className="text-base text-muted" style={{ padding: '24px 16px', textAlign: 'center' }}>
-            padding: '24px 16px',
-            color: 'var(--text-muted)',
-            fontSize: '13px',
-            textAlign: 'center',
-          }}>
             No conversations yet
           </div>
         )}

packages/chat-client/src/config/constants.js

@@ -1,4 +1,4 @@
-export const MODELS = [
+export const FALLBACK_MODELS = [
     { value: 'companion:latest',    label: 'Companion' },
     { value: 'mistral-nemo:latest', label: 'Mistral Nemo' },
     { value: 'coder:latest',        label: 'Coder' },

packages/chat-client/src/hooks/useModels.js

@@ -0,0 +1,24 @@
+// hooks/useModels.js
+import { useState, useEffect } from 'react';
+import { fetchModels } from '../api/orchestration';
+import { FALLBACK_MODELS, DEFAULT_MODEL } from '../config/constants';
+
+export function useModels() {
+  const [models, setModels] = useState(FALLBACK_MODELS);
+  const [selectedModel, setSelectedModel] = useState(DEFAULT_MODEL);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    fetchModels()
+      .then(data => {
+        setModels(data);
+        setSelectedModel(data[0]?.value ?? DEFAULT_MODEL);
+      })
+      .catch(err => {
+        console.warn('[useModels] Falling back to static list:', err.message);
+      })
+      .finally(() => setLoading(false));
+  }, []);
+
+  return { models, selectedModel, setSelectedModel, loading };
+}

packages/chat-client/src/index.css

@@ -15,6 +15,9 @@
   --sidebar-width:  280px;
   --panel-width:    260px;
   --header-height:  56px;
+  --radius-sm:      6px;
+  --radius-md:      8px;
+  --radius-lg:      12px;
 }
 
 html, body, #root {
@@ -26,6 +29,81 @@ html, body, #root {
 }
 
 @keyframes blink {
-    0%, 100% { opacity: 1; }
+  0%, 100% { opacity: 1; }
-    50%       { opacity: 0; }
+  50%       { opacity: 0; }
 }
+
+/* ── Layout ─────────────────────────────────────────── */
+
+.flex        { display: flex; }
+.flex-col    { display: flex; flex-direction: column; }
+.flex-1      { flex: 1; }
+.flex-shrink { flex-shrink: 0; }
+.items-center   { align-items: center; }
+.justify-center { justify-content: center; }
+.justify-between { justify-content: space-between; }
+.overflow-hidden { overflow: hidden; }
+.scroll-y    { overflow-y: auto; overflow-x: hidden; }
+
+/* ── Panel header — shared by all three sidebars ────── */
+
+.panel-header {
+  height: var(--header-height);
+  display: flex;
+  align-items: center;
+  border-bottom: 1px solid var(--border);
+  flex-shrink: 0;
+  background: var(--bg-surface);
+}
+
+/* ── Button resets ──────────────────────────────────── */
+
+.btn-reset {
+  background: none;
+  border: none;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.btn-icon {
+  background: none;
+  border: none;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  padding: 6px;
+  border-radius: var(--radius-sm);
+  color: var(--text-muted);
+  font-size: 16px;
+  line-height: 1;
+}
+
+.btn-icon:hover { background: var(--bg-elevated); }
+
+.btn-primary {
+  background: var(--accent);
+  border: none;
+  border-radius: var(--radius-md);
+  color: white;
+  cursor: pointer;
+  font-size: 13px;
+  font-weight: 500;
+  transition: background 0.15s;
+}
+
+.btn-primary:hover    { background: var(--accent-hover); }
+.btn-primary:disabled { background: var(--bg-elevated); color: var(--text-muted); cursor: default; }
+
+/* ── Typography helpers ─────────────────────────────── */
+
+.text-xs      { font-size: 11px; }
+.text-sm      { font-size: 12px; }
+.text-base    { font-size: 13px; }
+.text-muted   { color: var(--text-muted); }
+.text-secondary { color: var(--text-secondary); }
+.text-accent  { color: var(--accent); }
+.label-upper  { font-size: 11px; font-weight: 500; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.08em; }
+.truncate     { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }

packages/inference-service/src/providers/llamacpp.js

@@ -1,80 +1,90 @@
-const { getEnv, LLAMACPP, INFERENCE_DEFAULTS } = require('@nexusai/shared');
+const { getEnv, LLAMACPP, INFERENCE_DEFAULTS } = require("@nexusai/shared");
 
-const BASE_URL      = getEnv('INFERENCE_URL', LLAMACPP.DEFAULT_URL);
+const BASE_URL = getEnv("INFERENCE_URL", LLAMACPP.DEFAULT_URL);
-const DEFAULT_MODEL = getEnv('DEFAULT_MODEL', LLAMACPP.DEFAULT_MODEL);
+const DEFAULT_MODEL = getEnv("DEFAULT_MODEL", LLAMACPP.DEFAULT_MODEL);
 
 function resolveOptions(options) {
-    return {
+  return {
-        temperature:    options.temperature    ?? INFERENCE_DEFAULTS.TEMPERATURE,
+    temperature: options.temperature ?? INFERENCE_DEFAULTS.TEMPERATURE,
-        maxTokens:      options.maxTokens      ?? INFERENCE_DEFAULTS.MAX_TOKENS,
+    maxTokens: options.maxTokens ?? INFERENCE_DEFAULTS.MAX_TOKENS,
-        topP:           options.topP           ?? INFERENCE_DEFAULTS.TOP_P,
+    topP: options.topP ?? INFERENCE_DEFAULTS.TOP_P,
-        topK:           options.topK           ?? INFERENCE_DEFAULTS.TOP_K,
+    topK: options.topK ?? INFERENCE_DEFAULTS.TOP_K,
-        repeatPenalty:  options.repeatPenalty  ?? INFERENCE_DEFAULTS.REPEAT_PENALTY,
+    repeatPenalty: options.repeatPenalty ?? INFERENCE_DEFAULTS.REPEAT_PENALTY,
-        seed:           options.seed           ?? INFERENCE_DEFAULTS.SEED,
+    seed: options.seed ?? INFERENCE_DEFAULTS.SEED,
-    };
+  };
 }
 
-function buildPayload(prompt, options, stream = false){
+function buildPayload(prompt, options, stream = false) {
-    const opts = resolveOptions(options);
+  const opts = resolveOptions(options);
 
-    return {
+  return {
-        model: options.model || DEFAULT_MODEL,
+    model: options.model || DEFAULT_MODEL,
-        messages: [{ role: 'user', content: prompt }],
+    messages: [{ role: "user", content: prompt }],
-        temperature:    opts.temperature,
+    temperature: opts.temperature,
-        max_tokens:     opts.maxTokens,
+    max_tokens: opts.maxTokens,
-        top_p:          opts.topP,
+    top_p: opts.topP,
-        top_k:          opts.topK,
+    top_k: opts.topK,
-        repeat_penalty: opts.repeatPenalty,
+    repeat_penalty: opts.repeatPenalty,
-        stream,
+    stream,
-        ...(opts.seed !== null && { seed: opts.seed }),
+    ...(opts.seed !== null && { seed: opts.seed }),
-    };
+  };
 }
 
-async function complete(prompt, options = {} ) {
+async function complete(prompt, options = {}) {
-    const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
+  const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
-        method: 'POST',
+    method: "POST",
-        headers: { 'Content-Type': 'application/json' },
+    headers: { "Content-Type": "application/json" },
-        body: JSON.stringify(buildPayload(prompt, options, false))
+    body: JSON.stringify(buildPayload(prompt, options, false)),
-    })
+  });
 
-    if (!res.ok) throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
+  if (!res.ok)
+    throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
 
-    const data = await res.json();
+  const data = await res.json();
-    const choice = data.choices[0];
+  const choice = data.choices[0];
 
-    return {
+  return {
-        text: choice.message.content,
+    text: choice.message.content,
-        model: data.model,
+    model: data.model,
-        done: choice.finish_reason === 'stop',
+    done: choice.finish_reason === "stop",
-        evalCount: data.usage?.completion_tokens,
+    evalCount: data.usage?.completion_tokens,
-        promptEvalCount: data.usage?.prompt_tokens,
+    promptEvalCount: data.usage?.prompt_tokens,
-    }
+  };
 }
 
 async function* completeStream(prompt, options = {}) {
-    const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
+  let finalModel = DEFAULT_MODEL;
-        method: 'POST',
+  let finalTokenCount = 0;
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify(buildPayload(prompt, options, true))
-    });
 
-    if (!res.ok) throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
+  const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(buildPayload(prompt, options, true)),
+  });
 
-    //OpenAI streaming sends newline-delimited JSON (NDJSON) with "data: " prefix for each chunk
+  if (!res.ok)
-    //Example chunk: data: {"choices":[{"delta":{"content":"Hello"},"finish_reason":null,"index":0}]}
+    throw new Error(`llama.cpp error: ${res.status} ${res.statusText}`);
-    //we parse each chunk as it arrives
-    for await (const chunk of res.body){
-        const lines = Buffer.from(chunk).toString('utf8')
-            .split('\n')
-            .filter(l => l.startsWith('data: ') && l !== 'data: [DONE]');
 
-        for (const line of lines) {
+  for await (const chunk of res.body) {
-            const json = JSON.parse(line.slice(6)); //remove 'data: ' prefix
+    const lines = Buffer.from(chunk)
-            const delta = json.choices?.[0]?.delta?.content;
+      .toString("utf8")
-            if (delta) yield {response: delta, done: false};
+      .split("\n")
-        }        
+      .filter((l) => l.startsWith("data: ") && l !== "data: [DONE]");
+
+    for (const line of lines) {
+      const json = JSON.parse(line.slice(6));
+      const delta = json.choices?.[0]?.delta?.content;
+
+      // Capture final metadata from the stop chunk
+      if (json.choices?.[0]?.finish_reason === "stop") {
+        finalModel = json.model ?? finalModel;
+        finalTokenCount = json.usage?.completion_tokens ?? finalTokenCount;
+      }
+
+      if (delta) yield { response: delta, done: false };
     }
-    yield { response: '', done: true}; //signal completion at the end of the stream
+  }
+  yield { response: '', done: true, model: finalModel, tokenCount: finalTokenCount };
 }
 
 module.exports = { complete, completeStream };

packages/inference-service/src/routes/inference.js

@@ -24,22 +24,34 @@ router.post('/complete', async (req, res) => {
 router.post('/complete/stream', async (req, res) => {
     const { prompt, model, temperature } = req.body;
 
-    if (!prompt) {
+    if (!prompt) return res.status(400).json({ error: 'prompt is required' });
-        return res.status(400).json({error: 'prompt is required'});
-    }
 
     res.setHeader('Content-Type', 'text/event-stream');
     res.setHeader('Cache-Control', 'no-cache');
     res.setHeader('Connection', 'keep-alive');
 
     try {
-        for await (const chunk of completeStream(prompt, {model, temperature})) {
+        let lastModel = model;
-            res.write(`data: ${JSON.stringify(chunk)}\n\n`);
+        let tokenCount = 0;
+
+        for await (const chunk of completeStream(prompt, { model, temperature })) {
+            if (chunk.response) {
+                res.write(`data: ${JSON.stringify({ response: chunk.response })}\n\n`);
+            }
+            if (chunk.done) {
+                // capture final metadata from the done signal
+                lastModel  = chunk.model      ?? lastModel;
+                tokenCount = chunk.tokenCount ?? tokenCount;
+            }
         }
+
+        // Send a single done event with metadata after stream closes
+        res.write(`data: ${JSON.stringify({ done: true, model: lastModel, tokenCount })}\n\n`);
         res.write('data: [DONE]\n\n');
-    } catch (error) {
+
-        console.error('[Inference] Streaming error:', error.message);
+    } catch (err) {
-        res.write(`data: ${JSON.stringify({ error: error.message })}\n\n`);
+        console.error('[Inference] Streaming error:', err.message);
+        res.write(`data: ${JSON.stringify({ error: err.message })}\n\n`);
     } finally {
         res.end();
     }

packages/orchestration-service/src/chat/index.js

@@ -109,31 +109,35 @@ async function chatStream(externalId, userMessage, onChunk, options = {} ) {
     let tokenCount = 0;
 
     // 5. Parse SSE chunks
-    for await (const chunk of res.body){
+    // Replace the current SSE parsing block in chatStream:
+    for await (const chunk of res.body) {
         const lines = chunk.toString().split('\n');
 
         for (const line of lines) {
             if (!line.startsWith('data: ')) continue;
             const raw = line.slice(6).trim();
-            if (raw === '[DONE]') continue //stream closed sentinel
+            if (raw === '[DONE]') continue;
 
             try {
                 const data = JSON.parse(raw);
-                if (data.model) model = data.model
 
+                // llama.cpp provider shape: { response, done }
                 if (data.response) {
                     fullText += data.response;
                     onChunk(data.response);
                 }
 
-                if (data.done && data.eval_count !== undefined) {
+                // model comes through on done chunk from inference route
-                    tokenCount = (data.eval_count || 0) + (data.prompt_eval_count || 0)
+                if (data.model) model = data.model;
-                }
-            } catch {
-                //partial chunk
-                //skip and wait for next
-            }
 
+                // token count — inference.js route sends this on the done chunk
+                if (data.done && data.tokenCount !== undefined) {
+                    tokenCount = data.tokenCount;
+                }
+
+            } catch {
+                // partial chunk — skip
+            }
         }
     }
 

packages/orchestration-service/src/index.js

@@ -3,6 +3,7 @@ const express = require('express');
 const {getEnv, PORTS, SERVICES, ORCHESTRATION} = require('@nexusai/shared');
 const chatRouter = require('./routes/chat');
 const sessionsRouter = require('./routes/sessions');
+const modelsRouter = require('./routes/models')
 const cors = require('cors');
 
 const app = express();
@@ -35,6 +36,7 @@ app.get('/health', (req, res) => {
 
 app.use('/chat', chatRouter);
 app.use('/sessions', sessionsRouter);
+app.use('/models', modelsRouter);
 
 /******* Start the server ************/
 app.listen(PORT, () => {

packages/orchestration-service/src/routes/chat.js

@@ -36,10 +36,14 @@ router.post('/stream', async (req, res) => {
     res.flushHeaders();
 
     try {
-        await chatStream(sessionId, message, (delta) => {
+        const { model, tokenCount } = await chatStream(
-            res.write(`data: ${JSON.stringify({ text: delta})}\n\n`)
+            sessionId, 
-        })
+            message, 
-        res.write(`data: ${JSON.stringify({done: true})}\n\n`);
+            (delta) => { res.write(`data: ${JSON.stringify({ text: delta })}\n\n`) },
+            { model: req.body.model, temperature: req.body.temperature }
+        );
+
+        res.write(`data: ${JSON.stringify({ done: true, model, tokenCount })}\n\n`);
     } catch (err) {
         res.write(`data: ${JSON.stringify({error: err.message})}\n\n`);
     } finally {

packages/orchestration-service/src/routes/models.js

@@ -0,0 +1,21 @@
+// routes/models.js
+const express = require('express');
+const router = express.Router();
+const fs = require('fs');
+const path = require('path');
+const {getEnv} = require('@nexusai/shared');
+
+const MODELS_PATH = getEnv('MODELS_MANIFEST_PATH', path.join(__dirname, '../models.json'));
+
+router.get('/', (req, res) => {
+  try {
+    const raw = fs.readFileSync(MODELS_PATH, 'utf8');
+    const models = JSON.parse(raw);
+    res.json(models);
+  } catch (err) {
+    console.error('[models] Failed to read manifest:', err.message);
+    res.status(500).json({ error: 'Could not load models manifest' });
+  }
+});
+
+module.exports = router;