storme/nexusAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed token count reading

Browse Source
This commit is contained in:
Storme-bit
2026-04-19 07:50:10 -07:00
parent 225728e531
commit 0619c4c7f3
1 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/inference-service/src/providers/llamacpp.js
View File

@@ -76,10 +76,13 @@ async function* completeStream(prompt, options = {}) {
const json = JSON.parse(line.slice(6)); const json = JSON.parse(line.slice(6));
const delta = json.choices?.[0]?.delta?.content; const delta = json.choices?.[0]?.delta?.content;
// Capture final metadata from the stop chunk if (json.choices?.[0]?.finish_reason === 'stop') {
if (json.choices?.[0]?.finish_reason === "stop") {
finalModel = json.model ?? finalModel; finalModel = json.model ?? finalModel;
finalTokenCount = json.usage?.completion_tokens ?? finalTokenCount; }
// usage arrives in a separate final chunk with empty choices array
if (json.usage) {
finalTokenCount = (json.usage.completion_tokens ?? 0) + (json.usage.prompt_tokens ?? 0);
} }
if (delta) yield { response: delta, done: false }; if (delta) yield { response: delta, done: false };