diff --git a/app/frontend/src/App.css b/app/frontend/src/App.css
index 6912785..55df099 100644
--- a/app/frontend/src/App.css
+++ b/app/frontend/src/App.css
@@ -330,6 +330,23 @@ body {
font-size: 0.78rem;
}
+.chat-generation-stats {
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ margin-top: 9px;
+ padding-top: 7px;
+ border-top: 1px solid color-mix(in srgb, var(--md-sys-color-outline) 22%, transparent);
+ color: var(--md-sys-color-on-surface-variant);
+ font-size: 0.72rem;
+ font-weight: 600;
+ line-height: 1.2;
+}
+
+.chat-generation-stats.complete {
+ color: var(--md-sys-color-primary);
+}
+
.chat-thinking,
.text-progress {
display: flex;
diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx
index fa66173..2627105 100644
--- a/app/frontend/src/components/TextChat.jsx
+++ b/app/frontend/src/components/TextChat.jsx
@@ -25,6 +25,12 @@ function TextChat({
setShowHistory,
saveConversationState
}) {
+ const formatGenerationTime = (seconds) => {
+ const value = Number(seconds) || 0;
+ if (value < 1) return `${Math.round(value * 1000)} ms`;
+ return `${value.toFixed(value < 10 ? 2 : 1)} s`;
+ };
+
const [models, setModels] = useState([]);
const [status, setStatus] = useState({ ready: false, running: false, settings: {} });
const [selectedModel, setSelectedModel] = useState("");
@@ -44,6 +50,7 @@ function TextChat({
const [attachments, setAttachments] = useState([]);
const fileInputRef = useRef(null);
+ const supportsVision = Boolean(status.ready && status.settings?.supportsVision);
const isImage = (file) => {
return /\.(jpe?g|png|webp)$/i.test(file.name) || file.type.startsWith("image/");
@@ -104,8 +111,15 @@ function TextChat({
loadingModelRef.current = loadingModel;
}, [loadingModel]);
+ useEffect(() => {
+ if (!supportsVision) {
+ setAttachments((current) => current.filter((attachment) => attachment.type !== "image"));
+ }
+ }, [supportsVision]);
+
// Load conversation messages when activeConversationId changes
useEffect(() => {
+ if (isBusy) return;
if (activeConversationId) {
const conv = conversations.find(c => c.id === activeConversationId);
if (conv) {
@@ -129,7 +143,7 @@ function TextChat({
setMessages([]);
setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 });
}
- }, [activeConversationId, conversations, models]);
+ }, [activeConversationId, conversations, models, isBusy]);
const refresh = useCallback(async () => {
const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]);
@@ -279,7 +293,12 @@ function TextChat({
const firstTitle = isNew ? (displayTitleText.slice(0, 26) + (displayTitleText.length > 26 ? "..." : "")) : null;
saveConversationState(convId, nextMessages, selectedModel, firstTitle);
- setMessages([...nextMessages, { role: "assistant", content: "" }]);
+ const requestStartedAt = performance.now();
+ setMessages([...nextMessages, {
+ role: "assistant",
+ content: "",
+ generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 },
+ }]);
try {
const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant.";
@@ -289,11 +308,15 @@ function TextChat({
];
let assistantText = "";
+ let streamedTokens = 0;
const response = await streamChatWithLlm(requestMessages, {
temperature: textSettings?.temperature || 0.7,
maxTokens: 768,
}, (_token, fullText) => {
+ const now = performance.now();
+ streamedTokens += 1;
+ const generationSeconds = Math.max(0.05, (now - requestStartedAt) / 1000);
assistantText = fullText;
setMessages((prev) => {
const updated = [...prev];
@@ -301,13 +324,37 @@ function TextChat({
updated[updated.length - 1] = {
...updated[updated.length - 1],
content: fullText,
+ generationStats: {
+ status: "streaming",
+ tokens: streamedTokens,
+ tokensPerSecond: streamedTokens / generationSeconds,
+ seconds: (now - requestStartedAt) / 1000,
+ },
};
}
return updated;
});
});
- const finalMessages = [...nextMessages, { role: "assistant", content: assistantText }];
+ const completedAt = performance.now();
+ const exactTokens = Number(response.timings?.predicted_n) || streamedTokens;
+ const backendTotalMs = Number(response.timings?.prompt_ms || 0) + Number(response.timings?.predicted_ms || 0);
+ const exactSeconds = backendTotalMs > 0
+ ? backendTotalMs / 1000
+ : (completedAt - requestStartedAt) / 1000;
+ const exactTokensPerSecond = Number(response.timings?.predicted_per_second)
+ || (exactTokens / Math.max(0.001, exactSeconds));
+ const generationStats = {
+ status: "complete",
+ tokens: exactTokens,
+ tokensPerSecond: exactTokensPerSecond,
+ seconds: exactSeconds,
+ };
+ const finalMessages = [...nextMessages, {
+ role: "assistant",
+ content: assistantText,
+ generationStats,
+ }];
setMessages(finalMessages);
saveConversationState(convId, finalMessages, selectedModel);
if (response.usage) setTokenUsage(response.usage);
@@ -547,9 +594,23 @@ function TextChat({
{message.content}
)}
+ {message.role === "assistant" && message.generationStats && !message.error && (
+
+ {message.generationStats.status === "starting" ? (
+ <>Waiting for first token...
+ ) : message.generationStats.status === "streaming" ? (
+ <>{message.generationStats.tokensPerSecond.toFixed(1)} tokens/sec
+ ) : (
+ <>
+ {message.generationStats.tokens} tokens
+ •
+ {formatGenerationTime(message.generationStats.seconds)}
+
+ )}
+
+ )}
))}
- {isBusy && status.ready && Generating...
}
)}
@@ -620,8 +681,8 @@ function TextChat({
className="m3-btn m3-btn-tonal"
onClick={() => fileInputRef.current?.click()}
style={{ padding: "0 12px", height: "48px", display: "flex", alignItems: "center", justifyContent: "center" }}
- disabled={!status.ready || isBusy}
- title="Attach files or images"
+ disabled={!supportsVision || isBusy}
+ title={supportsVision ? "Attach files or images" : "Image attachment requires a vision model with an mmproj file"}
>
diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js
index f96a918..9b7d463 100644
--- a/app/frontend/src/services/api.js
+++ b/app/frontend/src/services/api.js
@@ -398,6 +398,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
let buffer = "";
let content = "";
let usage = null;
+ let timings = null;
const consumeEvent = (eventText) => {
const data = eventText
@@ -415,6 +416,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
onToken(token, content);
}
if (parsed.usage) usage = parsed.usage;
+ if (parsed.timings) timings = parsed.timings;
return false;
};
@@ -436,7 +438,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
if (!finished && buffer.trim()) consumeEvent(buffer);
if (!content) throw new Error("The text model returned an empty streamed response.");
- return { content, usage };
+ return { content, usage, timings };
}
export async function downloadLlmModel(url) {
diff --git a/scripts/serve.cjs b/scripts/serve.cjs
index ea2aeca..595566d 100644
--- a/scripts/serve.cjs
+++ b/scripts/serve.cjs
@@ -127,6 +127,7 @@ let llmSettings = {
gpuLayers: -1,
backendMode: "",
backendBinary: "",
+ supportsVision: false,
};
let backendLoadState = {
active: false,
@@ -219,10 +220,48 @@ function detectLinuxGpuFromSysfs() {
return null;
}
+function detectLlamaVulkanGpu() {
+ const backendPath = osPlatform === "win32"
+ ? LLM_BACKEND_PATHS.winVulkan
+ : osPlatform === "linux"
+ ? LLM_BACKEND_PATHS.linuxVulkan
+ : "";
+ if (!backendPath || !fs.existsSync(backendPath)) return null;
+
+ try {
+ const result = spawnSync(backendPath, ["--list-devices"], {
+ encoding: "utf8",
+ windowsHide: true,
+ timeout: 10000,
+ maxBuffer: 1024 * 1024,
+ });
+ const output = `${result.stdout || ""}\n${result.stderr || ""}`;
+ const devices = [];
+ const devicePattern = /Vulkan\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi;
+ let match;
+ while ((match = devicePattern.exec(output)) !== null) {
+ devices.push({
+ name: match[1].trim(),
+ vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100,
+ free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null,
+ });
+ }
+ return devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
+ } catch (_) {
+ return null;
+ }
+}
+
function getGpuInfo() {
if (cachedGpuInfo) return cachedGpuInfo;
if (osPlatform === "win32") {
+ const llamaVulkanGpu = detectLlamaVulkanGpu();
+ if (llamaVulkanGpu) {
+ cachedGpuInfo = llamaVulkanGpu;
+ return cachedGpuInfo;
+ }
+
try {
const stdout = execSync(
'powershell -NoProfile -Command "Get-CimInstance Win32_VideoController | ForEach-Object { [PSCustomObject]@{ Name = $_.Name; VRAM = [math]::Round($_.AdapterRAM / 1GB, 2) } } | ConvertTo-Json"',
@@ -939,6 +978,7 @@ async function waitForLlmReady(maxAttempts = 240) {
function killLlm() {
return new Promise((resolve) => {
llmReady = false;
+ llmSettings.supportsVision = false;
if (!llmProc) {
resolve();
return;
@@ -1631,14 +1671,21 @@ function getTextModelFit(sizeBytes, specs = getHardwareSpecs()) {
const fastGpuFit = vramBytes >= runtimeNeed;
const cpuOrOffloadFit = usableRam >= runtimeNeed;
const unifiedMemoryFit = isAppleSilicon && usableRam >= runtimeNeed;
+ const hybridFit = !fastGpuFit && vramBytes > 0 && cpuOrOffloadFit;
const recommended = fastGpuFit || cpuOrOffloadFit || unifiedMemoryFit;
if (recommended) {
- const mode = fastGpuFit ? "GPU memory" : isAppleSilicon ? "unified memory" : "system RAM with CPU/GPU offload";
+ const mode = fastGpuFit
+ ? "GPU memory"
+ : isAppleSilicon
+ ? "unified memory"
+ : hybridFit
+ ? "combined GPU and system memory"
+ : "system RAM";
return {
recommended: true,
- mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : "ram",
- label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : "Fits in RAM",
+ mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : hybridFit ? "hybrid" : "ram",
+ label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : hybridFit ? "GPU + RAM Fit" : "Fits in RAM",
reason: `${formatBytes(sizeBytes)} weights fit the estimated ${mode} budget with runtime headroom.`,
};
}
@@ -2029,6 +2076,8 @@ async function startLlm(settings = {}) {
if (mmprojPath) {
args.push("--mmproj", mmprojPath);
}
+ llmSettings.supportsVision = Boolean(mmprojPath);
+ llmSettings.mmproj = mmprojPath ? path.basename(mmprojPath) : null;
const spawnEnv = { ...process.env };
const backendDir = path.dirname(backend.path);
if (osPlatform === "linux") {