From 647341b11fb9095b24ad01dae52bf0d7387675c8 Mon Sep 17 00:00:00 2001 From: techjarves Date: Mon, 15 Jun 2026 07:51:54 +0530 Subject: [PATCH] feat: implement frontend text chat interface with core styles, messaging component, and API service integration. --- app/frontend/src/App.css | 17 ++++++ app/frontend/src/components/TextChat.jsx | 73 ++++++++++++++++++++++-- app/frontend/src/services/api.js | 4 +- scripts/serve.cjs | 55 +++++++++++++++++- 4 files changed, 139 insertions(+), 10 deletions(-) diff --git a/app/frontend/src/App.css b/app/frontend/src/App.css index 6912785..55df099 100644 --- a/app/frontend/src/App.css +++ b/app/frontend/src/App.css @@ -330,6 +330,23 @@ body { font-size: 0.78rem; } +.chat-generation-stats { + display: flex; + align-items: center; + gap: 6px; + margin-top: 9px; + padding-top: 7px; + border-top: 1px solid color-mix(in srgb, var(--md-sys-color-outline) 22%, transparent); + color: var(--md-sys-color-on-surface-variant); + font-size: 0.72rem; + font-weight: 600; + line-height: 1.2; +} + +.chat-generation-stats.complete { + color: var(--md-sys-color-primary); +} + .chat-thinking, .text-progress { display: flex; diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx index fa66173..2627105 100644 --- a/app/frontend/src/components/TextChat.jsx +++ b/app/frontend/src/components/TextChat.jsx @@ -25,6 +25,12 @@ function TextChat({ setShowHistory, saveConversationState }) { + const formatGenerationTime = (seconds) => { + const value = Number(seconds) || 0; + if (value < 1) return `${Math.round(value * 1000)} ms`; + return `${value.toFixed(value < 10 ? 2 : 1)} s`; + }; + const [models, setModels] = useState([]); const [status, setStatus] = useState({ ready: false, running: false, settings: {} }); const [selectedModel, setSelectedModel] = useState(""); @@ -44,6 +50,7 @@ function TextChat({ const [attachments, setAttachments] = useState([]); const fileInputRef = useRef(null); + const supportsVision = Boolean(status.ready && status.settings?.supportsVision); const isImage = (file) => { return /\.(jpe?g|png|webp)$/i.test(file.name) || file.type.startsWith("image/"); @@ -104,8 +111,15 @@ function TextChat({ loadingModelRef.current = loadingModel; }, [loadingModel]); + useEffect(() => { + if (!supportsVision) { + setAttachments((current) => current.filter((attachment) => attachment.type !== "image")); + } + }, [supportsVision]); + // Load conversation messages when activeConversationId changes useEffect(() => { + if (isBusy) return; if (activeConversationId) { const conv = conversations.find(c => c.id === activeConversationId); if (conv) { @@ -129,7 +143,7 @@ function TextChat({ setMessages([]); setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }); } - }, [activeConversationId, conversations, models]); + }, [activeConversationId, conversations, models, isBusy]); const refresh = useCallback(async () => { const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]); @@ -279,7 +293,12 @@ function TextChat({ const firstTitle = isNew ? (displayTitleText.slice(0, 26) + (displayTitleText.length > 26 ? "..." : "")) : null; saveConversationState(convId, nextMessages, selectedModel, firstTitle); - setMessages([...nextMessages, { role: "assistant", content: "" }]); + const requestStartedAt = performance.now(); + setMessages([...nextMessages, { + role: "assistant", + content: "", + generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 }, + }]); try { const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant."; @@ -289,11 +308,15 @@ function TextChat({ ]; let assistantText = ""; + let streamedTokens = 0; const response = await streamChatWithLlm(requestMessages, { temperature: textSettings?.temperature || 0.7, maxTokens: 768, }, (_token, fullText) => { + const now = performance.now(); + streamedTokens += 1; + const generationSeconds = Math.max(0.05, (now - requestStartedAt) / 1000); assistantText = fullText; setMessages((prev) => { const updated = [...prev]; @@ -301,13 +324,37 @@ function TextChat({ updated[updated.length - 1] = { ...updated[updated.length - 1], content: fullText, + generationStats: { + status: "streaming", + tokens: streamedTokens, + tokensPerSecond: streamedTokens / generationSeconds, + seconds: (now - requestStartedAt) / 1000, + }, }; } return updated; }); }); - const finalMessages = [...nextMessages, { role: "assistant", content: assistantText }]; + const completedAt = performance.now(); + const exactTokens = Number(response.timings?.predicted_n) || streamedTokens; + const backendTotalMs = Number(response.timings?.prompt_ms || 0) + Number(response.timings?.predicted_ms || 0); + const exactSeconds = backendTotalMs > 0 + ? backendTotalMs / 1000 + : (completedAt - requestStartedAt) / 1000; + const exactTokensPerSecond = Number(response.timings?.predicted_per_second) + || (exactTokens / Math.max(0.001, exactSeconds)); + const generationStats = { + status: "complete", + tokens: exactTokens, + tokensPerSecond: exactTokensPerSecond, + seconds: exactSeconds, + }; + const finalMessages = [...nextMessages, { + role: "assistant", + content: assistantText, + generationStats, + }]; setMessages(finalMessages); saveConversationState(convId, finalMessages, selectedModel); if (response.usage) setTokenUsage(response.usage); @@ -547,9 +594,23 @@ function TextChat({ {message.content} )} + {message.role === "assistant" && message.generationStats && !message.error && ( +
+ {message.generationStats.status === "starting" ? ( + <>Waiting for first token... + ) : message.generationStats.status === "streaming" ? ( + <>{message.generationStats.tokensPerSecond.toFixed(1)} tokens/sec + ) : ( + <> + {message.generationStats.tokens} tokens + • + {formatGenerationTime(message.generationStats.seconds)} + + )} +
+ )} ))} - {isBusy && status.ready &&
Generating...
} )}
@@ -620,8 +681,8 @@ function TextChat({ className="m3-btn m3-btn-tonal" onClick={() => fileInputRef.current?.click()} style={{ padding: "0 12px", height: "48px", display: "flex", alignItems: "center", justifyContent: "center" }} - disabled={!status.ready || isBusy} - title="Attach files or images" + disabled={!supportsVision || isBusy} + title={supportsVision ? "Attach files or images" : "Image attachment requires a vision model with an mmproj file"} > diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js index f96a918..9b7d463 100644 --- a/app/frontend/src/services/api.js +++ b/app/frontend/src/services/api.js @@ -398,6 +398,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => let buffer = ""; let content = ""; let usage = null; + let timings = null; const consumeEvent = (eventText) => { const data = eventText @@ -415,6 +416,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => onToken(token, content); } if (parsed.usage) usage = parsed.usage; + if (parsed.timings) timings = parsed.timings; return false; }; @@ -436,7 +438,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => if (!finished && buffer.trim()) consumeEvent(buffer); if (!content) throw new Error("The text model returned an empty streamed response."); - return { content, usage }; + return { content, usage, timings }; } export async function downloadLlmModel(url) { diff --git a/scripts/serve.cjs b/scripts/serve.cjs index ea2aeca..595566d 100644 --- a/scripts/serve.cjs +++ b/scripts/serve.cjs @@ -127,6 +127,7 @@ let llmSettings = { gpuLayers: -1, backendMode: "", backendBinary: "", + supportsVision: false, }; let backendLoadState = { active: false, @@ -219,10 +220,48 @@ function detectLinuxGpuFromSysfs() { return null; } +function detectLlamaVulkanGpu() { + const backendPath = osPlatform === "win32" + ? LLM_BACKEND_PATHS.winVulkan + : osPlatform === "linux" + ? LLM_BACKEND_PATHS.linuxVulkan + : ""; + if (!backendPath || !fs.existsSync(backendPath)) return null; + + try { + const result = spawnSync(backendPath, ["--list-devices"], { + encoding: "utf8", + windowsHide: true, + timeout: 10000, + maxBuffer: 1024 * 1024, + }); + const output = `${result.stdout || ""}\n${result.stderr || ""}`; + const devices = []; + const devicePattern = /Vulkan\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; + let match; + while ((match = devicePattern.exec(output)) !== null) { + devices.push({ + name: match[1].trim(), + vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100, + free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null, + }); + } + return devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null; + } catch (_) { + return null; + } +} + function getGpuInfo() { if (cachedGpuInfo) return cachedGpuInfo; if (osPlatform === "win32") { + const llamaVulkanGpu = detectLlamaVulkanGpu(); + if (llamaVulkanGpu) { + cachedGpuInfo = llamaVulkanGpu; + return cachedGpuInfo; + } + try { const stdout = execSync( 'powershell -NoProfile -Command "Get-CimInstance Win32_VideoController | ForEach-Object { [PSCustomObject]@{ Name = $_.Name; VRAM = [math]::Round($_.AdapterRAM / 1GB, 2) } } | ConvertTo-Json"', @@ -939,6 +978,7 @@ async function waitForLlmReady(maxAttempts = 240) { function killLlm() { return new Promise((resolve) => { llmReady = false; + llmSettings.supportsVision = false; if (!llmProc) { resolve(); return; @@ -1631,14 +1671,21 @@ function getTextModelFit(sizeBytes, specs = getHardwareSpecs()) { const fastGpuFit = vramBytes >= runtimeNeed; const cpuOrOffloadFit = usableRam >= runtimeNeed; const unifiedMemoryFit = isAppleSilicon && usableRam >= runtimeNeed; + const hybridFit = !fastGpuFit && vramBytes > 0 && cpuOrOffloadFit; const recommended = fastGpuFit || cpuOrOffloadFit || unifiedMemoryFit; if (recommended) { - const mode = fastGpuFit ? "GPU memory" : isAppleSilicon ? "unified memory" : "system RAM with CPU/GPU offload"; + const mode = fastGpuFit + ? "GPU memory" + : isAppleSilicon + ? "unified memory" + : hybridFit + ? "combined GPU and system memory" + : "system RAM"; return { recommended: true, - mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : "ram", - label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : "Fits in RAM", + mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : hybridFit ? "hybrid" : "ram", + label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : hybridFit ? "GPU + RAM Fit" : "Fits in RAM", reason: `${formatBytes(sizeBytes)} weights fit the estimated ${mode} budget with runtime headroom.`, }; } @@ -2029,6 +2076,8 @@ async function startLlm(settings = {}) { if (mmprojPath) { args.push("--mmproj", mmprojPath); } + llmSettings.supportsVision = Boolean(mmprojPath); + llmSettings.mmproj = mmprojPath ? path.basename(mmprojPath) : null; const spawnEnv = { ...process.env }; const backendDir = path.dirname(backend.path); if (osPlatform === "linux") {