diff --git a/app/frontend/src/App.jsx b/app/frontend/src/App.jsx index ad5a78e..d52277f 100644 --- a/app/frontend/src/App.jsx +++ b/app/frontend/src/App.jsx @@ -129,7 +129,7 @@ function App() { contextSize: 0, temperature: 0.7, systemPrompt: "You are a helpful local AI assistant.", - threads: 4, + threads: Math.max(4, Math.min(16, (navigator.hardwareConcurrency || 4) - 2)), enableThinking: true }; }); diff --git a/app/frontend/src/components/Settings.jsx b/app/frontend/src/components/Settings.jsx index 090ca83..3b3ca4a 100644 --- a/app/frontend/src/components/Settings.jsx +++ b/app/frontend/src/components/Settings.jsx @@ -1,5 +1,5 @@ import React, { memo, useEffect } from "react"; -import { Crop, Sliders, Cpu, Info, MessageSquare, SlidersHorizontal } from "lucide-react"; +import { Crop, Sliders, Cpu, Info, MessageSquare, SlidersHorizontal, Zap } from "lucide-react"; import { stopServer, formatBytes } from "../services/api"; const ASPECT_RATIOS = [ @@ -567,25 +567,97 @@ function Settings({ - {/* CPU Threads slider */} -
-
- CPU Threads - {textSettings.threads} threads + {/* CPU/GPU Mode Toggle */} + {(specs?.gpu_name && !specs.gpu_name.includes("Loading") && specs.gpu_name !== "Unavailable") && ( +
+
+ Text Generation Backend + + {(textSettings.gpuLayers === 0 || textSettings.gpuLayers === undefined) ? "CPU" : "GPU"} + +
+
+
updateTextSetting("gpuLayers", 0)} + title="Run model entirely on CPU. Best for Intel Arc GPUs which have poor Vulkan performance." + style={{ cursor: "pointer" }} + > + + CPU +
+
updateTextSetting("gpuLayers", -1)} + title="Offload model layers to GPU for faster generation." + style={{ cursor: "pointer" }} + > + + GPU +
+
+ + {(textSettings.gpuLayers === 0 || textSettings.gpuLayers === undefined) ? ( + + CPU mode: Uses processor only. Recommended for Intel Arc GPUs which currently perform better on CPU with llama.cpp. + + ) : ( + + GPU mode: Offloads model layers to graphics card. Best for NVIDIA/AMD GPUs. Intel Arc may be slower due to Vulkan driver limitations. + + )} +
- updateTextSetting("threads", parseInt(e.target.value))} - min="1" - max={Math.max(16, specs?.cpu_cores_logical || 16)} - step="1" - /> - - Number of physical CPU threads allocated for computation. Typically should match or be slightly lower than your physical cores. - -
+ )} + + {/* GPU Layers slider - only show when GPU mode is selected */} + {(specs?.gpu_name && !specs.gpu_name.includes("Loading") && specs.gpu_name !== "Unavailable") && (textSettings.gpuLayers !== 0 && textSettings.gpuLayers !== undefined) && ( +
+
+ GPU Layer Offload + + {textSettings.gpuLayers === -1 ? "All layers" : `${textSettings.gpuLayers} layers`} + +
+ { + const val = parseInt(e.target.value); + updateTextSetting("gpuLayers", val === 999 ? -1 : val); + }} + min="0" + max="999" + step="1" + /> +
+ 0 (CPU only) + All (-1) +
+ + More layers = faster but uses more VRAM. Start with "All" and reduce if you get out-of-memory errors. + {textSettings.gpuLayers === -1 && " Currently all layers will be loaded into GPU memory."} + {textSettings.gpuLayers !== -1 && textSettings.gpuLayers !== 0 && ` Currently ${textSettings.gpuLayers} layers on GPU, rest on CPU.`} + +
+ )} + + {/* CPU-only fallback message when no GPU detected */} + {(!specs?.gpu_name || specs.gpu_name.includes("Loading") || specs.gpu_name === "Unavailable") && ( +
+ No GPU detected. Text generation will use CPU only. Install GPU drivers or restart if you have a dedicated graphics card. +
+ )}
diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx index 5ec8349..5c2c7d9 100644 --- a/app/frontend/src/components/TextChat.jsx +++ b/app/frontend/src/components/TextChat.jsx @@ -1,1074 +1,1108 @@ -import React, { useCallback, useEffect, useRef, useState } from "react"; -import { Bot, LoaderCircle, Send, Trash2, Square, History, Paperclip, X, ChevronDown } from "lucide-react"; -import { - getDownloadProgress, - getLlmStatus, - listLlmModels, - streamChatWithLlm, - startLlm, - stopLlm, -} from "../services/api"; - -const processMessageContent = (rawText, apiReasoning = "") => { - let content = rawText; - let reasoning = apiReasoning || ""; - - if (typeof rawText !== "string") { - return { content, reasoning }; - } - - const startTags = ["<|channel|>thought", "<|think|>", "<|thought|>", "", ""]; - const endTags = ["<|channel|>model", "<|turn>model", "<|im_start|>model", "", "", "", ""]; - - let startIdx = -1; - let matchedStartTag = ""; - - for (const tag of startTags) { - const idx = rawText.indexOf(tag); - if (idx !== -1 && (startIdx === -1 || idx < startIdx)) { - startIdx = idx; - matchedStartTag = tag; - } - } - - if (startIdx !== -1) { - let endIdx = -1; - let matchedEndTag = ""; - const searchArea = rawText.substring(startIdx + matchedStartTag.length); - - for (const tag of endTags) { - const idx = searchArea.indexOf(tag); - if (idx !== -1 && (endIdx === -1 || idx < endIdx)) { - endIdx = idx; - matchedEndTag = tag; - } - } - - if (endIdx !== -1) { - const actualEndIdxInRaw = startIdx + matchedStartTag.length + endIdx; - const extractedReasoning = rawText.substring(startIdx + matchedStartTag.length, actualEndIdxInRaw).trim(); - reasoning = (reasoning + "\n" + extractedReasoning).trim(); - - const afterEndText = rawText.substring(actualEndIdxInRaw + matchedEndTag.length); - content = rawText.substring(0, startIdx) + afterEndText; - - return processMessageContent(content, reasoning); - } else { - const extractedReasoning = rawText.substring(startIdx + matchedStartTag.length).trim(); - reasoning = (reasoning + "\n" + extractedReasoning).trim(); - content = rawText.substring(0, startIdx); - } - } - - return { content, reasoning }; -}; - -function ThinkingBlock({ reasoning, isComplete, thinkingDuration }) { - const [isExpanded, setIsExpanded] = useState(!isComplete); - - useEffect(() => { - if (isComplete) { - setIsExpanded(false); - } - }, [isComplete]); - - if (!reasoning) return null; - - const formattedTime = thinkingDuration > 0 ? ` (${thinkingDuration.toFixed(1)}s)` : ""; - - return ( -
- - {isExpanded && ( -
- {reasoning} -
- )} -
- ); -} - -function TextChat({ - specs, - showAlert, - showConfirm, - textSettings, - setTextSettings, - setActiveModel, - setServerRunning, - conversations, - setConversations, - activeConversationId, - setActiveConversationId, - showHistory, - setShowHistory, - saveConversationState -}) { - const formatGenerationTime = (seconds) => { - const value = Number(seconds) || 0; - if (value < 1) return `${Math.round(value * 1000)} ms`; - return `${value.toFixed(value < 10 ? 2 : 1)} s`; - }; - - const [models, setModels] = useState([]); - const [status, setStatus] = useState({ ready: false, running: false, settings: {} }); - const [selectedModel, setSelectedModel] = useState(""); - const [messages, setMessages] = useState([]); - const [input, setInput] = useState(""); - const [isBusy, setIsBusy] = useState(false); - const [loadingModel, setLoadingModel] = useState(null); - const [tokenUsage, setTokenUsage] = useState({ - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0 - }); - - const bottomRef = useRef(null); - const completedDownloadRef = useRef(""); - const loadingModelRef = useRef(null); - const chatMessagesRef = useRef(null); - const prevMessagesLengthRef = useRef(0); - const abortControllerRef = useRef(null); - - const [attachments, setAttachments] = useState([]); - const fileInputRef = useRef(null); - const supportsVision = Boolean(status.ready && status.settings?.supportsVision); - - const isImage = (file) => { - return /\.(jpe?g|png|webp)$/i.test(file.name) || file.type.startsWith("image/"); - }; - - const isTextFile = (file) => { - return /\.(txt|md|csv|js|jsx|ts|tsx|py|json|css|html|java|cpp|c|h|rs|go|sh|bat|xml|yaml|yml)$/i.test(file.name) || file.type.startsWith("text/"); - }; - - const handleFileChange = (e) => { - const files = Array.from(e.target.files); - if (!files || files.length === 0) return; - - files.forEach((file) => { - if (isImage(file)) { - const reader = new FileReader(); - reader.onload = (event) => { - setAttachments((prev) => [ - ...prev, - { - id: Math.random().toString(36).substring(7), - file, - type: "image", - name: file.name, - dataUrl: event.target.result, - }, - ]); - }; - reader.readAsDataURL(file); - } else if (isTextFile(file)) { - const reader = new FileReader(); - reader.onload = (event) => { - setAttachments((prev) => [ - ...prev, - { - id: Math.random().toString(36).substring(7), - file, - type: "document", - name: file.name, - content: event.target.result, - }, - ]); - }; - reader.readAsText(file); - } else { - showAlert({ - title: "Unsupported File", - message: `File "${file.name}" is not supported. Please select an image (JPEG, PNG, WEBP) or a text/code file.`, - danger: true, - }); - } - }); - - e.target.value = ""; - }; - - useEffect(() => { - loadingModelRef.current = loadingModel; - }, [loadingModel]); - - useEffect(() => { - if (!supportsVision) { - setAttachments((current) => current.filter((attachment) => attachment.type !== "image")); - } - }, [supportsVision]); - - // Load conversation messages when activeConversationId changes - useEffect(() => { - if (isBusy) return; - if (activeConversationId) { - const conv = conversations.find(c => c.id === activeConversationId); - if (conv) { - setMessages(conv.messages); - if (conv.model && models.some(m => m.filename === conv.model)) { - setSelectedModel(conv.model); - } - const total = conv.messages.reduce((sum, m) => { - const text = Array.isArray(m.content) - ? m.content.map(c => c.text || "").join(" ") - : (m.content || ""); - return sum + text.split(/\s+/).length; - }, 0); - setTokenUsage({ - prompt_tokens: Math.round(total * 0.7), - completion_tokens: Math.round(total * 0.3), - total_tokens: total - }); - } - } else { - setMessages([]); - setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }); - } - }, [activeConversationId, conversations, models, isBusy]); - - const refresh = useCallback(async () => { - const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]); - setModels(nextModels); - setStatus(nextStatus); - const active = nextStatus.settings?.model; - setSelectedModel((current) => { - const saved = localStorage.getItem("selectedLlmModel"); - const savedExists = nextModels.some((m) => m.filename === saved); - return active || current || (savedExists ? saved : "") || nextModels[0]?.filename || ""; - }); - }, []); - - useEffect(() => { - refresh().catch(() => {}); - const timer = setInterval(() => { - getLlmStatus().then((nextStatus) => { - setStatus(nextStatus); - // If it suddenly loaded or became ready externally, update selection and reset loading states - if (nextStatus.ready && nextStatus.settings?.model) { - setSelectedModel(nextStatus.settings.model); - setLoadingModel(null); - } - }).catch(() => {}); - getDownloadProgress().then((state) => { - if (state.kind === "text" && (state.active || state.error || state.progress === 100)) { - const completionKey = `${state.filename || ""}:${state.downloadedBytes || 0}`; - if (!state.active && !state.error && completedDownloadRef.current !== completionKey) { - completedDownloadRef.current = completionKey; - refresh().catch(() => {}); - } - } - }).catch(() => {}); - }, 1500); - return () => clearInterval(timer); - }, [refresh]); - - const handleStopGeneration = () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - }; - - useEffect(() => { - const container = chatMessagesRef.current; - if (!container) return; - - const len = messages.length; - const prevLen = prevMessagesLengthRef.current; - prevMessagesLengthRef.current = len; - - const lastMessage = messages[len - 1]; - const isNewUserMessage = len > prevLen && lastMessage?.role === "user"; - - if (isNewUserMessage) { - bottomRef.current?.scrollIntoView({ behavior: "smooth" }); - } else { - const threshold = 150; - const isNearBottom = container.scrollHeight - container.scrollTop - container.clientHeight <= threshold; - if (isNearBottom) { - bottomRef.current?.scrollIntoView({ behavior: "smooth" }); - } - } - }, [messages, isBusy, loadingModel]); - - const handleModelChange = async (filename) => { - if (!filename) { - if (status.ready) { - setIsBusy(true); - try { - await stopLlm(); - setStatus((prev) => ({ ...prev, ready: false, running: false })); - setSelectedModel(""); - setMessages([]); - setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }); - } catch (err) { - showAlert({ title: "Unload Failed", message: err.message || String(err), danger: true }); - } finally { - setIsBusy(false); - } - } - return; - } - - setSelectedModel(filename); - localStorage.setItem("selectedLlmModel", filename); - setIsBusy(true); - setLoadingModel(filename); - try { - // Unload active image engine if running - if (setActiveModel) setActiveModel(null); - if (setServerRunning) setServerRunning(false); - - const result = await startLlm(filename, { - threads: textSettings?.threads || specs?.cpu_cores_physical || 4, - contextSize: textSettings?.contextSize || 4096, - gpuLayers: -1, - enableThinking: textSettings?.enableThinking !== false, - }); - setStatus({ ...status, ...result, ready: true, running: true, settings: result.settings }); - setMessages([]); - setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }); - } catch (err) { - if (loadingModelRef.current === filename) { - showAlert({ title: "Text Model Load Failed", message: err.message, danger: true }); - } - } finally { - setLoadingModel(null); - setIsBusy(false); - } - }; - - const handleCancelLlmLoad = async () => { - try { - await stopLlm(); - } catch (_) {} - setLoadingModel(null); - setIsBusy(false); - setSelectedModel(""); - }; - - const handleNewChat = () => { - setActiveConversationId(null); - }; - - const sendMessage = async () => { - const text = input.trim(); - const hasAttachments = attachments.length > 0; - if ((!text && !hasAttachments) || isBusy || !status.ready) return; - - let convId = activeConversationId; - let isNew = false; - if (!convId) { - convId = "chat_" + Date.now(); - setActiveConversationId(convId); - isNew = true; - } - - // Delimited context blocks for document attachments - let documentContext = ""; - attachments.filter(att => att.type === "document").forEach((att) => { - documentContext += `\n[Attached File: ${att.name}]\n${att.content}\n`; - }); - - const combinedText = documentContext ? `${text}\n\n${documentContext}`.trim() : text; - const imageAttachments = attachments.filter(att => att.type === "image"); - - let userMessageContent; - if (imageAttachments.length > 0) { - userMessageContent = [ - { - type: "text", - text: combinedText - }, - ...imageAttachments.map((img) => ({ - type: "image_url", - image_url: { - url: img.dataUrl - } - })) - ]; - } else { - userMessageContent = combinedText; - } - - const nextMessages = [...messages, { role: "user", content: userMessageContent }]; - setMessages(nextMessages); - setInput(""); - setIsBusy(true); - - const displayTitleText = text || (imageAttachments.length > 0 ? "Sent Image" : "Sent File"); - const firstTitle = isNew ? (displayTitleText.slice(0, 26) + (displayTitleText.length > 26 ? "..." : "")) : null; - saveConversationState(convId, nextMessages, selectedModel, firstTitle); - - const requestStartedAt = performance.now(); - setMessages([...nextMessages, { - role: "assistant", - content: "", - generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 }, - }]); - - const controller = new AbortController(); - abortControllerRef.current = controller; - - try { - const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant."; - const requestMessages = [ - ...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []), - ...nextMessages, - ]; - - let assistantText = ""; - let assistantReasoning = ""; - let streamedTokens = 0; - let firstTokenAt = null; - let thinkingStartedAt = null; - let thinkingEndedAt = null; - let thinkingDuration = 0; - - const response = await streamChatWithLlm(requestMessages, { - temperature: textSettings?.temperature || 0.7, - maxTokens: 768, - signal: controller.signal, - }, (_token, fullText, _reasoningToken, fullReasoning) => { - const now = performance.now(); - if (streamedTokens === 0) { - firstTokenAt = now; - } - streamedTokens += 1; - const generationSeconds = firstTokenAt - ? Math.max(0.05, (now - firstTokenAt) / 1000) - : Math.max(0.05, (now - requestStartedAt) / 1000); - - const processed = processMessageContent(fullText, fullReasoning); - assistantText = processed.content; - assistantReasoning = processed.reasoning; - - // Calculate thinking duration - if (processed.reasoning && !thinkingStartedAt) { - thinkingStartedAt = now; - } - if (processed.content && thinkingStartedAt && !thinkingEndedAt) { - thinkingEndedAt = now; - thinkingDuration = (thinkingEndedAt - thinkingStartedAt) / 1000; - } - const currentThinkingDuration = thinkingEndedAt - ? thinkingDuration - : (thinkingStartedAt ? (now - thinkingStartedAt) / 1000 : 0); - - setMessages((prev) => { - const updated = [...prev]; - if (updated.length > 0 && updated[updated.length - 1].role === "assistant") { - updated[updated.length - 1] = { - ...updated[updated.length - 1], - content: processed.content, - reasoning: processed.reasoning, - thinkingDuration: currentThinkingDuration, - generationStats: { - status: "streaming", - tokens: streamedTokens, - tokensPerSecond: streamedTokens / generationSeconds, - seconds: (now - requestStartedAt) / 1000, - }, - }; - } - return updated; - }); - }); - - const completedAt = performance.now(); - let finalThinkingDuration = thinkingDuration; - if (thinkingStartedAt && !thinkingEndedAt) { - thinkingEndedAt = completedAt; - finalThinkingDuration = (thinkingEndedAt - thinkingStartedAt) / 1000; - } - - const exactTokens = Number(response.timings?.predicted_n) || streamedTokens; - const backendTotalMs = Number(response.timings?.prompt_ms || 0) + Number(response.timings?.predicted_ms || 0); - const exactSeconds = backendTotalMs > 0 - ? backendTotalMs / 1000 - : (completedAt - requestStartedAt) / 1000; - const exactTokensPerSecond = Number(response.timings?.predicted_per_second) - || (exactTokens / Math.max(0.001, exactSeconds)); - const generationStats = { - status: "complete", - tokens: exactTokens, - tokensPerSecond: exactTokensPerSecond, - seconds: exactSeconds, - }; - - const processed = processMessageContent(assistantText, response.reasoningContent || assistantReasoning); - const finalMessages = [...nextMessages, { - role: "assistant", - content: processed.content, - reasoning: processed.reasoning, - thinkingDuration: finalThinkingDuration, - generationStats, - }]; - setMessages(finalMessages); - saveConversationState(convId, finalMessages, selectedModel); - if (response.usage) setTokenUsage(response.usage); - - // Clean up attached files on success - setAttachments([]); - } catch (err) { - if (err.name === "AbortError") { - setMessages((prev) => { - const updated = [...prev]; - if (updated.length > 0 && updated[updated.length - 1].role === "assistant") { - const lastMsg = updated[updated.length - 1]; - updated[updated.length - 1] = { - ...lastMsg, - generationStats: lastMsg.generationStats ? { - ...lastMsg.generationStats, - status: "complete", - } : null - }; - saveConversationState(convId, updated, selectedModel); - } - return updated; - }); - } else { - const finalMessages = [...nextMessages, { role: "assistant", content: `Error: ${err.message}`, error: true }]; - setMessages(finalMessages); - saveConversationState(convId, finalMessages, selectedModel); - } - } finally { - setIsBusy(false); - abortControllerRef.current = null; - } - }; - - const handleClearChat = () => { - setMessages([]); - setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }); - if (activeConversationId) { - saveConversationState(activeConversationId, [], selectedModel); - } - }; - - return ( -
-
- {/* ─── Header ─────────────────────────────────────────── */} -
-
- - - - {isBusy && !loadingModel && } - {selectedModel && (!status.ready || status.settings?.model !== selectedModel) && !loadingModel && ( - - )} -
- -
- {/* Context gauge */} - {(() => { - const maxTokens = status.settings?.contextSize || 4096; - const used = tokenUsage.total_tokens || 0; - const percent = Math.min(100, Math.round((used / maxTokens) * 100)); - return ( -
-
- - - - -
- {percent}% -
-
-
- Context - {used} / {maxTokens} -
-
- ); - })()} - - -
-
- - {/* ─── Messages area ──────────────────────────────────── */} -
- {loadingModel ? ( -
- -

Loading Text Model

- - {loadingModel} - -

- Initializing llama.cpp server and loading the model weights into memory. This can take up to 30 seconds depending on model size and hardware speed. -

- -
- ) : ( - <> - {messages.length === 0 && ( -
-
- -
-

Local AI Chat

-

Your private, offline AI assistant. Choose a GGUF model above and start a conversation — everything stays on your machine.

- {status.ready && ( -
- {[ - { icon: "✍️", text: "Write a professional email to reschedule a meeting" }, - { icon: "💡", text: "Explain how transformers work in simple terms" }, - { icon: "🐛", text: "Help me debug this Python code" }, - { icon: "📋", text: "Summarize the key points of a topic" }, - ].map((s, i) => ( - - ))} -
- )} -
- )} - - {messages.map((message, index) => { - const processed = processMessageContent( - Array.isArray(message.content) ? "" : (message.content || ""), - message.reasoning || "" - ); - const displayContent = Array.isArray(message.content) ? message.content : processed.content; - const displayReasoning = processed.reasoning; - - return ( -
- {/* Avatar */} -
- {message.role === "user" ? "You" : "AI"} -
- - {/* Bubble + stats */} -
- - {message.role === "user" ? "You" : "Local AI"} - - {message.role === "assistant" && displayReasoning && textSettings?.enableThinking !== false && ( - - )} -
- {Array.isArray(displayContent) ? ( -
- {displayContent.map((item, idx) => { - if (item.type === "text") return ; - if (item.type === "image_url") return ( - Attached image - ); - return null; - })} -
- ) : ( - - )} -
- - {/* Generation stats pill */} - {message.role === "assistant" && message.generationStats && !message.error && ( -
- {message.generationStats.status === "starting" ? ( - <> Waiting for first token... - ) : message.generationStats.status === "streaming" ? ( - <>⚡ {message.generationStats.tokensPerSecond.toFixed(1)} tok/s - ) : ( - <>{message.generationStats.tokens} tokens • {formatGenerationTime(message.generationStats.seconds)} - )} -
- )} -
-
- ); - })} - - )} -
-
- - {/* ─── Composer ───────────────────────────────────────── */} -
- {/* Attachment previews */} - {attachments.length > 0 && ( -
- {attachments.map((att) => ( -
- {att.type === "image" ? ( - {att.name} - ) : ( - 📄 - )} - - {att.name} - - -
- ))} -
- )} - - {/* Input row */} -
- - - -
-