From 8fa0a670c185e142806825870fa5551a86640be8 Mon Sep 17 00:00:00 2001 From: techjarves Date: Sun, 21 Jun 2026 13:53:40 +0530 Subject: [PATCH] add text chat UI with LLM configuration and infrastructure support for the new backend port --- app/frontend/src/App.css | 147 ++++++-- app/frontend/src/App.jsx | 44 ++- app/frontend/src/components/ModelManager.jsx | 66 +++- app/frontend/src/components/Settings.jsx | 28 +- app/frontend/src/components/TextChat.jsx | 168 ++++++--- app/frontend/src/services/api.js | 1 + linux.sh | 9 +- mac.sh | 9 +- scripts/serve.cjs | 347 ++++++++++++++----- windows.bat | 7 +- 10 files changed, 631 insertions(+), 195 deletions(-) diff --git a/app/frontend/src/App.css b/app/frontend/src/App.css index 45ab2e6..8202fde 100644 --- a/app/frontend/src/App.css +++ b/app/frontend/src/App.css @@ -677,6 +677,10 @@ body { animation: msgSlideIn 0.22s cubic-bezier(0.22, 1, 0.36, 1) both; } +.chat-message-row.ai { + align-items: flex-start; +} + @keyframes msgSlideIn { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: translateY(0); } @@ -714,7 +718,9 @@ body { .chat-avatar.ai { background: linear-gradient(135deg, var(--md-sys-color-primary), color-mix(in srgb, var(--md-sys-color-primary) 70%, #7c3aed)); color: var(--md-sys-color-on-primary); - box-shadow: 0 2px 8px color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent); + box-shadow: + 0 0 0 3px color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent), + 0 8px 20px color-mix(in srgb, var(--md-sys-color-primary) 28%, transparent); } .chat-avatar.user { @@ -728,7 +734,7 @@ body { display: flex; flex-direction: column; max-width: min(1180px, 92%); - gap: 4px; + gap: 5px; } .chat-message-row.user .chat-bubble-wrap { @@ -777,12 +783,13 @@ body { /* Sender label */ .chat-sender-label { - font-size: 0.7rem; - font-weight: 700; - letter-spacing: 0.06em; + font-size: 0.66rem; + font-weight: 800; + letter-spacing: 0.08em; text-transform: uppercase; color: var(--md-sys-color-outline); padding: 0 2px; + line-height: 1.1; } /* ─── Generation Stats ────────────────────────────────────────── */ @@ -802,23 +809,23 @@ body { display: inline-flex; align-items: center; gap: 6px; - margin-top: 4px; - padding: 3px 10px; + margin-top: 2px; + padding: 4px 10px; border-radius: 99px; - background: color-mix(in srgb, var(--md-sys-color-surface-variant) 70%, transparent); - border: 1px solid var(--border-color); + background: color-mix(in srgb, var(--md-sys-color-surface-variant) 82%, transparent); + border: 1px solid color-mix(in srgb, var(--border-color) 88%, var(--md-sys-color-primary) 12%); color: var(--md-sys-color-on-surface-variant); - font-size: 0.7rem; - font-weight: 600; - backdrop-filter: blur(4px); + font-size: 0.68rem; + font-weight: 800; + backdrop-filter: blur(8px); line-height: 1.4; } .chat-generation-stats.streaming { color: var(--md-sys-color-primary); - border-color: color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent); - background: color-mix(in srgb, var(--md-sys-color-primary) 8%, var(--md-sys-color-surface-variant)); - animation: statsGlow 1.5s ease-in-out infinite; + border-color: color-mix(in srgb, var(--md-sys-color-primary) 38%, transparent); + background: color-mix(in srgb, var(--md-sys-color-primary) 10%, var(--md-sys-color-surface-variant)); + box-shadow: 0 6px 18px color-mix(in srgb, var(--md-sys-color-primary) 10%, transparent); } @keyframes statsGlow { @@ -2582,54 +2589,128 @@ html[data-theme^="dark"] .m3-segment-item:hover:not(.active):not(.disabled) { /* Chat Thinking Block (Reasoning Process) */ .chat-thinking-container { - margin-bottom: 6px; - border-radius: 8px; - border: 1px dashed rgba(255, 255, 255, 0.15); - background: rgba(255, 255, 255, 0.03); + margin-bottom: 4px; + width: fit-content; + max-width: min(420px, 100%); + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--md-sys-color-primary) 24%, var(--border-color)); + background: + linear-gradient(135deg, + color-mix(in srgb, var(--md-sys-color-primary) 10%, var(--md-sys-color-surface-variant)), + color-mix(in srgb, var(--md-sys-color-surface-variant) 88%, transparent)); + box-shadow: 0 8px 24px color-mix(in srgb, var(--md-sys-color-primary) 9%, transparent); overflow: hidden; - max-width: 100%; + backdrop-filter: blur(10px); +} + +.chat-thinking-container:has(.chat-thinking-content) { + border-radius: 13px; + max-width: min(720px, 92vw); } .chat-thinking-header { - width: 100%; + width: auto; + min-width: 0; display: flex; align-items: center; justify-content: space-between; - padding: 6px 10px; - background: rgba(255, 255, 255, 0.05); + gap: 10px; + padding: 7px 11px; + background: transparent; border: none; color: var(--md-sys-color-on-surface-variant); - font-size: 0.76rem; - font-weight: 500; + font-size: 0.75rem; + font-weight: 800; cursor: pointer; text-align: left; - transition: background 0.2s; + transition: background 0.18s ease, color 0.18s ease; outline: none; } .chat-thinking-header:hover { - background: rgba(255, 255, 255, 0.08); + background: color-mix(in srgb, var(--md-sys-color-primary) 7%, transparent); + color: var(--md-sys-color-on-surface); } .chat-thinking-title { display: flex; align-items: center; - gap: 6px; - opacity: 0.8; + gap: 8px; + opacity: 0.92; + white-space: nowrap; +} + +.chat-thinking-title::before { + content: ""; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--md-sys-color-primary); + box-shadow: 0 0 0 4px color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent); + animation: thinkingPulse 1.35s ease-in-out infinite; +} + +@keyframes thinkingPulse { + 0%, 100% { + transform: scale(0.9); + opacity: 0.72; + } + 50% { + transform: scale(1.12); + opacity: 1; + } } .chat-thinking-content { - padding: 8px 10px; + padding: 9px 12px 10px; font-size: 0.76rem; line-height: 1.45; color: var(--md-sys-color-on-surface-variant); - opacity: 0.75; - border-top: 1px dashed rgba(255, 255, 255, 0.1); + opacity: 0.78; + border-top: 1px solid color-mix(in srgb, var(--border-color) 78%, transparent); white-space: pre-wrap; font-family: var(--font-family-monospace, monospace); - background: rgba(0, 0, 0, 0.1); + background: color-mix(in srgb, var(--md-sys-color-surface) 28%, transparent); max-height: 150px; overflow-y: auto; + scrollbar-width: thin; + scrollbar-color: + color-mix(in srgb, var(--md-sys-color-primary) 38%, var(--md-sys-color-outline)) + color-mix(in srgb, var(--md-sys-color-surface) 65%, transparent); +} + +.chat-thinking-content::-webkit-scrollbar { + width: 10px; +} + +.chat-thinking-content::-webkit-scrollbar-track { + background: color-mix(in srgb, var(--md-sys-color-surface) 60%, transparent); + border-left: 1px solid color-mix(in srgb, var(--border-color) 70%, transparent); +} + +.chat-thinking-content::-webkit-scrollbar-thumb { + min-height: 34px; + border-radius: 999px; + border: 2px solid transparent; + background: + linear-gradient( + 180deg, + color-mix(in srgb, var(--md-sys-color-primary) 62%, #ffffff 8%), + color-mix(in srgb, var(--md-sys-color-primary) 38%, var(--md-sys-color-outline)) + ) + border-box; + background-clip: padding-box; +} + +.chat-thinking-content::-webkit-scrollbar-thumb:hover { + background: + linear-gradient( + 180deg, + color-mix(in srgb, var(--md-sys-color-primary) 78%, #ffffff 10%), + color-mix(in srgb, var(--md-sys-color-primary) 52%, var(--md-sys-color-outline)) + ) + border-box; + background-clip: padding-box; } /* ============================================================ diff --git a/app/frontend/src/App.jsx b/app/frontend/src/App.jsx index 139c5aa..0449505 100644 --- a/app/frontend/src/App.jsx +++ b/app/frontend/src/App.jsx @@ -122,7 +122,8 @@ function App() { return { ...parsed, enableThinking: parsed.enableThinking === true, - maxTokens: parsed.maxTokens || 384, + maxTokens: !parsed.maxTokens || parsed.maxTokens === 384 ? 1024 : parsed.maxTokens, + responseTokenMode: parsed.responseTokenMode || "auto", }; } catch (_) {} } @@ -134,7 +135,8 @@ function App() { enableThinking: false, // New performance settings gpuLayers: -1, - maxTokens: 384, + maxTokens: 1024, + responseTokenMode: "auto", topP: 0.95, topK: 40, minP: 0.05, @@ -167,10 +169,42 @@ function App() { if (saved) { try { setConversations(JSON.parse(saved)); - } catch (_) {} + } catch (_) { + localStorage.removeItem("chat_conversations"); + } } }, []); + const sanitizeMessageForStorage = (message) => { + if (!Array.isArray(message?.content)) return message; + return { + ...message, + content: message.content.map((item) => { + if (item?.type !== "image_url") return item; + return { + type: "text", + text: "[Attached image omitted from saved chat history]", + }; + }), + }; + }; + + const sanitizeConversationForStorage = (conversation) => ({ + ...conversation, + messages: Array.isArray(conversation.messages) + ? conversation.messages.map(sanitizeMessageForStorage) + : [], + }); + + const persistConversations = (list) => { + const compactList = list.map(sanitizeConversationForStorage); + try { + localStorage.setItem("chat_conversations", JSON.stringify(compactList)); + } catch (err) { + console.warn("Could not save chat history:", err); + } + }; + const saveConversationState = useCallback((id, msgs, modelName, newTitle = null) => { setConversations((prev) => { const list = [...prev]; @@ -192,7 +226,7 @@ function App() { timestamp: Date.now() }); } - localStorage.setItem("chat_conversations", JSON.stringify(list)); + persistConversations(list); return list; }); }, []); @@ -201,7 +235,7 @@ function App() { if (e) e.stopPropagation(); setConversations((prev) => { const filtered = prev.filter((c) => c.id !== id); - localStorage.setItem("chat_conversations", JSON.stringify(filtered)); + persistConversations(filtered); return filtered; }); if (activeConversationId === id) { diff --git a/app/frontend/src/components/ModelManager.jsx b/app/frontend/src/components/ModelManager.jsx index c3a84ab..dd76c05 100644 --- a/app/frontend/src/components/ModelManager.jsx +++ b/app/frontend/src/components/ModelManager.jsx @@ -249,6 +249,7 @@ function ModelManager({ const [hasMoreModels, setHasMoreModels] = useState(false); const [isLoadingMoreModels, setIsLoadingMoreModels] = useState(false); const searchRequestRef = React.useRef(0); + const pendingCompanionDownloadRef = React.useRef(null); const cancelLoadRef = React.useRef(false); @@ -261,7 +262,10 @@ function ModelManager({ } }; - const modelNames = localModels.map((model) => normalizeModel(model).filename); + const normalizedLocalModels = localModels.map(normalizeModel).filter((model) => model.filename); + const displayedLocalModels = normalizedLocalModels.filter((model) => !model.isProjector); + const allModelNames = normalizedLocalModels.map((model) => model.filename); + const modelNames = displayedLocalModels.map((model) => model.filename); const isBusy = loadingModelId !== null || isUnloading; const openvinoSupported = Boolean(backendOptions?.openvinoNpu?.supported); @@ -430,10 +434,10 @@ function ModelManager({ } }; - const downloadByUrl = async (url, expectedFilename) => { + const downloadByUrl = async (url, expectedFilename, companion = null, options = {}) => { if (downloadingModelId) return; - if (modelNames.includes(expectedFilename)) { + if (!options.skipExistingConfirm && allModelNames.includes(expectedFilename)) { if (!(await showConfirm({ title: "Overwrite Model?", message: `"${expectedFilename}" is already downloaded. Do you want to download and overwrite it?`, @@ -452,6 +456,7 @@ function ModelManager({ } try { + pendingCompanionDownloadRef.current = companion; let res; if (activeModelType === "image") { res = await downloadModel(url); @@ -461,9 +466,11 @@ function ModelManager({ if (res && res.ok) { startProgressPolling(expectedFilename); } else { + pendingCompanionDownloadRef.current = null; showAlert({ title: "Download Failed", message: res.error || "Unknown error", danger: true }); } } catch (err) { + pendingCompanionDownloadRef.current = null; showAlert({ title: "Download Error", message: err.message, danger: true }); } }; @@ -486,11 +493,15 @@ function ModelManager({ } } else { fetchModels(); - if (modelId === "ggml-model-q4_k.gguf") { + const companion = pendingCompanionDownloadRef.current; + pendingCompanionDownloadRef.current = null; + if (companion?.url && companion?.filename && !allModelNames.includes(companion.filename)) { setTimeout(() => { downloadByUrl( - "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf", - "mmproj-model-f16.gguf" + companion.url, + companion.filename, + null, + { skipExistingConfirm: true } ); }, 500); } @@ -538,13 +549,35 @@ function ModelManager({ return; } + const hasMainModel = modelNames.includes(model.filename); + const hasProjector = !model.projectorFilename || allModelNames.includes(model.projectorFilename); + if (hasMainModel && model.projectorUrl && model.projectorFilename && !hasProjector) { + const confirmed = await showConfirm({ + title: "Download Vision Projector?", + message: `"${model.name}" is downloaded, but image input needs the matching projector file (${model.projectorFilename}). Download it now?`, + confirmLabel: "Download Projector", + }); + if (confirmed) { + await downloadByUrl(model.projectorUrl, model.projectorFilename, null, { skipExistingConfirm: true }); + } + return; + } + const confirmed = await showConfirm({ title: "Download Model?", - message: `Download "${model.name}" (~${model.approxSize})?`, + message: model.projectorUrl + ? `Download "${model.name}" (~${model.approxSize}) and its required vision projector?` + : `Download "${model.name}" (~${model.approxSize})?`, confirmLabel: "Download", }); if (confirmed) { - await downloadByUrl(model.url, model.filename); + await downloadByUrl( + model.url, + model.filename, + model.projectorUrl && model.projectorFilename + ? { url: model.projectorUrl, filename: model.projectorFilename } + : null + ); } }; @@ -627,7 +660,7 @@ function ModelManager({ device: "", }); const cores = textSettings?.threads || specs?.cpu_cores_physical || 4; - const context = textSettings?.contextSize || 4096; + const context = textSettings?.contextSize ?? 0; const res = await startLlm(modelId, { threads: cores, contextSize: context, @@ -1066,7 +1099,7 @@ function ModelManager({

- Local {activeModelType === "image" ? "Image" : "Text"} Models ({localModels.length}) + Local {activeModelType === "image" ? "Image" : "Text"} Models ({displayedLocalModels.length})

{isLoadingModels ? ( @@ -1074,7 +1107,7 @@ function ModelManager({ Scanning models folder...
- ) : localModels.length === 0 ? ( + ) : displayedLocalModels.length === 0 ? (

{activeModelType === "image" ? "No image models detected in app/models/. Download from the library below or import a file." @@ -1082,7 +1115,7 @@ function ModelManager({

) : (
- {localModels.map((model) => { + {displayedLocalModels.map((model) => { const filename = model.filename; const isActive = activeModelType === "image" ? activeModel === filename : activeLlmModel === filename; @@ -1221,8 +1254,11 @@ function ModelManager({

{section.group}

{section.items.map((model) => { - const installed = modelNames.includes(model.filename); - const downloading = downloadingModelId === model.filename || downloadingModelId === `${model.filename}.zip`; + const hasMainModel = modelNames.includes(model.filename); + const hasProjector = !model.projectorFilename || allModelNames.includes(model.projectorFilename); + const needsProjector = Boolean(hasMainModel && model.projectorFilename && !hasProjector); + const installed = hasMainModel && hasProjector; + const downloading = downloadingModelId === model.filename || downloadingModelId === model.projectorFilename || downloadingModelId === `${model.filename}.zip`; const systemTier = getHardwareTier(specs); const isRecommended = typeof model.recommendedFit === "boolean" ? model.recommendedFit @@ -1289,7 +1325,7 @@ function ModelManager({ disabled={installed || downloadingModelId !== null} > {installed ? : downloading ? : } - {installed ? "Downloaded" : downloading ? "Downloading" : "Download"} + {installed ? "Downloaded" : downloading ? "Downloading" : needsProjector ? "Download Vision File" : "Download"} ({ threads: settings?.threads || specs?.cpu_cores_physical || 4, - contextSize: settings?.contextSize || 4096, + contextSize: settings?.contextSize ?? 0, gpuLayers: settings?.gpuLayers ?? -1, enableThinking: settings?.enableThinking !== false, flashAttn: settings?.flashAttn, @@ -685,17 +685,37 @@ function Settings({
Max Response Tokens - {textSettings.maxTokens || 384} + + {(textSettings.responseTokenMode || "auto") === "auto" ? "Auto" : (textSettings.maxTokens || 1024)} + +
+
+ {[ + { id: "auto", label: "Auto" }, + { id: "manual", label: "Manual" }, + ].map((mode) => ( + + ))}
updateTextSetting("maxTokens", parseInt(e.target.value))} min="64" - max="2048" + max="4096" step="64" + disabled={(textSettings.responseTokenMode || "auto") === "auto"} /> + + Auto uses remaining context with a safety buffer. Manual uses the slider value. +
diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx index 9b08da8..01b9a63 100644 --- a/app/frontend/src/components/TextChat.jsx +++ b/app/frontend/src/components/TextChat.jsx @@ -150,6 +150,17 @@ function TextChat({ const wordCount = value.split(/\s+/).filter(Boolean).length; return Math.max(wordCount, Math.ceil(value.length / 4)); }; + + const clamp = (value, min, max) => Math.max(min, Math.min(max, value)); + + const getAutoMaxResponseTokens = (promptTokens, thinkingEnabled) => { + const contextLimit = Number(status.settings?.contextSize || textSettings?.contextSize || 4096); + const safetyBuffer = thinkingEnabled ? 768 : 512; + const preferredMinimum = thinkingEnabled ? 512 : 256; + const available = contextLimit - promptTokens - safetyBuffer; + if (available <= 0) return 64; + return clamp(available, Math.min(preferredMinimum, available), 4096); + }; const bottomRef = useRef(null); const textareaRef = useRef(null); @@ -192,14 +203,45 @@ function TextChat({ return /\.(txt|md|csv|js|jsx|ts|tsx|py|json|css|html|java|cpp|c|h|rs|go|sh|bat|xml|yaml|yml)$/i.test(file.name) || file.type.startsWith("text/"); }; + const optimizeImageForVision = (file, maxSide = 1024, quality = 0.92) => new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => reject(new Error(`Could not read ${file.name}`)); + reader.onload = () => { + const originalDataUrl = reader.result; + const img = new Image(); + img.onerror = () => reject(new Error(`Could not decode ${file.name}`)); + img.onload = () => { + const scale = Math.min(1, maxSide / Math.max(img.width, img.height)); + const width = Math.max(1, Math.round(img.width * scale)); + const height = Math.max(1, Math.round(img.height * scale)); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0, width, height); + const sendDataUrl = canvas.toDataURL("image/jpeg", quality); + resolve({ + previewDataUrl: originalDataUrl, + sendDataUrl, + originalWidth: img.width, + originalHeight: img.height, + width, + height, + }); + }; + img.src = originalDataUrl; + }; + reader.readAsDataURL(file); + }); + const handleFileChange = (e) => { const files = Array.from(e.target.files); if (!files || files.length === 0) return; files.forEach((file) => { if (isImage(file)) { - const reader = new FileReader(); - reader.onload = (event) => { + optimizeImageForVision(file) + .then((image) => { setAttachments((prev) => [ ...prev, { @@ -207,11 +249,18 @@ function TextChat({ file, type: "image", name: file.name, - dataUrl: event.target.result, + dataUrl: image.previewDataUrl, + sendDataUrl: image.sendDataUrl, + width: image.width, + height: image.height, + originalWidth: image.originalWidth, + originalHeight: image.originalHeight, }, ]); - }; - reader.readAsDataURL(file); + }) + .catch((err) => { + showAlert({ title: "Image Error", message: err.message || String(err), danger: true }); + }); } else if (isTextFile(file)) { const reader = new FileReader(); reader.onload = (event) => { @@ -279,7 +328,7 @@ function TextChat({ const buildTextStartOptions = (settings) => ({ threads: settings?.threads || specs?.cpu_cores_physical || 4, - contextSize: settings?.contextSize || 4096, + contextSize: settings?.contextSize ?? 0, gpuLayers: settings?.gpuLayers ?? -1, enableThinking: settings?.enableThinking !== false, flashAttn: settings?.flashAttn, @@ -360,13 +409,14 @@ function TextChat({ const refresh = useCallback(async () => { const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]); - setModels(nextModels); + const selectableModels = nextModels.filter((model) => !model.isProjector); + setModels(selectableModels); setStatus(nextStatus); const active = nextStatus.settings?.model; setSelectedModel((current) => { const saved = localStorage.getItem("selectedLlmModel"); - const savedExists = nextModels.some((m) => m.filename === saved); - return active || current || (savedExists ? saved : "") || nextModels[0]?.filename || ""; + const savedExists = selectableModels.some((m) => m.filename === saved); + return active || current || (savedExists ? saved : "") || selectableModels[0]?.filename || ""; }); }, []); @@ -453,7 +503,7 @@ function TextChat({ const result = await startLlm(filename, { threads: textSettings?.threads || specs?.cpu_cores_physical || 4, - contextSize: textSettings?.contextSize || 4096, + contextSize: textSettings?.contextSize ?? 0, gpuLayers: textSettings?.gpuLayers ?? -1, enableThinking: textSettings?.enableThinking !== false, flashAttn: textSettings?.flashAttn, @@ -512,15 +562,29 @@ function TextChat({ documentContext += `\n[Attached File: ${att.name}]\n${att.content}\n`; }); - const combinedText = documentContext ? `${text}\n\n${documentContext}`.trim() : text; const imageAttachments = attachments.filter(att => att.type === "image"); + const visionInstruction = imageAttachments.length > 0 + ? "For attached images: answer in natural language, describe the actual image content, and read visible text carefully when relevant. Do not output raw JSON, bounding boxes, OCR layout objects, or detection labels unless the user explicitly asks for that format." + : ""; + const requestText = imageAttachments.length > 0 + ? (text && text !== "?" ? text : "Describe what is visible in the image.") + : text; + const displayText = text || (imageAttachments.length > 0 ? "Describe what is visible in the image." : ""); + const requestCombinedText = [ + requestText, + documentContext, + ].filter(Boolean).join("\n\n").trim(); + const displayCombinedText = documentContext + ? [displayText, documentContext].filter(Boolean).join("\n\n").trim() + : displayText; let userMessageContent; + let requestUserMessageContent; if (imageAttachments.length > 0) { userMessageContent = [ { type: "text", - text: combinedText + text: displayCombinedText }, ...imageAttachments.map((img) => ({ type: "image_url", @@ -529,11 +593,25 @@ function TextChat({ } })) ]; + requestUserMessageContent = [ + { + type: "text", + text: requestCombinedText + }, + ...imageAttachments.map((img) => ({ + type: "image_url", + image_url: { + url: img.sendDataUrl || img.dataUrl + } + })) + ]; } else { - userMessageContent = combinedText; + userMessageContent = displayCombinedText; + requestUserMessageContent = requestCombinedText; } const nextMessages = [...messages, { role: "user", content: userMessageContent }]; + const requestConversationMessages = [...messages, { role: "user", content: requestUserMessageContent }]; followGenerationRef.current = true; setMessages(nextMessages); setInput(""); @@ -547,7 +625,7 @@ function TextChat({ setMessages([...nextMessages, { role: "assistant", content: "", - generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 }, + generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0, vision: imageAttachments.length > 0 }, }]); const controller = new AbortController(); @@ -555,9 +633,13 @@ function TextChat({ try { const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant."; + const combinedSystemPrompt = [ + systemPrompt.trim(), + visionInstruction, + ].filter(Boolean).join("\n\n"); const requestMessages = [ - ...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []), - ...nextMessages, + ...(combinedSystemPrompt ? [{ role: "system", content: combinedSystemPrompt }] : []), + ...requestConversationMessages, ]; const promptTokenEstimate = requestMessages.reduce((sum, message) => { const messageText = Array.isArray(message.content) @@ -579,10 +661,11 @@ function TextChat({ let thinkingEndedAt = null; let thinkingDuration = 0; - const baseMaxTokens = textSettings?.maxTokens || 384; - const effectiveMaxTokens = textSettings?.enableThinking !== false - ? Math.max(baseMaxTokens, 1024) - : baseMaxTokens; + const thinkingEnabled = textSettings?.enableThinking !== false; + const manualMaxTokens = textSettings?.maxTokens || 1024; + const effectiveMaxTokens = textSettings?.responseTokenMode === "manual" + ? (thinkingEnabled ? Math.max(manualMaxTokens, 1024) : manualMaxTokens) + : getAutoMaxResponseTokens(promptTokenEstimate, thinkingEnabled); const response = await streamChatWithLlm(requestMessages, { temperature: textSettings?.temperature || 0.7, @@ -592,7 +675,7 @@ function TextChat({ minP: textSettings?.minP, repeatPenalty: textSettings?.repeatPenalty, seed: textSettings?.seed, - enableThinking: textSettings?.enableThinking !== false, + enableThinking: thinkingEnabled, signal: controller.signal, }, (_token, fullText, _reasoningToken, fullReasoning) => { const now = performance.now(); @@ -929,6 +1012,9 @@ function TextChat({ ); const displayContent = Array.isArray(message.content) ? message.content : processed.content; const displayReasoning = processed.reasoning; + const hasDisplayContent = Array.isArray(displayContent) + ? displayContent.length > 0 + : Boolean(String(displayContent || "").trim()); return (
)} -
- {Array.isArray(displayContent) ? ( -
- {displayContent.map((item, idx) => { - if (item.type === "text") return ; - if (item.type === "image_url") return ( - Attached image - ); - return null; - })} -
- ) : ( - - )} -
+ {(hasDisplayContent || message.error) && ( +
+ {Array.isArray(displayContent) ? ( +
+ {displayContent.map((item, idx) => { + if (item.type === "text") return ; + if (item.type === "image_url") return ( + Attached image + ); + return null; + })} +
+ ) : ( + + )} +
+ )} {/* Generation stats pill */} {message.role === "assistant" && message.generationStats && !message.error && ( <> {message.generationStats.truncated && (
- Response reached the token limit. Increase Max Response Tokens or ask "continue". + Response reached the token limit. Ask "continue" or switch Max Response Tokens to Manual for a larger cap.
)}
{message.generationStats.status === "starting" ? ( - <> Waiting for first token... + <> {message.generationStats.vision ? "Processing image..." : "Waiting for first token..."} ) : message.generationStats.status === "streaming" ? ( <>⚡ {message.generationStats.tokensPerSecond.toFixed(1)} tok/s ) : ( diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js index 6f56907..9c529e3 100644 --- a/app/frontend/src/services/api.js +++ b/app/frontend/src/services/api.js @@ -54,6 +54,7 @@ export function normalizeModel(model) { format: model?.format || "Local Weights File", backendType: model?.backendType || "", resolution: model?.resolution || "", + isProjector: Boolean(model?.isProjector), }; } diff --git a/linux.sh b/linux.sh index acce2af..828d285 100755 --- a/linux.sh +++ b/linux.sh @@ -27,6 +27,7 @@ SETUP_SCRIPT="$SCRIPT_DIR/scripts/setup.sh" SERVE_SCRIPT="$SCRIPT_DIR/scripts/serve.cjs" FRONTEND_PORT="${FRONTEND_PORT:-1420}" +LLM_PORT="${LLM_PORT:-10086}" SETUP_REASON="" SETUP_MODE="Repair" MAX_PERF_FLAG="" @@ -155,10 +156,11 @@ if [[ -n "$SETUP_REASON" ]]; then # Clear any existing frontend and backend server processes if command -v lsof >/dev/null 2>&1; then - lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true + lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true elif command -v fuser >/dev/null 2>&1; then fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true fuser -k "8080/tcp" >/dev/null 2>&1 || true + fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true fi if ! bash "$SETUP_SCRIPT" $MAX_PERF_FLAG; then @@ -179,10 +181,11 @@ echo "" # Clear frontend and backend ports if command -v lsof >/dev/null 2>&1; then - lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true + lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true elif command -v fuser >/dev/null 2>&1; then fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true fuser -k "8080/tcp" >/dev/null 2>&1 || true + fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true fi # Start the server @@ -210,7 +213,7 @@ echo " ============================================================" echo " Running!" echo " Web UI: http://localhost:${FRONTEND_PORT}" echo " GPU API: Auto-selected by the app (starts at 8080)" -echo " Text API: Starts when a GGUF model is loaded (port 10086)" +echo " Text API: Starts when a GGUF model is loaded (port ${LLM_PORT})" echo "" echo " Press Ctrl+C in this window to stop all services." echo " ============================================================" diff --git a/mac.sh b/mac.sh index 14bbdc5..7175b6c 100755 --- a/mac.sh +++ b/mac.sh @@ -30,6 +30,7 @@ SETUP_SCRIPT="$SCRIPT_DIR/scripts/setup.sh" SERVE_SCRIPT="$SCRIPT_DIR/scripts/serve.cjs" FRONTEND_PORT="${FRONTEND_PORT:-1420}" +LLM_PORT="${LLM_PORT:-10086}" SETUP_REASON="" SETUP_MODE="Repair" @@ -129,10 +130,11 @@ if [[ -n "$SETUP_REASON" ]]; then # Clear any existing frontend and backend server processes if command -v lsof >/dev/null 2>&1; then - lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true + lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true elif command -v fuser >/dev/null 2>&1; then fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true fuser -k "8080/tcp" >/dev/null 2>&1 || true + fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true fi if ! bash "$SETUP_SCRIPT"; then @@ -153,10 +155,11 @@ echo "" # Clear frontend and backend ports if command -v lsof >/dev/null 2>&1; then - lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true + lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true elif command -v fuser >/dev/null 2>&1; then fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true fuser -k "8080/tcp" >/dev/null 2>&1 || true + fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true fi # Start the server @@ -184,7 +187,7 @@ echo " ============================================================" echo " Running!" echo " Web UI: http://localhost:${FRONTEND_PORT}" echo " GPU API: Auto-selected by the app (starts at 8080)" -echo " Text API: Starts when a GGUF model is loaded (port 10086)" +echo " Text API: Starts when a GGUF model is loaded (port ${LLM_PORT})" echo "" echo " Press Ctrl+C in this window to stop all services." echo " ============================================================" diff --git a/scripts/serve.cjs b/scripts/serve.cjs index 769d17a..6361529 100644 --- a/scripts/serve.cjs +++ b/scripts/serve.cjs @@ -129,8 +129,10 @@ let openvinoModel = null; let openvinoWidth = null; let openvinoHeight = null; let llmProc = null; +let llmProcSeq = 0; let llmReady = false; let llmError = null; +let llmOperationQueue = Promise.resolve(); let llmSettings = { model: null, threads: Math.max(1, Math.min(16, os.cpus().length || 4)), @@ -152,6 +154,12 @@ let llmSettings = { ubatchSize: 512, performanceProfile: "balanced", }; + +function runExclusiveLlmOperation(operation) { + const run = llmOperationQueue.catch(() => {}).then(operation); + llmOperationQueue = run.catch(() => {}); + return run; +} let backendLoadState = { active: false, phase: "", @@ -1254,6 +1262,16 @@ function checkPort(port) { }); } +async function waitForPortAvailable(port, timeoutMs = 5000) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const status = await checkPort(port); + if (status.available) return true; + await new Promise((resolve) => setTimeout(resolve, 150)); + } + return false; +} + async function findAvailableBackendPort() { const preferred = await checkPort(PREFERRED_BACKEND_PORT); if (preferred.available) return PREFERRED_BACKEND_PORT; @@ -1285,69 +1303,62 @@ async function findAvailableLlmPort() { } function getLlmBackend() { + return getLlmBackendCandidates()[0] || null; +} + +function getLlmBackendCandidates() { + const candidates = []; + const includeUndetectedGpu = process.env.LOCALAI_LLM_TRY_UNDETECTED_GPU === "1"; + const add = (key, backendPath, mode, detected = true) => { + if (!backendPath || !fs.existsSync(backendPath)) return; + if (key !== "cpu" && !detected && !includeUndetectedGpu) return; + if (candidates.some((item) => item.path === backendPath)) return; + candidates.push({ key, path: backendPath, mode, detected }); + }; + if (osPlatform === "win32") { - if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) { - return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) { - return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) { - return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) { - return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) { - return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) { - return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" }; - } - return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" }; + add("cuda", LLM_BACKEND_PATHS.winCuda, "Auto (CUDA/CPU)", Boolean(detectLlamaCudaGpu())); + add("vulkan", LLM_BACKEND_PATHS.winVulkan, "Auto (Vulkan/CPU)", Boolean(detectLlamaVulkanGpu())); + add("sycl", LLM_BACKEND_PATHS.winSycl, "Auto (SYCL/CPU)", Boolean(detectLlamaSyclGpu())); + add("cpu", LLM_BACKEND_PATHS.winCpu, "CPU", true); } else if (osPlatform === "darwin") { - return { - path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64, - mode: process.arch === "arm64" ? "Metal GPU" : "CPU", - }; + add("metal", process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64, process.arch === "arm64" ? "Metal GPU" : "CPU", true); } else { - if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda) && detectLlamaCudaGpu()) { - return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl) && detectLlamaSyclGpu()) { - return { path: LLM_BACKEND_PATHS.linuxSycl, mode: "Auto (SYCL/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan) && detectLlamaVulkanGpu()) { - return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) { - return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl)) { - return { path: LLM_BACKEND_PATHS.linuxSycl, mode: "Auto (SYCL/CPU)" }; - } - if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) { - return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" }; - } - return { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" }; + add("cuda", LLM_BACKEND_PATHS.linuxCuda, "Auto (CUDA/CPU)", Boolean(detectLlamaCudaGpu())); + add("sycl", LLM_BACKEND_PATHS.linuxSycl, "Auto (SYCL/CPU)", Boolean(detectLlamaSyclGpu())); + add("vulkan", LLM_BACKEND_PATHS.linuxVulkan, "Auto (Vulkan/CPU)", Boolean(detectLlamaVulkanGpu())); + add("cpu", LLM_BACKEND_PATHS.linuxCpu, "CPU", true); } + + const priority = osPlatform === "win32" + ? { cuda: 0, vulkan: 1, sycl: 2, metal: 3, cpu: 9 } + : osPlatform === "darwin" + ? { metal: 0, cpu: 9 } + : { cuda: 0, sycl: 1, vulkan: 2, cpu: 9 }; + + return candidates.sort((a, b) => { + if (a.key === "cpu" && b.key !== "cpu") return 1; + if (b.key === "cpu" && a.key !== "cpu") return -1; + const detectedDelta = Number(b.detected) - Number(a.detected); + if (detectedDelta) return detectedDelta; + return (priority[a.key] ?? 8) - (priority[b.key] ?? 8); + }); } function getLlmModels() { try { return fs.readdirSync(LLM_MODELS) - .filter((filename) => { - const lower = filename.toLowerCase(); - return lower.endsWith(".gguf") && !lower.includes("mmproj"); - }) + .filter((filename) => filename.toLowerCase().endsWith(".gguf")) .map((filename) => { const stats = fs.statSync(path.join(LLM_MODELS, filename)); + const lower = filename.toLowerCase(); return { filename, name: filename, sizeBytes: stats.size, size: formatBytes(stats.size), format: "GGUF", + isProjector: lower.includes("mmproj"), }; }); } catch (_) { @@ -1355,11 +1366,35 @@ function getLlmModels() { } } -function pingLlmReady() { +function pingLlmReady(expectedModel = "") { return new Promise((resolve) => { const req = http.get(`http://127.0.0.1:${PORT_LLM}/v1/models`, (res) => { - res.resume(); - resolve(res.statusCode >= 200 && res.statusCode < 500); + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => { + if (res.statusCode < 200 || res.statusCode >= 500) { + resolve(false); + return; + } + if (!expectedModel) { + resolve(true); + return; + } + try { + const parsed = JSON.parse(body || "{}"); + const servedIds = Array.isArray(parsed.data) + ? parsed.data.map((item) => String(item.id || item.model || "")) + : []; + const expectedBase = path.basename(expectedModel).toLowerCase(); + resolve(servedIds.some((id) => { + const lower = id.toLowerCase(); + return lower === expectedBase || lower.includes(expectedBase); + })); + } catch (_) { + resolve(false); + } + }); }); req.setTimeout(1000, () => { req.destroy(); @@ -1369,39 +1404,66 @@ function pingLlmReady() { }); } -async function waitForLlmReady(maxAttempts = 240) { +async function waitForLlmReady(maxAttempts = 240, expectedProc = llmProc, expectedModel = "") { for (let attempt = 0; attempt < maxAttempts; attempt += 1) { - if (!llmProc) throw new Error(llmError || "llama.cpp exited during startup."); - if (await pingLlmReady()) { + if (!expectedProc || llmProc !== expectedProc) throw new Error(llmError || "llama.cpp exited during startup."); + if (await pingLlmReady(expectedModel)) { llmReady = true; return; } await new Promise((resolve) => setTimeout(resolve, 500)); } - throw new Error("llama.cpp did not become ready within 2 minutes."); + throw new Error(`llama.cpp did not become ready within ${Math.round(maxAttempts / 120)} minutes.`); } -function killLlm() { +function waitForChildExit(proc, timeoutMs) { return new Promise((resolve) => { - llmReady = false; - llmSettings.supportsVision = false; - llmSettings.visionMode = "none"; - llmSettings.visionStatus = "Projector not loaded"; - llmSettings.mmproj = null; - if (!llmProc) { - resolve(); + if (!proc || proc.exitCode !== null || proc.signalCode !== null) { + resolve(true); return; } - const proc = llmProc; - llmProc = null; - try { proc.kill("SIGTERM"); } catch (_) {} - setTimeout(() => { - try { proc.kill("SIGKILL"); } catch (_) {} - resolve(); - }, 1200); + let done = false; + const finish = (exited) => { + if (done) return; + done = true; + clearTimeout(timer); + proc.off("exit", onExit); + resolve(exited); + }; + const onExit = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + proc.once("exit", onExit); }); } +async function killLlm() { + llmReady = false; + llmSettings.supportsVision = false; + llmSettings.visionMode = "none"; + llmSettings.visionStatus = "Projector not loaded"; + llmSettings.mmproj = null; + if (!llmProc) { + await waitForPortAvailable(PORT_LLM, 1500); + return; + } + + const proc = llmProc; + const procPort = PORT_LLM; + llmProc = null; + + try { proc.kill("SIGTERM"); } catch (_) {} + const exitedAfterTerm = await waitForChildExit(proc, 2500); + if (!exitedAfterTerm) { + try { proc.kill("SIGKILL"); } catch (_) {} + await waitForChildExit(proc, 2500); + } + + const released = await waitForPortAvailable(procPort, 5000); + if (!released) { + console.warn(` [llm] Port ${procPort} is still busy after stopping llama.cpp.`); + } +} + function getSetupPaths() { const appDir = path.join(ROOT, "app"); if (osPlatform === "win32") { @@ -2055,6 +2117,28 @@ function selectGgufFile(siblings = [], tier = "low") { return files[0] || ""; } +function selectMmprojFile(siblings = [], modelFilename = "") { + const files = siblings + .map((item) => typeof item === "string" ? item : item?.rfilename) + .filter((name) => name && /\.gguf$/i.test(name)) + .filter((name) => /(?:^|[-_/])mmproj(?:[-_.\/]|$)|mmproj/i.test(name)); + if (files.length === 0) return ""; + + const modelBase = path.basename(modelFilename).toLowerCase(); + const modelStem = modelBase.replace(/\.gguf$/i, "").replace(/\.(q\d(?:_k_[ms])?|iq\d_[a-z0-9]+|bf16|f16|f32)$/i, ""); + const scored = files.map((name) => { + const lower = name.toLowerCase(); + let score = 0; + if (/bf16/i.test(lower)) score += 6; + if (/f16/i.test(lower)) score += 5; + if (/mmproj-model-f16/i.test(lower)) score += 4; + if (modelStem && lower.includes(modelStem)) score += 10; + return { name, score }; + }); + scored.sort((a, b) => b.score - a.score || a.name.length - b.name.length); + return scored[0]?.name || ""; +} + function classifyHuggingFaceModel(model, filename) { const searchable = `${model.id || ""} ${filename || ""} ${(model.tags || []).join(" ")}`.toLowerCase(); const parameters = getModelParameterBillions(searchable); @@ -2179,6 +2263,7 @@ async function searchHuggingFaceModels(query, filters) { if (model.private || model.gated || model.disabled) continue; const filename = selectGgufFile(model.siblings, tier); if (!filename) continue; + const projectorFilename = selectMmprojFile(model.siblings, filename); const traits = classifyHuggingFaceModel(model, filename); if (filters.includes("vision") && !traits.vision) continue; if (filters.includes("uncensored") && !traits.uncensored) continue; @@ -2206,6 +2291,8 @@ async function searchHuggingFaceModels(query, filters) { resolution: "N/A", notes: `Community GGUF from ${String(model.id || "").split("/")[0]}. ${Number(model.downloads || 0).toLocaleString()} Hugging Face downloads.`, url: `https://huggingface.co/${model.id}/resolve/main/${filename.split("/").map(encodeURIComponent).join("/")}`, + projectorUrl: projectorFilename ? `https://huggingface.co/${model.id}/resolve/main/${projectorFilename.split("/").map(encodeURIComponent).join("/")}` : "", + projectorFilename: projectorFilename ? `${model.id.replace(/\//g, "--")}--${path.basename(projectorFilename)}` : "", pageUrl: `https://huggingface.co/${model.id}`, downloads: Number(model.downloads || 0), likes: Number(model.likes || 0), @@ -2478,6 +2565,29 @@ function chooseAutoContext(modelFilename, isGpu) { return 2048; } +function buildLlmLoadProfiles(settings = {}, backend = {}) { + const requestedContext = Number(settings.contextSize); + const safeContext = requestedContext && requestedContext > 0 + ? Math.max(512, Math.min(4096, requestedContext)) + : 4096; + const isCpu = backend.key === "cpu" || backend.mode === "CPU"; + const profiles = [ + { name: "requested", gpuLayers: isCpu ? 0 : undefined }, + { + name: "safe", + contextSize: safeContext, + gpuLayers: isCpu ? 0 : 0, + flashAttn: false, + cacheTypeK: "q4_0", + cacheTypeV: "q4_0", + batchSize: 256, + ubatchSize: 256, + enableThinking: false, + }, + ]; + return profiles; +} + async function startLlm(settings = {}) { const filename = path.basename(String(settings.model || "")); const modelPath = path.join(LLM_MODELS, filename); @@ -2488,7 +2598,47 @@ async function startLlm(settings = {}) { throw new Error("Text generation requires a .gguf model."); } - const backend = getLlmBackend(); + const candidates = getLlmBackendCandidates(); + if (candidates.length === 0) { + throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend."); + } + + const failures = []; + for (const backend of candidates) { + for (const profile of buildLlmLoadProfiles(settings, backend)) { + try { + await startLlmWithBackend({ ...settings, __loadProfile: profile }, backend); + llmSettings.backendFallbacks = failures; + return; + } catch (err) { + const message = err.message || String(err); + failures.push({ + backend: backend.mode, + binary: path.basename(backend.path), + profile: profile.name, + error: message.slice(-500), + }); + console.warn(` [llm] Load failed on ${backend.mode} (${profile.name}): ${message}`); + await killLlm(); + } + } + } + + const last = failures[failures.length - 1]; + llmSettings.backendFallbacks = failures; + throw new Error(`Text model failed on all available llama.cpp backends. Last failure: ${last?.error || "unknown error"}`); +} + +async function startLlmWithBackend(settings = {}, backend) { + const filename = path.basename(String(settings.model || "")); + const modelPath = path.join(LLM_MODELS, filename); + if (!filename || !pathInside(modelPath, LLM_MODELS) || !fs.existsSync(modelPath)) { + throw new Error("Select a downloaded GGUF text model first."); + } + if (!filename.toLowerCase().endsWith(".gguf")) { + throw new Error("Text generation requires a .gguf model."); + } + if (!backend) { throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend."); } @@ -2499,7 +2649,8 @@ async function startLlm(settings = {}) { PORT_LLM = await findAvailableLlmPort(); llmError = null; - let contextSize = Number(settings.contextSize); + const loadProfile = settings.__loadProfile || {}; + let contextSize = Number(loadProfile.contextSize ?? settings.contextSize); if (!contextSize || contextSize <= 0) { const isGpu = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.startsWith("Auto"); contextSize = chooseAutoContext(filename, isGpu); @@ -2509,14 +2660,14 @@ async function startLlm(settings = {}) { } const isSyclBackend = backend.mode.includes("SYCL") || path.basename(path.dirname(backend.path)).toLowerCase() === "sycl"; - const requestedGpuLayers = Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1; + const requestedGpuLayers = Number.isFinite(Number(loadProfile.gpuLayers ?? settings.gpuLayers)) ? Number(loadProfile.gpuLayers ?? settings.gpuLayers) : -1; const effectiveGpuLayers = isSyclBackend ? getStableSyclGpuLayers(filename, requestedGpuLayers) : requestedGpuLayers; - const effectiveFlashAttn = isSyclBackend ? false : settings.flashAttn !== false; - const effectiveCacheTypeK = isSyclBackend ? "q4_0" : settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0"; - const effectiveCacheTypeV = isSyclBackend ? "q4_0" : settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0"; - const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(settings.batchSize) || llmSettings.batchSize || 512) : Number(settings.batchSize) || llmSettings.batchSize || 512; - const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(settings.ubatchSize) || llmSettings.ubatchSize || 512; - const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : settings.enableThinking !== false; + const effectiveFlashAttn = isSyclBackend ? false : (loadProfile.flashAttn ?? (settings.flashAttn !== false)); + const effectiveCacheTypeK = isSyclBackend ? "q4_0" : loadProfile.cacheTypeK || settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0"; + const effectiveCacheTypeV = isSyclBackend ? "q4_0" : loadProfile.cacheTypeV || settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0"; + const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512) : Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512; + const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512; + const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : (loadProfile.enableThinking ?? (settings.enableThinking !== false)); llmSettings = { ...llmSettings, @@ -2527,6 +2678,7 @@ async function startLlm(settings = {}) { requestedGpuLayers, backendMode: backend.mode, backendBinary: path.basename(backend.path), + loadProfile: loadProfile.name || "requested", enableThinking: effectiveEnableThinking, supportsThinking: /deepseek|qwen3|gemma-4|gemma4|think|r1|e2b|reasoning/i.test(filename), // New performance settings @@ -2692,9 +2844,15 @@ async function startLlm(settings = {}) { } console.log(" [llm] Starting:", backend.path, args.join(" ")); - llmProc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv }); - llmProc.stdout.on("data", (data) => process.stdout.write(" [llm] " + data.toString())); - llmProc.stderr.on("data", (data) => { + const proc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv }); + const procSeq = ++llmProcSeq; + llmProc = proc; + proc.stdout.on("data", (data) => { + if (llmProc !== proc) return; + process.stdout.write(" [llm] " + data.toString()); + }); + proc.stderr.on("data", (data) => { + if (llmProc !== proc) return; const output = data.toString(); process.stderr.write(" [llm-err] " + output); if (/Vulkan\d+\s*:/i.test(output)) llmSettings.backendMode = "Vulkan GPU"; @@ -2706,14 +2864,18 @@ async function startLlm(settings = {}) { llmError = output.trim().slice(-1200); } }); - llmProc.on("exit", (code) => { + proc.on("exit", (code) => { + if (llmProc !== proc) { + console.log(" [llm] stale process exited with code", code); + return; + } llmReady = false; llmProc = null; if (code !== 0 && code !== null && !llmError) llmError = `llama.cpp exited with code ${code}`; - console.log(" [llm] exited with code", code); + console.log(" [llm] exited with code", code, `(process ${procSeq})`); }); - await waitForLlmReady(); + await waitForLlmReady(backend.key === "cpu" ? 720 : 360, proc, filename); } async function startBackend(settings = {}) { @@ -3155,7 +3317,11 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki kind, }; - console.log(` [download] Starting download of ${filename} from ${url}`); + if (redirectCount > 0) { + console.log(` [download] Continuing redirected download of ${filename}`); + } else { + console.log(` [download] Starting download of ${filename} from ${url}`); + } let downloadFinalized = false; const failDownload = (message, err = null) => { @@ -3198,16 +3364,17 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki failDownload("Too many redirects"); return; } - console.log(` [download] Redirected to ${redirectUrl}`); + console.log(` [download] Following redirect for ${filename}`); + downloadState.speed = "Following redirect"; // Clean up redirected request to avoid triggering error handlers later request.removeAllListeners("error"); request.destroy(); + response.resume(); fileStream.close(); try { fs.unlinkSync(tempPath); } catch (_) {} activeDownload = null; - downloadState.active = false; startModelDownload(redirectUrl, filename, targetDir, kind, redirectCount + 1); return; } @@ -3818,17 +3985,17 @@ const server = http.createServer(async (req, res) => { const body = await readJsonBody(req, res); if (!body) return; try { - await startLlm(body); + await runExclusiveLlmOperation(() => startLlm(body)); return json(res, 200, { ok: true, ready: llmReady, port: PORT_LLM, settings: llmSettings }); } catch (err) { llmError = err.message || String(err); - await killLlm(); + await runExclusiveLlmOperation(() => killLlm()); return json(res, 500, { ok: false, error: llmError }); } } if (req.url === "/api/llm/stop" && req.method === "POST") { - await killLlm(); + await runExclusiveLlmOperation(() => killLlm()); return json(res, 200, { ok: true }); } @@ -3862,7 +4029,7 @@ async function retryLowerContext() { enableThinking: llmSettings.enableThinking, }; - await startLlm(newSettings); + await runExclusiveLlmOperation(() => startLlm(newSettings)); } async function doLlmChat(req, res, body, retryCount = 0) { diff --git a/windows.bat b/windows.bat index 94e435f..04a105d 100644 --- a/windows.bat +++ b/windows.bat @@ -14,6 +14,7 @@ set LLM_VULKAN_BACKEND=%APP%\llm-backend\win\vulkan\llama-server.exe set LLM_CPU_BACKEND=%APP%\llm-backend\win\cpu\llama-server.exe set SERVE=%~dp0scripts\serve.cjs if "%FRONTEND_PORT%"=="" set FRONTEND_PORT=1420 +if "%LLM_PORT%"=="" set LLM_PORT=10086 set SETUP_REASON= set SETUP_MODE=Repair @@ -60,6 +61,7 @@ pause >nul :: Clear old frontend and backend server processes before setup so app/tools/node-win can be replaced for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul +for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%LLM_PORT% "') do taskkill /f /pid %%a >nul 2>nul powershell -ExecutionPolicy Bypass -File "%SETUP%" if errorlevel 1 ( @@ -81,9 +83,10 @@ echo ============================================================ echo. :: Clear frontend and backend ports to prevent address conflicts. -echo Clearing frontend port %FRONTEND_PORT% and backend port 8080... +echo Clearing frontend port %FRONTEND_PORT%, backend port 8080, and text port %LLM_PORT%... for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul +for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%LLM_PORT% "') do taskkill /f /pid %%a >nul 2>nul :: Start frontend server + backend manager (serve.cjs manages sd-vulkan.exe) echo Starting Local AI Image Generator... @@ -95,7 +98,7 @@ echo ============================================================ echo Running! echo Web UI: http://localhost:%FRONTEND_PORT% echo GPU API: Auto-selected by the app (starts at 8080) -echo Text API: Starts when a GGUF model is loaded (port 10086) +echo Text API: Starts when a GGUF model is loaded (port %LLM_PORT%) echo. echo Press Ctrl+C in this window to stop all services. echo ============================================================