diff --git a/app/frontend/src/App.css b/app/frontend/src/App.css index c9335ac..f40c87d 100644 --- a/app/frontend/src/App.css +++ b/app/frontend/src/App.css @@ -784,6 +784,18 @@ body { } /* ─── Generation Stats ────────────────────────────────────────── */ +.chat-generation-warning { + margin-top: 4px; + padding: 7px 10px; + border-radius: var(--md-shape-corner-small); + border: 1px solid color-mix(in srgb, var(--md-sys-color-warning, #f59e0b) 35%, transparent); + background: color-mix(in srgb, var(--md-sys-color-warning, #f59e0b) 10%, transparent); + color: var(--md-sys-color-on-surface-variant); + font-size: 0.76rem; + line-height: 1.35; + max-width: min(720px, 84vw); +} + .chat-generation-stats { display: inline-flex; align-items: center; diff --git a/app/frontend/src/App.jsx b/app/frontend/src/App.jsx index 6dabb4d..139c5aa 100644 --- a/app/frontend/src/App.jsx +++ b/app/frontend/src/App.jsx @@ -120,8 +120,9 @@ function App() { try { const parsed = JSON.parse(saved); return { - enableThinking: true, - ...parsed + ...parsed, + enableThinking: parsed.enableThinking === true, + maxTokens: parsed.maxTokens || 384, }; } catch (_) {} } @@ -130,10 +131,10 @@ function App() { temperature: 0.7, systemPrompt: "You are a helpful local AI assistant.", threads: Math.max(4, Math.min(16, (navigator.hardwareConcurrency || 4) - 2)), - enableThinking: true, + enableThinking: false, // New performance settings gpuLayers: -1, - maxTokens: 1024, + maxTokens: 384, topP: 0.95, topK: 40, minP: 0.05, @@ -147,6 +148,8 @@ function App() { mmap: true, cachePrompt: true, defragThold: 0.1, + batchSize: 512, + ubatchSize: 512, }; }); diff --git a/app/frontend/src/components/Settings.jsx b/app/frontend/src/components/Settings.jsx index ae2b0ce..7e072df 100644 --- a/app/frontend/src/components/Settings.jsx +++ b/app/frontend/src/components/Settings.jsx @@ -5,7 +5,7 @@ import { Monitor, HardDrive, MemoryStick, Thermometer, Hash, Layers, ChevronRight, Box, Wand2, Lightbulb, RotateCcw, Check, Palette } from "lucide-react"; -import { stopServer, formatBytes, getLlmStatus } from "../services/api"; +import { stopServer, formatBytes, getLlmStatus, startLlm, stopLlm } from "../services/api"; import { THEMES } from "../themes"; const ASPECT_RATIOS = [ @@ -230,6 +230,56 @@ function Settings({ })); }; + const buildTextStartOptions = (settings) => ({ + threads: settings?.threads || specs?.cpu_cores_physical || 4, + contextSize: settings?.contextSize || 4096, + gpuLayers: settings?.gpuLayers ?? -1, + enableThinking: settings?.enableThinking !== false, + flashAttn: settings?.flashAttn, + cacheTypeK: settings?.cacheTypeK, + cacheTypeV: settings?.cacheTypeV, + mlock: settings?.mlock, + mmap: settings?.mmap, + cachePrompt: settings?.cachePrompt, + defragThold: settings?.defragThold, + batchSize: settings?.batchSize, + ubatchSize: settings?.ubatchSize, + performanceProfile: settings?.performanceProfile, + }); + + const handleThinkingToggle = async (enabled) => { + const nextSettings = { ...textSettings, enableThinking: enabled }; + + let status = null; + try { + status = await getLlmStatus(); + } catch (_) {} + if (!status?.ready || !status?.settings?.model) { + setTextSettings(nextSettings); + return; + } + + const reload = await showConfirm({ + title: enabled ? "Reload With DeepThink?" : "Reload Without DeepThink?", + message: "Changing DeepThink requires reloading the text model before it affects new replies. Reload now, or skip and keep the currently loaded model as-is?", + confirmLabel: "Reload", + cancelLabel: "Skip", + }); + if (!reload) return; + + try { + setTextSettings(nextSettings); + await stopLlm(); + await startLlm(status.settings.model, buildTextStartOptions(nextSettings)); + } catch (err) { + await showAlert({ + title: "Reload Failed", + message: err.message || String(err), + danger: true, + }); + } + }; + const handleAspectRatioChange = (ratio, sizeType) => { if (isOpenVinoNpu && ratio !== "1:1") return; const isSDXL = sizeType === "sdxl" && !isSD15OrCustom; @@ -632,6 +682,22 @@ function Settings({ /> +
+
+ Max Response Tokens + {textSettings.maxTokens || 384} +
+ updateTextSetting("maxTokens", parseInt(e.target.value))} + min="64" + max="2048" + step="64" + /> +
+
Min P @@ -787,7 +853,7 @@ function Settings({
updateTextSetting("enableThinking", v)} + onChange={handleThinkingToggle} label="DeepThink" description={supportsThinking ? "Show model's reasoning process" diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx index b6b0e94..5bc4277 100644 --- a/app/frontend/src/components/TextChat.jsx +++ b/app/frontend/src/components/TextChat.jsx @@ -143,6 +143,13 @@ function TextChat({ completion_tokens: 0, total_tokens: 0 }); + + const estimateTokens = (text) => { + const value = String(text || "").trim(); + if (!value) return 0; + const wordCount = value.split(/\s+/).filter(Boolean).length; + return Math.max(wordCount, Math.ceil(value.length / 4)); + }; const bottomRef = useRef(null); const completedDownloadRef = useRef(""); @@ -254,6 +261,53 @@ function TextChat({ prevContextSizeRef.current = currentContextSize; }, [textSettings?.gpuLayers, textSettings?.threads, textSettings?.contextSize]); + const buildTextStartOptions = (settings) => ({ + threads: settings?.threads || specs?.cpu_cores_physical || 4, + contextSize: settings?.contextSize || 4096, + gpuLayers: settings?.gpuLayers ?? -1, + enableThinking: settings?.enableThinking !== false, + flashAttn: settings?.flashAttn, + cacheTypeK: settings?.cacheTypeK, + cacheTypeV: settings?.cacheTypeV, + mlock: settings?.mlock, + mmap: settings?.mmap, + cachePrompt: settings?.cachePrompt, + defragThold: settings?.defragThold, + batchSize: settings?.batchSize, + ubatchSize: settings?.ubatchSize, + performanceProfile: settings?.performanceProfile, + }); + + const handleThinkingToggle = async () => { + const enabled = textSettings.enableThinking === false; + const nextSettings = { ...textSettings, enableThinking: enabled }; + + if (!status.ready || !status.settings?.model) { + setTextSettings(nextSettings); + return; + } + + const reload = await showConfirm({ + title: enabled ? "Reload With DeepThink?" : "Reload Without DeepThink?", + message: "Changing DeepThink requires reloading the text model before it affects new replies. Reload now, or skip and keep the currently loaded model as-is?", + confirmLabel: "Reload", + cancelLabel: "Skip", + }); + if (!reload) return; + + setIsBusy(true); + try { + setTextSettings(nextSettings); + await stopLlm(); + const result = await startLlm(status.settings.model, buildTextStartOptions(nextSettings)); + setStatus({ ...status, ...result, ready: true, running: true, settings: result.settings }); + } catch (err) { + showAlert({ title: "Text Model Reload Failed", message: err.message || String(err), danger: true }); + } finally { + setIsBusy(false); + } + }; + useEffect(() => { if (!supportsVision) { setAttachments((current) => current.filter((attachment) => attachment.type !== "image")); @@ -274,7 +328,7 @@ function TextChat({ const text = Array.isArray(m.content) ? m.content.map(c => c.text || "").join(" ") : (m.content || ""); - return sum + text.split(/\s+/).length; + return sum + estimateTokens(text) + estimateTokens(m.reasoning || ""); }, 0); setTokenUsage({ prompt_tokens: Math.round(total * 0.7), @@ -386,6 +440,16 @@ function TextChat({ contextSize: textSettings?.contextSize || 4096, gpuLayers: textSettings?.gpuLayers ?? -1, enableThinking: textSettings?.enableThinking !== false, + flashAttn: textSettings?.flashAttn, + cacheTypeK: textSettings?.cacheTypeK, + cacheTypeV: textSettings?.cacheTypeV, + mlock: textSettings?.mlock, + mmap: textSettings?.mmap, + cachePrompt: textSettings?.cachePrompt, + defragThold: textSettings?.defragThold, + batchSize: textSettings?.batchSize, + ubatchSize: textSettings?.ubatchSize, + performanceProfile: textSettings?.performanceProfile, }); setStatus({ ...status, ...result, ready: true, running: true, settings: result.settings }); setMessages([]); @@ -478,6 +542,17 @@ function TextChat({ ...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []), ...nextMessages, ]; + const promptTokenEstimate = requestMessages.reduce((sum, message) => { + const messageText = Array.isArray(message.content) + ? message.content.map((item) => item.text || "").join(" ") + : (message.content || ""); + return sum + estimateTokens(messageText); + }, 0); + setTokenUsage({ + prompt_tokens: promptTokenEstimate, + completion_tokens: 0, + total_tokens: promptTokenEstimate, + }); let assistantText = ""; let assistantReasoning = ""; @@ -487,14 +562,20 @@ function TextChat({ let thinkingEndedAt = null; let thinkingDuration = 0; + const baseMaxTokens = textSettings?.maxTokens || 384; + const effectiveMaxTokens = textSettings?.enableThinking !== false + ? Math.max(baseMaxTokens, 1024) + : baseMaxTokens; + const response = await streamChatWithLlm(requestMessages, { temperature: textSettings?.temperature || 0.7, - maxTokens: textSettings?.maxTokens || 1024, + maxTokens: effectiveMaxTokens, topP: textSettings?.topP, topK: textSettings?.topK, minP: textSettings?.minP, repeatPenalty: textSettings?.repeatPenalty, seed: textSettings?.seed, + enableThinking: textSettings?.enableThinking !== false, signal: controller.signal, }, (_token, fullText, _reasoningToken, fullReasoning) => { const now = performance.now(); @@ -530,7 +611,6 @@ function TextChat({ tokensPerSecond: streamedTokens / generationSeconds, seconds: (now - requestStartedAt) / 1000, }; - // rAF batching: accumulate updates and flush once per frame (~16ms) // This reduces React re-renders from N per token to ~60/sec tokenBufferRef.current = { @@ -583,6 +663,8 @@ function TextChat({ tokens: exactTokens, tokensPerSecond: exactTokensPerSecond, seconds: exactSeconds, + finishReason: response.finishReason || null, + truncated: response.finishReason === "length", }; const processed = processMessageContent(assistantText, response.reasoningContent || assistantReasoning, textSettings?.enableThinking !== false); @@ -595,7 +677,23 @@ function TextChat({ }]; setMessages(finalMessages); saveConversationState(convId, finalMessages, selectedModel); - if (response.usage) setTokenUsage(response.usage); + const finalCompletionEstimate = Math.max( + exactTokens, + estimateTokens(processed.content) + estimateTokens(processed.reasoning) + ); + const finalUsageEstimate = { + prompt_tokens: promptTokenEstimate, + completion_tokens: finalCompletionEstimate, + total_tokens: promptTokenEstimate + finalCompletionEstimate, + }; + setTokenUsage(response.usage + ? { + prompt_tokens: Math.max(Number(response.usage.prompt_tokens) || 0, finalUsageEstimate.prompt_tokens), + completion_tokens: Math.max(Number(response.usage.completion_tokens) || 0, finalUsageEstimate.completion_tokens), + total_tokens: Math.max(Number(response.usage.total_tokens) || 0, finalUsageEstimate.total_tokens), + } + : finalUsageEstimate + ); // Clean up attached files on success setAttachments([]); @@ -845,7 +943,13 @@ function TextChat({ {/* Generation stats pill */} {message.role === "assistant" && message.generationStats && !message.error && ( -
+ <> + {message.generationStats.truncated && ( +
+ Response reached the token limit. Increase Max Response Tokens or ask "continue". +
+ )} +
{message.generationStats.status === "starting" ? ( <> Waiting for first token... ) : message.generationStats.status === "streaming" ? ( @@ -853,7 +957,8 @@ function TextChat({ ) : ( <>{message.generationStats.tokens} tokens • {formatGenerationTime(message.generationStats.seconds)} )} -
+
+ )}
@@ -932,13 +1037,7 @@ function TextChat({ {status.ready && supportsThinking && (
diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js index e5115dc..6f56907 100644 --- a/app/frontend/src/services/api.js +++ b/app/frontend/src/services/api.js @@ -347,6 +347,16 @@ export async function startLlm(model, options = {}) { contextSize: options.contextSize, gpuLayers: options.gpuLayers, enableThinking: options.enableThinking, + flashAttn: options.flashAttn, + cacheTypeK: options.cacheTypeK, + cacheTypeV: options.cacheTypeV, + mlock: options.mlock, + mmap: options.mmap, + cachePrompt: options.cachePrompt, + defragThold: options.defragThold, + batchSize: options.batchSize, + ubatchSize: options.ubatchSize, + performanceProfile: options.performanceProfile, }), }); return await readJsonResponse(res, "The local server returned an invalid text backend response."); @@ -423,6 +433,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => let reasoningContent = ""; let usage = null; let timings = null; + let finishReason = null; const consumeEvent = (eventText) => { try { @@ -435,8 +446,10 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => if (!data || data === "[DONE]") return data === "[DONE]"; const parsed = JSON.parse(data); - const token = parsed.choices?.[0]?.delta?.content || ""; - const reasoningToken = parsed.choices?.[0]?.delta?.reasoning_content || ""; + const choice = parsed.choices?.[0]; + const token = choice?.delta?.content || ""; + const reasoningToken = choice?.delta?.reasoning_content || ""; + if (choice?.finish_reason) finishReason = choice.finish_reason; if (token || reasoningToken) { content += token; @@ -470,7 +483,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => if (!finished && buffer.trim()) consumeEvent(buffer); if (!content && !reasoningContent) throw new Error("The text model returned an empty streamed response."); - return { content, reasoningContent, usage, timings }; + return { content, reasoningContent, usage, timings, finishReason }; } export async function downloadLlmModel(url, filename = null) { diff --git a/scripts/serve.cjs b/scripts/serve.cjs index 0c9140d..c934c32 100644 --- a/scripts/serve.cjs +++ b/scripts/serve.cjs @@ -9,7 +9,7 @@ const fs = require("fs"); const net = require("net"); const os = require("os"); const path = require("path"); -const { spawn, spawnSync, execSync, exec } = require("child_process"); +const { spawn, spawnSync, execSync, exec, execFile } = require("child_process"); // HTTP keep-alive agent for llama-server (eliminates TCP handshake per request) const llmHttpAgent = new http.Agent({ @@ -697,6 +697,32 @@ function chooseAutoLayers(modelFilename, freeVramBytes, cacheTypeK = "q8_0", cac return Math.max(0, maxLayers); } +function getGgufModelSizeGb(modelFilename) { + try { + const modelPath = path.join(LLM_MODELS, modelFilename); + if (fs.existsSync(modelPath)) { + return fs.statSync(modelPath).size / (1024 * 1024 * 1024); + } + } catch (_) {} + return 4; +} + +function getStableSyclGpuLayers(modelFilename, requestedLayers) { + if (process.env.LOCALAI_LLM_ALLOW_FULL_SYCL === "1") return requestedLayers; + if (requestedLayers !== -1) return requestedLayers; + + const syclGpu = detectLlamaSyclGpu(); + const vramGb = syclGpu?.free_vram_gb || syclGpu?.vram_gb || 0; + const modelSizeGb = getGgufModelSizeGb(modelFilename); + if (vramGb > 0 && vramGb <= 8.5 && modelSizeGb >= 2.5) { + return 0; + } + if (vramGb > 0 && vramGb <= 12 && modelSizeGb >= 5) { + return 0; + } + return requestedLayers; +} + let nvidiaSmiCmd = "nvidia-smi"; let hasNvidiaSmi = null; @@ -768,6 +794,77 @@ function getNvidiaVram() { return cachedVramInfo; } +let cachedLlamaVramInfo = null; +let isPollingLlamaVram = false; +let lastLlamaVramPollTime = 0; + +function parseLlamaDeviceMemory(output) { + const devices = []; + const devicePattern = /(CUDA|Vulkan|SYCL)\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; + let match; + while ((match = devicePattern.exec(output || "")) !== null) { + const totalMiB = Number(match[3]); + const freeMiB = match[4] ? Number(match[4]) : NaN; + if (!Number.isFinite(totalMiB) || totalMiB <= 0) continue; + + const usedMiB = Number.isFinite(freeMiB) ? Math.max(0, totalMiB - freeMiB) : 0; + devices.push({ + gpu_name: match[2].trim(), + vram_used_gb: Math.round((usedMiB / 1024) * 100) / 100, + vram_total_gb: Math.round((totalMiB / 1024) * 100) / 100, + }); + } + return devices.sort((a, b) => b.vram_total_gb - a.vram_total_gb)[0] || null; +} + +function getLlamaTelemetryBackendPath() { + if (osPlatform === "win32") { + if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) return LLM_BACKEND_PATHS.winVulkan; + if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) return LLM_BACKEND_PATHS.winSycl; + if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) return LLM_BACKEND_PATHS.winCuda; + } else if (osPlatform === "linux") { + if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) return LLM_BACKEND_PATHS.linuxVulkan; + if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl)) return LLM_BACKEND_PATHS.linuxSycl; + if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) return LLM_BACKEND_PATHS.linuxCuda; + } + return null; +} + +function pollLlamaVram(force = false) { + if (isPollingLlamaVram) return; + + const now = Date.now(); + if (!force && now - lastLlamaVramPollTime < 4500) { + return; + } + + const backendPath = getLlamaTelemetryBackendPath(); + if (!backendPath) return; + + isPollingLlamaVram = true; + lastLlamaVramPollTime = now; + execFile( + backendPath, + ["--list-devices"], + { windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024 }, + (error, stdout, stderr) => { + isPollingLlamaVram = false; + if (error) { + cachedLlamaVramInfo = null; + return; + } + cachedLlamaVramInfo = parseLlamaDeviceMemory(`${stdout || ""}\n${stderr || ""}`); + } + ); +} + +setInterval(() => pollLlamaVram(false), 5000); +pollLlamaVram(true); + +function getLlamaVram() { + return cachedLlamaVramInfo; +} + let cachedMacRamUsedGb = null; function pollMacRam() { if (osPlatform !== "darwin") return; @@ -963,6 +1060,7 @@ function getHardwareSpecs() { function getTelemetry() { const vram = getNvidiaVram(); + const llamaVram = vram || getLlamaVram(); let ram_used_gb = roundGb(os.totalmem() - os.freemem()); if (osPlatform === "darwin" && cachedMacRamUsedGb !== null) { ram_used_gb = cachedMacRamUsedGb; @@ -972,9 +1070,9 @@ function getTelemetry() { cpu_usage: getCpuUsagePercent(), ram_used_gb, ram_total_gb: roundGb(os.totalmem()), - gpu_name: vram?.gpu_name || gpu.name, - vram_used_gb: vram?.vram_used_gb || 0, - vram_total_gb: vram?.vram_total_gb || gpu.vram_gb || 0, + gpu_name: llamaVram?.gpu_name || gpu.name, + vram_used_gb: Number.isFinite(Number(llamaVram?.vram_used_gb)) ? llamaVram.vram_used_gb : 0, + vram_total_gb: llamaVram?.vram_total_gb || gpu.vram_gb || 0, }; } @@ -1189,21 +1287,21 @@ function getLlmBackend() { if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) { return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" }; } - if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) { - return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" }; - } if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) { return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" }; } + if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) { + return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" }; + } if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) { return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" }; } - if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) { - return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" }; - } if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) { return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" }; } + if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) { + return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" }; + } return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" }; } else if (osPlatform === "darwin") { return { @@ -2405,26 +2503,37 @@ async function startLlm(settings = {}) { contextSize = Math.max(512, Math.min(32768, contextSize)); } + const isSyclBackend = backend.mode.includes("SYCL") || path.basename(path.dirname(backend.path)).toLowerCase() === "sycl"; + const requestedGpuLayers = Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1; + const effectiveGpuLayers = isSyclBackend ? getStableSyclGpuLayers(filename, requestedGpuLayers) : requestedGpuLayers; + const effectiveFlashAttn = isSyclBackend ? false : settings.flashAttn !== false; + const effectiveCacheTypeK = isSyclBackend ? "q4_0" : settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0"; + const effectiveCacheTypeV = isSyclBackend ? "q4_0" : settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0"; + const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(settings.batchSize) || llmSettings.batchSize || 512) : Number(settings.batchSize) || llmSettings.batchSize || 512; + const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(settings.ubatchSize) || llmSettings.ubatchSize || 512; + const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : settings.enableThinking !== false; + llmSettings = { ...llmSettings, model: filename, threads: Math.max(1, Math.min(64, Number(settings.threads) || llmSettings.threads)), contextSize: contextSize, - gpuLayers: Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1, + gpuLayers: effectiveGpuLayers, + requestedGpuLayers, backendMode: backend.mode, backendBinary: path.basename(backend.path), - enableThinking: settings.enableThinking !== false, + enableThinking: effectiveEnableThinking, supportsThinking: /deepseek|qwen3|gemma-4|gemma4|think|r1|e2b|reasoning/i.test(filename), // New performance settings - flashAttn: settings.flashAttn !== false, - cacheTypeK: settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0", - cacheTypeV: settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0", + flashAttn: effectiveFlashAttn, + cacheTypeK: effectiveCacheTypeK, + cacheTypeV: effectiveCacheTypeV, mlock: settings.mlock || llmSettings.mlock || false, mmap: settings.mmap !== false && llmSettings.mmap !== false, cachePrompt: settings.cachePrompt !== false && llmSettings.cachePrompt !== false, defragThold: Number(settings.defragThold) || llmSettings.defragThold || 0.1, - batchSize: Math.max(128, Math.min(2048, Number(settings.batchSize) || llmSettings.batchSize || 512)), - ubatchSize: Math.max(128, Math.min(2048, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512)), + batchSize: Math.max(128, Math.min(2048, effectiveBatchSize)), + ubatchSize: Math.max(128, Math.min(2048, effectiveUbatchSize)), performanceProfile: settings.performanceProfile || llmSettings.performanceProfile || "balanced", }; @@ -2500,21 +2609,26 @@ async function startLlm(settings = {}) { "--model", modelPath, "--host", "127.0.0.1", "--port", String(PORT_LLM), + "-lv", "1", "--ctx-size", String(llmSettings.contextSize), "--threads", String(llmSettings.threads), "--n-gpu-layers", String(llmSettings.gpuLayers), "--parallel", "1", - "--flash-attn", "on", // ✅ Memory-efficient attention (all backends) "--cache-type-k", String(llmSettings.cacheTypeK), // ✅ KV cache quantization K "--cache-type-v", String(llmSettings.cacheTypeV), // ✅ KV cache quantization V "--cache-prompt", // ✅ Prompt caching for multi-turn (major UX win) "--ctx-checkpoints", "8", // ✅ Context shifting recovery "--poll", "30", // ✅ More aggressive polling "--metrics", // ✅ Metrics endpoint for tuning - "--defrag-thold", String(llmSettings.defragThold), // ✅ KV cache defragmentation "--batch-size", String(llmSettings.batchSize), // ✅ Prompt processing batch "--ubatch-size", String(llmSettings.ubatchSize), // ✅ Micro-batch size ]; + if (llmSettings.flashAttn) { + args.push("--flash-attn", "on"); + } + if (isSyclBackend) { + args.push("--no-warmup"); + } // CPU mode: lock memory to prevent OS paging (critical for consistent speed) const isGpuMode = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || @@ -2565,7 +2679,6 @@ async function startLlm(settings = {}) { // SYCL-specific optimizations (Intel Arc/Graphics) if (backend.mode.includes("SYCL")) { - spawnEnv.GGML_SYCL_ENABLE_FLASH_ATTN = "1"; spawnEnv.ZES_ENABLE_SYSMAN = "1"; }