truetype/transcriber
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add hardware telemetry dashboard and unified API client for system monitoring

Browse Source
This commit is contained in:
techjarves committed 2026-06-21 08:47:24 +05:30
1 parent f6d441e5d7
commit 9e67084e50
7 files changed
+353 -45
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/frontend/src/App.css
+12
View File
@@ -784,6 +784,18 @@ body {
}
/* ─── Generation Stats ────────────────────────────────────────── */
.chat-generation-warning {
margin-top: 4px;
padding: 7px 10px;
border-radius: var(--md-shape-corner-small);
border: 1px solid color-mix(in srgb, var(--md-sys-color-warning, #f59e0b) 35%, transparent);
background: color-mix(in srgb, var(--md-sys-color-warning, #f59e0b) 10%, transparent);
color: var(--md-sys-color-on-surface-variant);
font-size: 0.76rem;
line-height: 1.35;
max-width: min(720px, 84vw);
}
.chat-generation-stats {
display: inline-flex;
align-items: center;
app/frontend/src/App.jsx
+7 -4
View File
@@ -120,8 +120,9 @@ function App() {
try {
const parsed = JSON.parse(saved);
return {
enableThinking: true,
...parsed
...parsed,
enableThinking: parsed.enableThinking === true,
maxTokens: parsed.maxTokens || 384,
};
} catch (_) {}
}
@@ -130,10 +131,10 @@ function App() {
temperature: 0.7,
systemPrompt: "You are a helpful local AI assistant.",
threads: Math.max(4, Math.min(16, (navigator.hardwareConcurrency || 4) - 2)),
enableThinking: true,
enableThinking: false,
// New performance settings
gpuLayers: -1,
maxTokens: 1024,
maxTokens: 384,
topP: 0.95,
topK: 40,
minP: 0.05,
@@ -147,6 +148,8 @@ function App() {
mmap: true,
cachePrompt: true,
defragThold: 0.1,
batchSize: 512,
ubatchSize: 512,
};
});
app/frontend/src/components/Settings.jsx
+68 -2
View File
@@ -5,7 +5,7 @@ import {
Monitor, HardDrive, MemoryStick, Thermometer, Hash, Layers,
ChevronRight, Box, Wand2, Lightbulb, RotateCcw, Check, Palette
} from "lucide-react";
import { stopServer, formatBytes, getLlmStatus } from "../services/api";
import { stopServer, formatBytes, getLlmStatus, startLlm, stopLlm } from "../services/api";
import { THEMES } from "../themes";
const ASPECT_RATIOS = [
@@ -230,6 +230,56 @@ function Settings({
}));
};
const buildTextStartOptions = (settings) => ({
threads: settings?.threads || specs?.cpu_cores_physical || 4,
contextSize: settings?.contextSize || 4096,
gpuLayers: settings?.gpuLayers ?? -1,
enableThinking: settings?.enableThinking !== false,
flashAttn: settings?.flashAttn,
cacheTypeK: settings?.cacheTypeK,
cacheTypeV: settings?.cacheTypeV,
mlock: settings?.mlock,
mmap: settings?.mmap,
cachePrompt: settings?.cachePrompt,
defragThold: settings?.defragThold,
batchSize: settings?.batchSize,
ubatchSize: settings?.ubatchSize,
performanceProfile: settings?.performanceProfile,
});
const handleThinkingToggle = async (enabled) => {
const nextSettings = { ...textSettings, enableThinking: enabled };
let status = null;
try {
status = await getLlmStatus();
} catch (_) {}
if (!status?.ready || !status?.settings?.model) {
setTextSettings(nextSettings);
return;
}
const reload = await showConfirm({
title: enabled ? "Reload With DeepThink?" : "Reload Without DeepThink?",
message: "Changing DeepThink requires reloading the text model before it affects new replies. Reload now, or skip and keep the currently loaded model as-is?",
confirmLabel: "Reload",
cancelLabel: "Skip",
});
if (!reload) return;
try {
setTextSettings(nextSettings);
await stopLlm();
await startLlm(status.settings.model, buildTextStartOptions(nextSettings));
} catch (err) {
await showAlert({
title: "Reload Failed",
message: err.message || String(err),
danger: true,
});
}
};
const handleAspectRatioChange = (ratio, sizeType) => {
if (isOpenVinoNpu && ratio !== "1:1") return;
const isSDXL = sizeType === "sdxl" && !isSD15OrCustom;
@@ -632,6 +682,22 @@ function Settings({
/>
</div>
<div className="m3-slider-group">
<div className="m3-slider-header">
<span className="m3-slider-label">Max Response Tokens</span>
<span className="settings-value-badge">{textSettings.maxTokens || 384}</span>
</div>
<input
type="range"
className="m3-slider"
value={textSettings.maxTokens || 384}
onChange={(e) => updateTextSetting("maxTokens", parseInt(e.target.value))}
min="64"
max="2048"
step="64"
/>
</div>
<div className="m3-slider-group">
<div className="m3-slider-header">
<span className="m3-slider-label">Min P</span>
@@ -787,7 +853,7 @@ function Settings({
<div className="m3-field-group">
<PremiumToggle
checked={textSettings.enableThinking !== false}
onChange={(v) => updateTextSetting("enableThinking", v)}
onChange={handleThinkingToggle}
label="DeepThink"
description={supportsThinking
? "Show model's reasoning process"
app/frontend/src/components/TextChat.jsx
+112 -13
View File
@@ -143,6 +143,13 @@ function TextChat({
completion_tokens: 0,
total_tokens: 0
});
const estimateTokens = (text) => {
const value = String(text || "").trim();
if (!value) return 0;
const wordCount = value.split(/\s+/).filter(Boolean).length;
return Math.max(wordCount, Math.ceil(value.length / 4));
};
const bottomRef = useRef(null);
const completedDownloadRef = useRef("");
@@ -254,6 +261,53 @@ function TextChat({
prevContextSizeRef.current = currentContextSize;
}, [textSettings?.gpuLayers, textSettings?.threads, textSettings?.contextSize]);
const buildTextStartOptions = (settings) => ({
threads: settings?.threads || specs?.cpu_cores_physical || 4,
contextSize: settings?.contextSize || 4096,
gpuLayers: settings?.gpuLayers ?? -1,
enableThinking: settings?.enableThinking !== false,
flashAttn: settings?.flashAttn,
cacheTypeK: settings?.cacheTypeK,
cacheTypeV: settings?.cacheTypeV,
mlock: settings?.mlock,
mmap: settings?.mmap,
cachePrompt: settings?.cachePrompt,
defragThold: settings?.defragThold,
batchSize: settings?.batchSize,
ubatchSize: settings?.ubatchSize,
performanceProfile: settings?.performanceProfile,
});
const handleThinkingToggle = async () => {
const enabled = textSettings.enableThinking === false;
const nextSettings = { ...textSettings, enableThinking: enabled };
if (!status.ready || !status.settings?.model) {
setTextSettings(nextSettings);
return;
}
const reload = await showConfirm({
title: enabled ? "Reload With DeepThink?" : "Reload Without DeepThink?",
message: "Changing DeepThink requires reloading the text model before it affects new replies. Reload now, or skip and keep the currently loaded model as-is?",
confirmLabel: "Reload",
cancelLabel: "Skip",
});
if (!reload) return;
setIsBusy(true);
try {
setTextSettings(nextSettings);
await stopLlm();
const result = await startLlm(status.settings.model, buildTextStartOptions(nextSettings));
setStatus({ ...status, ...result, ready: true, running: true, settings: result.settings });
} catch (err) {
showAlert({ title: "Text Model Reload Failed", message: err.message || String(err), danger: true });
} finally {
setIsBusy(false);
}
};
useEffect(() => {
if (!supportsVision) {
setAttachments((current) => current.filter((attachment) => attachment.type !== "image"));
@@ -274,7 +328,7 @@ function TextChat({
const text = Array.isArray(m.content)
? m.content.map(c => c.text || "").join(" ")
: (m.content || "");
return sum + text.split(/\s+/).length;
return sum + estimateTokens(text) + estimateTokens(m.reasoning || "");
}, 0);
setTokenUsage({
prompt_tokens: Math.round(total * 0.7),
@@ -386,6 +440,16 @@ function TextChat({
contextSize: textSettings?.contextSize || 4096,
gpuLayers: textSettings?.gpuLayers ?? -1,
enableThinking: textSettings?.enableThinking !== false,
flashAttn: textSettings?.flashAttn,
cacheTypeK: textSettings?.cacheTypeK,
cacheTypeV: textSettings?.cacheTypeV,
mlock: textSettings?.mlock,
mmap: textSettings?.mmap,
cachePrompt: textSettings?.cachePrompt,
defragThold: textSettings?.defragThold,
batchSize: textSettings?.batchSize,
ubatchSize: textSettings?.ubatchSize,
performanceProfile: textSettings?.performanceProfile,
});
setStatus({ ...status, ...result, ready: true, running: true, settings: result.settings });
setMessages([]);
@@ -478,6 +542,17 @@ function TextChat({
...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []),
...nextMessages,
];
const promptTokenEstimate = requestMessages.reduce((sum, message) => {
const messageText = Array.isArray(message.content)
? message.content.map((item) => item.text || "").join(" ")
: (message.content || "");
return sum + estimateTokens(messageText);
}, 0);
setTokenUsage({
prompt_tokens: promptTokenEstimate,
completion_tokens: 0,
total_tokens: promptTokenEstimate,
});
let assistantText = "";
let assistantReasoning = "";
@@ -487,14 +562,20 @@ function TextChat({
let thinkingEndedAt = null;
let thinkingDuration = 0;
const baseMaxTokens = textSettings?.maxTokens || 384;
const effectiveMaxTokens = textSettings?.enableThinking !== false
? Math.max(baseMaxTokens, 1024)
: baseMaxTokens;
const response = await streamChatWithLlm(requestMessages, {
temperature: textSettings?.temperature || 0.7,
maxTokens: textSettings?.maxTokens || 1024,
maxTokens: effectiveMaxTokens,
topP: textSettings?.topP,
topK: textSettings?.topK,
minP: textSettings?.minP,
repeatPenalty: textSettings?.repeatPenalty,
seed: textSettings?.seed,
enableThinking: textSettings?.enableThinking !== false,
signal: controller.signal,
}, (_token, fullText, _reasoningToken, fullReasoning) => {
const now = performance.now();
@@ -530,7 +611,6 @@ function TextChat({
tokensPerSecond: streamedTokens / generationSeconds,
seconds: (now - requestStartedAt) / 1000,
};
// rAF batching: accumulate updates and flush once per frame (~16ms)
// This reduces React re-renders from N per token to ~60/sec
tokenBufferRef.current = {
@@ -583,6 +663,8 @@ function TextChat({
tokens: exactTokens,
tokensPerSecond: exactTokensPerSecond,
seconds: exactSeconds,
finishReason: response.finishReason || null,
truncated: response.finishReason === "length",
};
const processed = processMessageContent(assistantText, response.reasoningContent || assistantReasoning, textSettings?.enableThinking !== false);
@@ -595,7 +677,23 @@ function TextChat({
}];
setMessages(finalMessages);
saveConversationState(convId, finalMessages, selectedModel);
if (response.usage) setTokenUsage(response.usage);
const finalCompletionEstimate = Math.max(
exactTokens,
estimateTokens(processed.content) + estimateTokens(processed.reasoning)
);
const finalUsageEstimate = {
prompt_tokens: promptTokenEstimate,
completion_tokens: finalCompletionEstimate,
total_tokens: promptTokenEstimate + finalCompletionEstimate,
};
setTokenUsage(response.usage
? {
prompt_tokens: Math.max(Number(response.usage.prompt_tokens) || 0, finalUsageEstimate.prompt_tokens),
completion_tokens: Math.max(Number(response.usage.completion_tokens) || 0, finalUsageEstimate.completion_tokens),
total_tokens: Math.max(Number(response.usage.total_tokens) || 0, finalUsageEstimate.total_tokens),
}
: finalUsageEstimate
);
// Clean up attached files on success
setAttachments([]);
@@ -845,7 +943,13 @@ function TextChat({
{/* Generation stats pill */}
{message.role === "assistant" && message.generationStats && !message.error && (
<div className={`chat-generation-stats ${message.generationStats.status}`}>
<>
{message.generationStats.truncated && (
<div className="chat-generation-warning">
Response reached the token limit. Increase Max Response Tokens or ask "continue".
</div>
)}
<div className={`chat-generation-stats ${message.generationStats.status}`}>
{message.generationStats.status === "starting" ? (
<><LoaderCircle size={11} className="progress-spinner" /> Waiting for first token...</>
) : message.generationStats.status === "streaming" ? (
@@ -853,7 +957,8 @@ function TextChat({
) : (
<>{message.generationStats.tokens} tokens <span style={{ opacity: 0.5 }}>•</span> {formatGenerationTime(message.generationStats.seconds)}</>
)}
</div>
</div>
</>
)}
</div>
</div>
@@ -932,13 +1037,7 @@ function TextChat({
{status.ready && supportsThinking && (
<button
className={`chat-composer-deepthink-btn ${textSettings.enableThinking !== false ? "active" : ""}`}
onClick={() => {
const newVal = textSettings.enableThinking === false;
setTextSettings(prev => ({
...prev,
enableThinking: newVal
}));
}}
onClick={handleThinkingToggle}
title={textSettings.enableThinking !== false ? "Disable DeepThink reasoning" : "Enable DeepThink reasoning"}
style={{
display: "flex",
app/frontend/src/components/TopStatusBar.jsx
+5 -3
View File
@@ -16,9 +16,11 @@ function TopStatusBar({ telemetry, serverRunning, activeModel, onStopServer, isS
return () => document.removeEventListener("mousedown", handleClickOutside);
}, []);
const formatGb = (value) => {
const formatGb = (value, { allowZero = false } = {}) => {
const number = Number(value);
return Number.isFinite(number) && number > 0 ? number.toFixed(number >= 10 ? 0 : 1) : "--";
if (!Number.isFinite(number) || number < 0) return "--";
if (number === 0 && !allowZero) return "--";
return number.toFixed(number >= 10 ? 0 : 1);
};
const getStatusText = () => {
@@ -128,7 +130,7 @@ function TopStatusBar({ telemetry, serverRunning, activeModel, onStopServer, isS
{telemetry.vram_total_gb > 0 && (
<div className="telemetry-chip" title={`${telemetry.gpu_name} VRAM`}>
<Database className="telemetry-chip-icon" style={{ color: "var(--md-sys-color-tertiary)" }} />
<span>VRAM: {formatGb(telemetry.vram_used_gb)} / {formatGb(telemetry.vram_total_gb)} GB</span>
<span>VRAM: {formatGb(telemetry.vram_used_gb, { allowZero: true })} / {formatGb(telemetry.vram_total_gb)} GB</span>
</div>
)}
</div>
app/frontend/src/services/api.js
+16 -3
View File
@@ -347,6 +347,16 @@ export async function startLlm(model, options = {}) {
contextSize: options.contextSize,
gpuLayers: options.gpuLayers,
enableThinking: options.enableThinking,
flashAttn: options.flashAttn,
cacheTypeK: options.cacheTypeK,
cacheTypeV: options.cacheTypeV,
mlock: options.mlock,
mmap: options.mmap,
cachePrompt: options.cachePrompt,
defragThold: options.defragThold,
batchSize: options.batchSize,
ubatchSize: options.ubatchSize,
performanceProfile: options.performanceProfile,
}),
});
return await readJsonResponse(res, "The local server returned an invalid text backend response.");
@@ -423,6 +433,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
let reasoningContent = "";
let usage = null;
let timings = null;
let finishReason = null;
const consumeEvent = (eventText) => {
try {
@@ -435,8 +446,10 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
if (!data || data === "[DONE]") return data === "[DONE]";
const parsed = JSON.parse(data);
const token = parsed.choices?.[0]?.delta?.content || "";
const reasoningToken = parsed.choices?.[0]?.delta?.reasoning_content || "";
const choice = parsed.choices?.[0];
const token = choice?.delta?.content || "";
const reasoningToken = choice?.delta?.reasoning_content || "";
if (choice?.finish_reason) finishReason = choice.finish_reason;
if (token || reasoningToken) {
content += token;
@@ -470,7 +483,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
if (!finished && buffer.trim()) consumeEvent(buffer);
if (!content && !reasoningContent) throw new Error("The text model returned an empty streamed response.");
return { content, reasoningContent, usage, timings };
return { content, reasoningContent, usage, timings, finishReason };
}
export async function downloadLlmModel(url, filename = null) {
scripts/serve.cjs
+133 -20
View File
@@ -9,7 +9,7 @@ const fs = require("fs");
const net = require("net");
const os = require("os");
const path = require("path");
const { spawn, spawnSync, execSync, exec } = require("child_process");
const { spawn, spawnSync, execSync, exec, execFile } = require("child_process");
// HTTP keep-alive agent for llama-server (eliminates TCP handshake per request)
const llmHttpAgent = new http.Agent({
@@ -697,6 +697,32 @@ function chooseAutoLayers(modelFilename, freeVramBytes, cacheTypeK = "q8_0", cac
return Math.max(0, maxLayers);
}
function getGgufModelSizeGb(modelFilename) {
try {
const modelPath = path.join(LLM_MODELS, modelFilename);
if (fs.existsSync(modelPath)) {
return fs.statSync(modelPath).size / (1024 * 1024 * 1024);
}
} catch (_) {}
return 4;
}
function getStableSyclGpuLayers(modelFilename, requestedLayers) {
if (process.env.LOCALAI_LLM_ALLOW_FULL_SYCL === "1") return requestedLayers;
if (requestedLayers !== -1) return requestedLayers;
const syclGpu = detectLlamaSyclGpu();
const vramGb = syclGpu?.free_vram_gb || syclGpu?.vram_gb || 0;
const modelSizeGb = getGgufModelSizeGb(modelFilename);
if (vramGb > 0 && vramGb <= 8.5 && modelSizeGb >= 2.5) {
return 0;
}
if (vramGb > 0 && vramGb <= 12 && modelSizeGb >= 5) {
return 0;
}
return requestedLayers;
}
let nvidiaSmiCmd = "nvidia-smi";
let hasNvidiaSmi = null;
@@ -768,6 +794,77 @@ function getNvidiaVram() {
return cachedVramInfo;
}
let cachedLlamaVramInfo = null;
let isPollingLlamaVram = false;
let lastLlamaVramPollTime = 0;
function parseLlamaDeviceMemory(output) {
const devices = [];
const devicePattern = /(CUDA|Vulkan|SYCL)\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi;
let match;
while ((match = devicePattern.exec(output || "")) !== null) {
const totalMiB = Number(match[3]);
const freeMiB = match[4] ? Number(match[4]) : NaN;
if (!Number.isFinite(totalMiB) || totalMiB <= 0) continue;
const usedMiB = Number.isFinite(freeMiB) ? Math.max(0, totalMiB - freeMiB) : 0;
devices.push({
gpu_name: match[2].trim(),
vram_used_gb: Math.round((usedMiB / 1024) * 100) / 100,
vram_total_gb: Math.round((totalMiB / 1024) * 100) / 100,
});
}
return devices.sort((a, b) => b.vram_total_gb - a.vram_total_gb)[0] || null;
}
function getLlamaTelemetryBackendPath() {
if (osPlatform === "win32") {
if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) return LLM_BACKEND_PATHS.winVulkan;
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) return LLM_BACKEND_PATHS.winSycl;
if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) return LLM_BACKEND_PATHS.winCuda;
} else if (osPlatform === "linux") {
if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) return LLM_BACKEND_PATHS.linuxVulkan;
if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl)) return LLM_BACKEND_PATHS.linuxSycl;
if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) return LLM_BACKEND_PATHS.linuxCuda;
}
return null;
}
function pollLlamaVram(force = false) {
if (isPollingLlamaVram) return;
const now = Date.now();
if (!force && now - lastLlamaVramPollTime < 4500) {
return;
}
const backendPath = getLlamaTelemetryBackendPath();
if (!backendPath) return;
isPollingLlamaVram = true;
lastLlamaVramPollTime = now;
execFile(
backendPath,
["--list-devices"],
{ windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024 },
(error, stdout, stderr) => {
isPollingLlamaVram = false;
if (error) {
cachedLlamaVramInfo = null;
return;
}
cachedLlamaVramInfo = parseLlamaDeviceMemory(`${stdout || ""}\n${stderr || ""}`);
}
);
}
setInterval(() => pollLlamaVram(false), 5000);
pollLlamaVram(true);
function getLlamaVram() {
return cachedLlamaVramInfo;
}
let cachedMacRamUsedGb = null;
function pollMacRam() {
if (osPlatform !== "darwin") return;
@@ -963,6 +1060,7 @@ function getHardwareSpecs() {
function getTelemetry() {
const vram = getNvidiaVram();
const llamaVram = vram || getLlamaVram();
let ram_used_gb = roundGb(os.totalmem() - os.freemem());
if (osPlatform === "darwin" && cachedMacRamUsedGb !== null) {
ram_used_gb = cachedMacRamUsedGb;
@@ -972,9 +1070,9 @@ function getTelemetry() {
cpu_usage: getCpuUsagePercent(),
ram_used_gb,
ram_total_gb: roundGb(os.totalmem()),
gpu_name: vram?.gpu_name || gpu.name,
vram_used_gb: vram?.vram_used_gb || 0,
vram_total_gb: vram?.vram_total_gb || gpu.vram_gb || 0,
gpu_name: llamaVram?.gpu_name || gpu.name,
vram_used_gb: Number.isFinite(Number(llamaVram?.vram_used_gb)) ? llamaVram.vram_used_gb : 0,
vram_total_gb: llamaVram?.vram_total_gb || gpu.vram_gb || 0,
};
}
@@ -1189,21 +1287,21 @@ function getLlmBackend() {
if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) {
return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) {
return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) {
return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) {
return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) {
return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) {
return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) {
return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
}
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) {
return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
}
return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" };
} else if (osPlatform === "darwin") {
return {
@@ -2405,26 +2503,37 @@ async function startLlm(settings = {}) {
contextSize = Math.max(512, Math.min(32768, contextSize));
}
const isSyclBackend = backend.mode.includes("SYCL") || path.basename(path.dirname(backend.path)).toLowerCase() === "sycl";
const requestedGpuLayers = Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1;
const effectiveGpuLayers = isSyclBackend ? getStableSyclGpuLayers(filename, requestedGpuLayers) : requestedGpuLayers;
const effectiveFlashAttn = isSyclBackend ? false : settings.flashAttn !== false;
const effectiveCacheTypeK = isSyclBackend ? "q4_0" : settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0";
const effectiveCacheTypeV = isSyclBackend ? "q4_0" : settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0";
const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(settings.batchSize) || llmSettings.batchSize || 512) : Number(settings.batchSize) || llmSettings.batchSize || 512;
const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(settings.ubatchSize) || llmSettings.ubatchSize || 512;
const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : settings.enableThinking !== false;
llmSettings = {
...llmSettings,
model: filename,
threads: Math.max(1, Math.min(64, Number(settings.threads) || llmSettings.threads)),
contextSize: contextSize,
gpuLayers: Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1,
gpuLayers: effectiveGpuLayers,
requestedGpuLayers,
backendMode: backend.mode,
backendBinary: path.basename(backend.path),
enableThinking: settings.enableThinking !== false,
enableThinking: effectiveEnableThinking,
supportsThinking: /deepseek|qwen3|gemma-4|gemma4|think|r1|e2b|reasoning/i.test(filename),
// New performance settings
flashAttn: settings.flashAttn !== false,
cacheTypeK: settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0",
cacheTypeV: settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0",
flashAttn: effectiveFlashAttn,
cacheTypeK: effectiveCacheTypeK,
cacheTypeV: effectiveCacheTypeV,
mlock: settings.mlock || llmSettings.mlock || false,
mmap: settings.mmap !== false && llmSettings.mmap !== false,
cachePrompt: settings.cachePrompt !== false && llmSettings.cachePrompt !== false,
defragThold: Number(settings.defragThold) || llmSettings.defragThold || 0.1,
batchSize: Math.max(128, Math.min(2048, Number(settings.batchSize) || llmSettings.batchSize || 512)),
ubatchSize: Math.max(128, Math.min(2048, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512)),
batchSize: Math.max(128, Math.min(2048, effectiveBatchSize)),
ubatchSize: Math.max(128, Math.min(2048, effectiveUbatchSize)),
performanceProfile: settings.performanceProfile || llmSettings.performanceProfile || "balanced",
};
@@ -2500,21 +2609,26 @@ async function startLlm(settings = {}) {
"--model", modelPath,
"--host", "127.0.0.1",
"--port", String(PORT_LLM),
"-lv", "1",
"--ctx-size", String(llmSettings.contextSize),
"--threads", String(llmSettings.threads),
"--n-gpu-layers", String(llmSettings.gpuLayers),
"--parallel", "1",
"--flash-attn", "on", // ✅ Memory-efficient attention (all backends)
"--cache-type-k", String(llmSettings.cacheTypeK), // ✅ KV cache quantization K
"--cache-type-v", String(llmSettings.cacheTypeV), // ✅ KV cache quantization V
"--cache-prompt", // ✅ Prompt caching for multi-turn (major UX win)
"--ctx-checkpoints", "8", // ✅ Context shifting recovery
"--poll", "30", // ✅ More aggressive polling
"--metrics", // ✅ Metrics endpoint for tuning
"--defrag-thold", String(llmSettings.defragThold), // ✅ KV cache defragmentation
"--batch-size", String(llmSettings.batchSize), // ✅ Prompt processing batch
"--ubatch-size", String(llmSettings.ubatchSize), // ✅ Micro-batch size
];
if (llmSettings.flashAttn) {
args.push("--flash-attn", "on");
}
if (isSyclBackend) {
args.push("--no-warmup");
}
// CPU mode: lock memory to prevent OS paging (critical for consistent speed)
const isGpuMode = backend.mode.includes("GPU") || backend.mode.includes("CUDA") ||
@@ -2565,7 +2679,6 @@ async function startLlm(settings = {}) {
// SYCL-specific optimizations (Intel Arc/Graphics)
if (backend.mode.includes("SYCL")) {
spawnEnv.GGML_SYCL_ENABLE_FLASH_ATTN = "1";
spawnEnv.ZES_ENABLE_SYSMAN = "1";
}