diff --git a/app/frontend/src/App.jsx b/app/frontend/src/App.jsx
index 855d107..36775bd 100644
--- a/app/frontend/src/App.jsx
+++ b/app/frontend/src/App.jsx
@@ -122,7 +122,7 @@ function App() {
} catch (_) {}
}
return {
- contextSize: 4096,
+ contextSize: 0,
temperature: 0.7,
systemPrompt: "You are a helpful local AI assistant.",
threads: 4
diff --git a/app/frontend/src/components/Settings.jsx b/app/frontend/src/components/Settings.jsx
index e42e839..92108f8 100644
--- a/app/frontend/src/components/Settings.jsx
+++ b/app/frontend/src/components/Settings.jsx
@@ -512,8 +512,9 @@ function Settings({
className="m3-input"
style={{ marginTop: "6px", height: "40px", cursor: "pointer" }}
>
+
-
+
diff --git a/linux.sh b/linux.sh
index f814a98..acce2af 100755
--- a/linux.sh
+++ b/linux.sh
@@ -127,12 +127,13 @@ fi
# At minimum we need CPU or Vulkan backend on Linux, and both CLI and server binaries must be executable
CPU_SERVER_PATH="$APP_DIR/backend/linux/cpu/sd-server-cpu"
VULKAN_SERVER_PATH="$APP_DIR/backend/linux/vulkan/sd-server-vulkan"
+LLM_CUDA_PATH="$APP_DIR/llm-backend/linux/cuda/llama-server"
LLM_VULKAN_PATH="$APP_DIR/llm-backend/linux/vulkan/llama-server"
LLM_CPU_PATH="$APP_DIR/llm-backend/linux/cpu/llama-server"
if [[ ! -x "$CPU_BACKEND_PATH" || ! -x "$CPU_SERVER_PATH" ]] && [[ ! -x "$BACKEND_PATH" || ! -x "$VULKAN_SERVER_PATH" ]]; then
SETUP_REASON="Linux backend binaries are missing or not executable."
fi
-if [[ ! -x "$LLM_VULKAN_PATH" && ! -x "$LLM_CPU_PATH" ]]; then
+if [[ ! -x "$LLM_CUDA_PATH" && ! -x "$LLM_VULKAN_PATH" && ! -x "$LLM_CPU_PATH" ]]; then
SETUP_REASON="Linux llama.cpp text backend is missing or not executable."
fi
diff --git a/scripts/serve.cjs b/scripts/serve.cjs
index 595566d..6b9aa27 100644
--- a/scripts/serve.cjs
+++ b/scripts/serve.cjs
@@ -78,8 +78,10 @@ if (!fs.existsSync(LLM_MODELS)) {
fs.mkdirSync(LLM_MODELS, { recursive: true });
}
const LLM_BACKEND_PATHS = {
+ winCuda: path.join(ROOT, "app", "llm-backend", "win", "cuda", "llama-server.exe"),
winVulkan: path.join(ROOT, "app", "llm-backend", "win", "vulkan", "llama-server.exe"),
winCpu: path.join(ROOT, "app", "llm-backend", "win", "cpu", "llama-server.exe"),
+ linuxCuda: path.join(ROOT, "app", "llm-backend", "linux", "cuda", "llama-server"),
linuxVulkan: path.join(ROOT, "app", "llm-backend", "linux", "vulkan", "llama-server"),
linuxCpu: path.join(ROOT, "app", "llm-backend", "linux", "cpu", "llama-server"),
macArm64: path.join(ROOT, "app", "llm-backend", "mac", "arm64", "llama-server"),
@@ -220,7 +222,50 @@ function detectLinuxGpuFromSysfs() {
return null;
}
+let cachedLlamaCudaGpu = null;
+let cachedLlamaCudaGpuChecked = false;
+function detectLlamaCudaGpu() {
+ if (cachedLlamaCudaGpuChecked) return cachedLlamaCudaGpu;
+ cachedLlamaCudaGpuChecked = true;
+
+ const backendPath = osPlatform === "win32"
+ ? LLM_BACKEND_PATHS.winCuda
+ : osPlatform === "linux"
+ ? LLM_BACKEND_PATHS.linuxCuda
+ : "";
+ if (!backendPath || !fs.existsSync(backendPath)) return null;
+
+ try {
+ const result = spawnSync(backendPath, ["--list-devices"], {
+ encoding: "utf8",
+ windowsHide: true,
+ timeout: 10000,
+ maxBuffer: 1024 * 1024,
+ });
+ const output = `${result.stdout || ""}\n${result.stderr || ""}`;
+ const devices = [];
+ const devicePattern = /CUDA\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi;
+ let match;
+ while ((match = devicePattern.exec(output)) !== null) {
+ devices.push({
+ name: match[1].trim(),
+ vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100,
+ free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null,
+ });
+ }
+ cachedLlamaCudaGpu = devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
+ } catch (_) {
+ cachedLlamaCudaGpu = null;
+ }
+ return cachedLlamaCudaGpu;
+}
+
+let cachedLlamaVulkanGpu = null;
+let cachedLlamaVulkanGpuChecked = false;
function detectLlamaVulkanGpu() {
+ if (cachedLlamaVulkanGpuChecked) return cachedLlamaVulkanGpu;
+ cachedLlamaVulkanGpuChecked = true;
+
const backendPath = osPlatform === "win32"
? LLM_BACKEND_PATHS.winVulkan
: osPlatform === "linux"
@@ -246,16 +291,23 @@ function detectLlamaVulkanGpu() {
free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null,
});
}
- return devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
+ cachedLlamaVulkanGpu = devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
} catch (_) {
- return null;
+ cachedLlamaVulkanGpu = null;
}
+ return cachedLlamaVulkanGpu;
}
function getGpuInfo() {
if (cachedGpuInfo) return cachedGpuInfo;
- if (osPlatform === "win32") {
+ if (osPlatform === "win32" || osPlatform === "linux") {
+ const llamaCudaGpu = detectLlamaCudaGpu();
+ if (llamaCudaGpu) {
+ cachedGpuInfo = llamaCudaGpu;
+ return cachedGpuInfo;
+ }
+
const llamaVulkanGpu = detectLlamaVulkanGpu();
if (llamaVulkanGpu) {
cachedGpuInfo = llamaVulkanGpu;
@@ -912,22 +964,40 @@ async function findAvailableLlmPort() {
}
function getLlmBackend() {
- const candidates = osPlatform === "win32"
- ? [
- { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" },
- { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" },
- ]
- : osPlatform === "darwin"
- ? [{
- path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64,
- mode: process.arch === "arm64" ? "Metal GPU" : "CPU",
- }]
- : [
- { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" },
- { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" },
- ];
-
- return candidates.find((candidate) => fs.existsSync(candidate.path)) || null;
+ if (osPlatform === "win32") {
+ if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) {
+ return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
+ }
+ if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) {
+ return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
+ }
+ if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) {
+ return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
+ }
+ if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) {
+ return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
+ }
+ return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" };
+ } else if (osPlatform === "darwin") {
+ return {
+ path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64,
+ mode: process.arch === "arm64" ? "Metal GPU" : "CPU",
+ };
+ } else {
+ if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda) && detectLlamaCudaGpu()) {
+ return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
+ }
+ if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan) && detectLlamaVulkanGpu()) {
+ return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
+ }
+ if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) {
+ return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
+ }
+ if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) {
+ return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
+ }
+ return { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" };
+ }
}
function getLlmModels() {
@@ -2007,6 +2077,56 @@ function killBackend() {
});
}
+function chooseAutoContext(modelFilename, isGpu) {
+ let modelSizeGb = 4;
+ try {
+ const modelPath = path.join(LLM_MODELS, modelFilename);
+ if (fs.existsSync(modelPath)) {
+ const stats = fs.statSync(modelPath);
+ modelSizeGb = stats.size / (1024 * 1024 * 1024);
+ }
+ } catch (_) {}
+
+ const systemRamGb = os.totalmem() / (1024 * 1024 * 1024);
+ let vramGb = 0;
+ try {
+ const gpu = getGpuInfo();
+ if (gpu && gpu.vram_gb) {
+ vramGb = gpu.vram_gb;
+ }
+ } catch (_) {}
+
+ let availableGb = 0;
+ let bufferGb = 2.0;
+
+ if (isGpu && vramGb > 0) {
+ availableGb = vramGb;
+ bufferGb = 1.5;
+ } else {
+ availableGb = systemRamGb;
+ bufferGb = 2.5;
+ }
+
+ const usableGb = Math.max(0.5, availableGb - modelSizeGb - bufferGb);
+
+ let kvPer4096Gb = 0.55;
+ if (modelSizeGb >= 5.5) {
+ kvPer4096Gb = 1.05;
+ } else if (modelSizeGb >= 4.0) {
+ kvPer4096Gb = 0.85;
+ }
+
+ const estimatedMaxCtx = (usableGb / kvPer4096Gb) * 4096;
+ const contextLadder = [32768, 24576, 16384, 12288, 8192, 4096, 2048];
+
+ for (const limit of contextLadder) {
+ if (limit <= estimatedMaxCtx) {
+ return limit;
+ }
+ }
+ return 2048;
+}
+
async function startLlm(settings = {}) {
const filename = path.basename(String(settings.model || ""));
const modelPath = path.join(LLM_MODELS, filename);
@@ -2027,11 +2147,21 @@ async function startLlm(settings = {}) {
await killLlm();
PORT_LLM = await findAvailableLlmPort();
llmError = null;
+
+ let contextSize = Number(settings.contextSize);
+ if (!contextSize || contextSize <= 0) {
+ const isGpu = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.startsWith("Auto");
+ contextSize = chooseAutoContext(filename, isGpu);
+ console.log(` [llm] Auto-selected context size: ${contextSize} tokens based on memory limits.`);
+ } else {
+ contextSize = Math.max(512, Math.min(32768, contextSize));
+ }
+
llmSettings = {
...llmSettings,
model: filename,
threads: Math.max(1, Math.min(64, Number(settings.threads) || llmSettings.threads)),
- contextSize: Math.max(512, Math.min(32768, Number(settings.contextSize) || 4096)),
+ contextSize: contextSize,
gpuLayers: Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1,
backendMode: backend.mode,
backendBinary: path.basename(backend.path),
@@ -2093,6 +2223,7 @@ async function startLlm(settings = {}) {
const output = data.toString();
process.stderr.write(" [llm-err] " + output);
if (/Vulkan\d+\s*:/i.test(output)) llmSettings.backendMode = "Vulkan GPU";
+ else if (/CUDA\d+\s*:/i.test(output)) llmSettings.backendMode = "CUDA GPU";
else if (/Metal/i.test(output) && /GPU|device/i.test(output)) llmSettings.backendMode = "Metal GPU";
else if (/\-\s+CPU\s+:/i.test(output) && llmSettings.backendMode.startsWith("Auto")) llmSettings.backendMode = "CPU";
if (/error|failed/i.test(output) && !/no error/i.test(output)) {
@@ -3225,79 +3356,145 @@ const server = http.createServer(async (req, res) => {
return json(res, 200, { ok: true });
}
+function isOomError(msg) {
+ if (!msg) return false;
+ const lower = String(msg).toLowerCase();
+ return lower.includes("out of memory") ||
+ lower.includes("failed to allocate") ||
+ lower.includes("failed to initialize the context") ||
+ lower.includes("oom") ||
+ lower.includes("not enough memory") ||
+ lower.includes("allocation failed") ||
+ lower.includes("failed to create context");
+}
+
+async function retryLowerContext() {
+ const currentCtx = llmSettings.contextSize || 4096;
+ const contextLadder = [32768, 24576, 16384, 12288, 8192, 4096, 2048, 1024, 512];
+ const nextLimit = contextLadder.find(limit => limit < currentCtx);
+ if (!nextLimit) {
+ throw new Error("Cannot lower context size any further.");
+ }
+ console.log(` [llm] Retrying with lower context limit: ${nextLimit} (was ${currentCtx})`);
+
+ const originalModel = llmSettings.model;
+ const newSettings = {
+ model: originalModel,
+ threads: llmSettings.threads,
+ contextSize: nextLimit,
+ gpuLayers: llmSettings.gpuLayers,
+ };
+
+ await startLlm(newSettings);
+}
+
+async function doLlmChat(req, res, body, retryCount = 0) {
+ try {
+ const isStream = body.stream === true;
+ const requestData = JSON.stringify({
+ model: llmSettings.model || "local-model",
+ messages: Array.isArray(body.messages) ? body.messages : [],
+ temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : 0.7,
+ max_tokens: Math.max(1, Math.min(4096, Number(body.max_tokens) || 512)),
+ stream: isStream,
+ });
+
+ if (isStream) {
+ const clientReq = http.request({
+ hostname: "127.0.0.1",
+ port: PORT_LLM,
+ path: "/v1/chat/completions",
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "Content-Length": Buffer.byteLength(requestData),
+ }
+ }, (clientRes) => {
+ if (clientRes.statusCode < 200 || clientRes.statusCode >= 300) {
+ let errorBody = "";
+ clientRes.setEncoding("utf8");
+ clientRes.on("data", (chunk) => { errorBody += chunk; });
+ clientRes.on("end", async () => {
+ let message = `Text backend returned HTTP ${clientRes.statusCode}`;
+ try {
+ message = JSON.parse(errorBody || "{}").error?.message || message;
+ } catch (_) {}
+
+ if (isOomError(message) && retryCount === 0) {
+ try {
+ await retryLowerContext();
+ return doLlmChat(req, res, body, retryCount + 1);
+ } catch (retryErr) {
+ console.error("Failed OOM recovery retry:", retryErr);
+ }
+ }
+ json(res, clientRes.statusCode || 500, { ok: false, error: message });
+ });
+ return;
+ }
+ res.writeHead(200, {
+ "Content-Type": "text/event-stream; charset=utf-8",
+ "Cache-Control": "no-cache, no-transform",
+ "Connection": "keep-alive",
+ "X-Accel-Buffering": "no",
+ });
+ res.flushHeaders?.();
+ clientRes.on("data", (chunk) => res.write(chunk));
+ clientRes.on("end", () => res.end());
+ clientRes.on("error", (err) => res.destroy(err));
+ });
+
+ clientReq.on("error", async (err) => {
+ console.error("LLM stream request error:", err);
+ if (isOomError(err.message) && retryCount === 0) {
+ try {
+ await retryLowerContext();
+ return doLlmChat(req, res, body, retryCount + 1);
+ } catch (retryErr) {
+ console.error("Failed OOM recovery retry:", retryErr);
+ }
+ }
+ if (!res.headersSent) {
+ res.writeHead(500, { "Content-Type": "application/json" });
+ res.end(JSON.stringify({ ok: false, error: err.message }));
+ } else {
+ res.end();
+ }
+ });
+
+ clientReq.write(requestData);
+ clientReq.end();
+ res.on("close", () => {
+ if (!res.writableEnded && !clientReq.destroyed) clientReq.destroy();
+ });
+ return;
+ } else {
+ try {
+ const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, JSON.parse(requestData), 300000);
+ return json(res, 200, result);
+ } catch (err) {
+ if (isOomError(err.message) && retryCount === 0) {
+ try {
+ await retryLowerContext();
+ return doLlmChat(req, res, body, retryCount + 1);
+ } catch (retryErr) {
+ console.error("Failed OOM recovery retry:", retryErr);
+ }
+ }
+ throw err;
+ }
+ }
+ } catch (err) {
+ return json(res, 500, { ok: false, error: err.message || String(err) });
+ }
+}
+
if (req.url === "/api/llm/chat" && req.method === "POST") {
const body = await readJsonBody(req, res);
if (!body) return;
if (!llmReady) return json(res, 409, { ok: false, error: "Load a text model before sending a message." });
- try {
- const isStream = body.stream === true;
- const requestData = JSON.stringify({
- model: llmSettings.model || "local-model",
- messages: Array.isArray(body.messages) ? body.messages : [],
- temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : 0.7,
- max_tokens: Math.max(1, Math.min(4096, Number(body.max_tokens) || 512)),
- stream: isStream,
- });
-
- if (isStream) {
- const clientReq = http.request({
- hostname: "127.0.0.1",
- port: PORT_LLM,
- path: "/v1/chat/completions",
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- "Content-Length": Buffer.byteLength(requestData),
- }
- }, (clientRes) => {
- if (clientRes.statusCode < 200 || clientRes.statusCode >= 300) {
- let errorBody = "";
- clientRes.setEncoding("utf8");
- clientRes.on("data", (chunk) => { errorBody += chunk; });
- clientRes.on("end", () => {
- let message = `Text backend returned HTTP ${clientRes.statusCode}`;
- try {
- message = JSON.parse(errorBody || "{}").error?.message || message;
- } catch (_) {}
- json(res, clientRes.statusCode || 500, { ok: false, error: message });
- });
- return;
- }
- res.writeHead(200, {
- "Content-Type": "text/event-stream; charset=utf-8",
- "Cache-Control": "no-cache, no-transform",
- "Connection": "keep-alive",
- "X-Accel-Buffering": "no",
- });
- res.flushHeaders?.();
- clientRes.on("data", (chunk) => res.write(chunk));
- clientRes.on("end", () => res.end());
- clientRes.on("error", (err) => res.destroy(err));
- });
-
- clientReq.on("error", (err) => {
- console.error("LLM stream request error:", err);
- if (!res.headersSent) {
- res.writeHead(500, { "Content-Type": "application/json" });
- res.end(JSON.stringify({ ok: false, error: err.message }));
- } else {
- res.end();
- }
- });
-
- clientReq.write(requestData);
- clientReq.end();
- res.on("close", () => {
- if (!res.writableEnded && !clientReq.destroyed) clientReq.destroy();
- });
- return;
- } else {
- const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, JSON.parse(requestData), 300000);
- return json(res, 200, result);
- }
- } catch (err) {
- return json(res, 500, { ok: false, error: err.message || String(err) });
- }
+ await doLlmChat(req, res, body);
+ return;
}
// GET /api/hardware-specs
diff --git a/scripts/setup-llama.ps1 b/scripts/setup-llama.ps1
index 20fcc02..0ab61a8 100644
--- a/scripts/setup-llama.ps1
+++ b/scripts/setup-llama.ps1
@@ -30,37 +30,23 @@ function Format-Speed {
return "{0:N0} KB/s" -f ($bps / 1KB)
}
-function Install-LlamaArchive {
- param([string]$Variant, [string]$AssetName)
-
- $dest = Join-Path $llmRoot $Variant
- $server = Join-Path $dest "llama-server.exe"
- if (Test-Path $server) {
- Write-Host " OK llama.cpp $Variant backend already ready."
- return
- }
-
- $archive = Join-Path $toolsDir $AssetName
- $extract = Join-Path $toolsDir "llama-$Variant-extract"
- $url = "https://github.com/ggml-org/llama.cpp/releases/download/$Release/$AssetName"
-
- New-Item -ItemType Directory -Force -Path $toolsDir, $dest | Out-Null
- Remove-Item $archive, $extract -Recurse -Force -ErrorAction SilentlyContinue
-
- Write-Host " >> Downloading llama.cpp $Variant backend ($Release)..."
+function Download-File {
+ param([string]$Url, [string]$DestPath, [string]$Label)
$barWidth = 48
$lastBytes = [long]0
$lastTime = [DateTime]::Now
+ Write-Host " >> Downloading $Label..."
+
try {
Enable-Tls12
- $req = [System.Net.HttpWebRequest]::Create($url)
+ $req = [System.Net.HttpWebRequest]::Create($Url)
$req.UserAgent = "Mozilla/5.0"
$resp = $req.GetResponse()
$total = [long]$resp.ContentLength
$stream = $resp.GetResponseStream()
- $out = [System.IO.File]::Create($archive)
+ $out = [System.IO.File]::Create($DestPath)
$buf = New-Object byte[] 65536
$done = [long]0
@@ -100,6 +86,26 @@ function Install-LlamaArchive {
Write-Host ""
throw "Download failed: $_"
}
+}
+
+function Install-LlamaArchive {
+ param([string]$Variant, [string]$AssetName)
+
+ $dest = Join-Path $llmRoot $Variant
+ $server = Join-Path $dest "llama-server.exe"
+ if (Test-Path $server) {
+ Write-Host " OK llama.cpp $Variant backend already ready."
+ return
+ }
+
+ $archive = Join-Path $toolsDir $AssetName
+ $extract = Join-Path $toolsDir "llama-$Variant-extract"
+ $url = "https://github.com/ggml-org/llama.cpp/releases/download/$Release/$AssetName"
+
+ New-Item -ItemType Directory -Force -Path $toolsDir, $dest | Out-Null
+ Remove-Item $archive, $extract -Recurse -Force -ErrorAction SilentlyContinue
+
+ Download-File -Url $url -DestPath $archive -Label "llama.cpp $Variant backend ($Release)"
Write-Host " >> Extracting llama.cpp $Variant backend..."
Expand-Archive -Path $archive -DestinationPath $extract -Force
@@ -109,11 +115,49 @@ function Install-LlamaArchive {
}
Remove-Item $archive, $extract -Recurse -Force -ErrorAction SilentlyContinue
+ if ($Variant -eq "cuda") {
+ $cudartAsset = "cudart-llama-bin-win-cuda-12.4-x64.zip"
+ $cudartArchive = Join-Path $toolsDir $cudartAsset
+ $cudartExtract = Join-Path $toolsDir "llama-cudart-extract"
+ $cudartUrl = "https://github.com/ggml-org/llama.cpp/releases/download/$Release/$cudartAsset"
+
+ Remove-Item $cudartArchive, $cudartExtract -Recurse -Force -ErrorAction SilentlyContinue
+
+ Download-File -Url $cudartUrl -DestPath $cudartArchive -Label "llama.cpp CUDA runtime library ($Release)"
+
+ Write-Host " >> Extracting CUDA runtime libraries..."
+ Expand-Archive -Path $cudartArchive -DestinationPath $cudartExtract -Force
+
+ Get-ChildItem $cudartExtract -Recurse -File | ForEach-Object {
+ Copy-Item $_.FullName (Join-Path $dest $_.Name) -Force
+ }
+ Remove-Item $cudartArchive, $cudartExtract -Recurse -Force -ErrorAction SilentlyContinue
+ }
+
if (-not (Test-Path $server)) {
throw "llama-server.exe was not found after extracting $AssetName"
}
Write-Host " OK llama.cpp $Variant backend installed."
}
+$hasNvidia = $false
+try {
+ $gpus = Get-CimInstance Win32_VideoController -ErrorAction SilentlyContinue
+ foreach ($gpu in $gpus) {
+ if ($gpu.Name -like "*NVIDIA*") {
+ $hasNvidia = $true
+ }
+ }
+} catch {}
+if (-not $hasNvidia) {
+ try {
+ & nvidia-smi *> $null
+ if ($LASTEXITCODE -eq 0) { $hasNvidia = $true }
+ } catch {}
+}
+
+if ($hasNvidia) {
+ Install-LlamaArchive -Variant "cuda" -AssetName "llama-$Release-bin-win-cuda-12.4-x64.zip"
+}
Install-LlamaArchive -Variant "vulkan" -AssetName "llama-$Release-bin-win-vulkan-x64.zip"
Install-LlamaArchive -Variant "cpu" -AssetName "llama-$Release-bin-win-cpu-x64.zip"
diff --git a/windows.bat b/windows.bat
index 2a2a95a..94e435f 100644
--- a/windows.bat
+++ b/windows.bat
@@ -9,6 +9,7 @@ set DIST=%APP%\dist\index.html
set SETUP=%~dp0scripts\setup.ps1
set CUDA_BACKEND=%APP%\backend\win\cuda\sd-cuda.exe
set VULKAN_BACKEND=%APP%\backend\win\vulkan\sd-vulkan.exe
+set LLM_CUDA_BACKEND=%APP%\llm-backend\win\cuda\llama-server.exe
set LLM_VULKAN_BACKEND=%APP%\llm-backend\win\vulkan\llama-server.exe
set LLM_CPU_BACKEND=%APP%\llm-backend\win\cpu\llama-server.exe
set SERVE=%~dp0scripts\serve.cjs
@@ -30,7 +31,7 @@ if not exist "%DIST%" (
set SETUP_REASON=Frontend build is missing.
goto :run_setup
)
-if not exist "%LLM_VULKAN_BACKEND%" if not exist "%LLM_CPU_BACKEND%" (
+if not exist "%LLM_CUDA_BACKEND%" if not exist "%LLM_VULKAN_BACKEND%" if not exist "%LLM_CPU_BACKEND%" (
set SETUP_REASON=llama.cpp text backend is missing.
goto :run_setup
)