feat: implement LLM backend management support with new CJS server and frontend components

app/frontend/src/App.jsx

@@ -122,7 +122,7 @@ function App() {
       } catch (_) {}
     }
     return {
-      contextSize: 4096,
+      contextSize: 0,
       temperature: 0.7,
       systemPrompt: "You are a helpful local AI assistant.",
       threads: 4

app/frontend/src/components/Settings.jsx

@@ -512,8 +512,9 @@ function Settings({
                   className="m3-input"
                   style={{ marginTop: "6px", height: "40px", cursor: "pointer" }}
                 >
+                  <option value="0">Auto-detect (Recommended)</option>
                   <option value="2048">2,048 tokens (Fast/Low Memory)</option>
-                  <option value="4096">4,096 tokens (Balanced / Recommended)</option>
+                  <option value="4096">4,096 tokens (Balanced)</option>
                   <option value="8192">8,192 tokens (More history)</option>
                   <option value="16384">16,384 tokens (Deep context)</option>
                 </select>

linux.sh

@@ -127,12 +127,13 @@ fi
 # At minimum we need CPU or Vulkan backend on Linux, and both CLI and server binaries must be executable
 CPU_SERVER_PATH="$APP_DIR/backend/linux/cpu/sd-server-cpu"
 VULKAN_SERVER_PATH="$APP_DIR/backend/linux/vulkan/sd-server-vulkan"
+LLM_CUDA_PATH="$APP_DIR/llm-backend/linux/cuda/llama-server"
 LLM_VULKAN_PATH="$APP_DIR/llm-backend/linux/vulkan/llama-server"
 LLM_CPU_PATH="$APP_DIR/llm-backend/linux/cpu/llama-server"
 if [[ ! -x "$CPU_BACKEND_PATH" || ! -x "$CPU_SERVER_PATH" ]] && [[ ! -x "$BACKEND_PATH" || ! -x "$VULKAN_SERVER_PATH" ]]; then
   SETUP_REASON="Linux backend binaries are missing or not executable."
 fi
-if [[ ! -x "$LLM_VULKAN_PATH" && ! -x "$LLM_CPU_PATH" ]]; then
+if [[ ! -x "$LLM_CUDA_PATH" && ! -x "$LLM_VULKAN_PATH" && ! -x "$LLM_CPU_PATH" ]]; then
   SETUP_REASON="Linux llama.cpp text backend is missing or not executable."
 fi
 

scripts/serve.cjs

@@ -78,8 +78,10 @@ if (!fs.existsSync(LLM_MODELS)) {
   fs.mkdirSync(LLM_MODELS, { recursive: true });
 }
 const LLM_BACKEND_PATHS = {
+  winCuda: path.join(ROOT, "app", "llm-backend", "win", "cuda", "llama-server.exe"),
   winVulkan: path.join(ROOT, "app", "llm-backend", "win", "vulkan", "llama-server.exe"),
   winCpu: path.join(ROOT, "app", "llm-backend", "win", "cpu", "llama-server.exe"),
+  linuxCuda: path.join(ROOT, "app", "llm-backend", "linux", "cuda", "llama-server"),
   linuxVulkan: path.join(ROOT, "app", "llm-backend", "linux", "vulkan", "llama-server"),
   linuxCpu: path.join(ROOT, "app", "llm-backend", "linux", "cpu", "llama-server"),
   macArm64: path.join(ROOT, "app", "llm-backend", "mac", "arm64", "llama-server"),
@@ -220,7 +222,50 @@ function detectLinuxGpuFromSysfs() {
   return null;
 }
 
+let cachedLlamaCudaGpu = null;
+let cachedLlamaCudaGpuChecked = false;
+function detectLlamaCudaGpu() {
+  if (cachedLlamaCudaGpuChecked) return cachedLlamaCudaGpu;
+  cachedLlamaCudaGpuChecked = true;
+
+  const backendPath = osPlatform === "win32"
+    ? LLM_BACKEND_PATHS.winCuda
+    : osPlatform === "linux"
+      ? LLM_BACKEND_PATHS.linuxCuda
+      : "";
+  if (!backendPath || !fs.existsSync(backendPath)) return null;
+
+  try {
+    const result = spawnSync(backendPath, ["--list-devices"], {
+      encoding: "utf8",
+      windowsHide: true,
+      timeout: 10000,
+      maxBuffer: 1024 * 1024,
+    });
+    const output = `${result.stdout || ""}\n${result.stderr || ""}`;
+    const devices = [];
+    const devicePattern = /CUDA\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi;
+    let match;
+    while ((match = devicePattern.exec(output)) !== null) {
+      devices.push({
+        name: match[1].trim(),
+        vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100,
+        free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null,
+      });
+    }
+    cachedLlamaCudaGpu = devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
+  } catch (_) {
+    cachedLlamaCudaGpu = null;
+  }
+  return cachedLlamaCudaGpu;
+}
+
+let cachedLlamaVulkanGpu = null;
+let cachedLlamaVulkanGpuChecked = false;
 function detectLlamaVulkanGpu() {
+  if (cachedLlamaVulkanGpuChecked) return cachedLlamaVulkanGpu;
+  cachedLlamaVulkanGpuChecked = true;
+
   const backendPath = osPlatform === "win32"
     ? LLM_BACKEND_PATHS.winVulkan
     : osPlatform === "linux"
@@ -246,16 +291,23 @@ function detectLlamaVulkanGpu() {
         free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null,
       });
     }
-    return devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
+    cachedLlamaVulkanGpu = devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
   } catch (_) {
-    return null;
+    cachedLlamaVulkanGpu = null;
   }
+  return cachedLlamaVulkanGpu;
 }
 
 function getGpuInfo() {
   if (cachedGpuInfo) return cachedGpuInfo;
 
-  if (osPlatform === "win32") {
+  if (osPlatform === "win32" || osPlatform === "linux") {
+    const llamaCudaGpu = detectLlamaCudaGpu();
+    if (llamaCudaGpu) {
+      cachedGpuInfo = llamaCudaGpu;
+      return cachedGpuInfo;
+    }
+
     const llamaVulkanGpu = detectLlamaVulkanGpu();
     if (llamaVulkanGpu) {
       cachedGpuInfo = llamaVulkanGpu;
@@ -912,22 +964,40 @@ async function findAvailableLlmPort() {
 }
 
 function getLlmBackend() {
-  const candidates = osPlatform === "win32"
-    ? [
-        { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" },
-        { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" },
-      ]
-    : osPlatform === "darwin"
-      ? [{
-          path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64,
-          mode: process.arch === "arm64" ? "Metal GPU" : "CPU",
-        }]
-      : [
-          { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" },
-          { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" },
-        ];
-
-  return candidates.find((candidate) => fs.existsSync(candidate.path)) || null;
+  if (osPlatform === "win32") {
+    if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) {
+      return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
+    }
+    if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) {
+      return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
+    }
+    if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) {
+      return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
+    }
+    if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) {
+      return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
+    }
+    return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" };
+  } else if (osPlatform === "darwin") {
+    return {
+      path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64,
+      mode: process.arch === "arm64" ? "Metal GPU" : "CPU",
+    };
+  } else {
+    if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda) && detectLlamaCudaGpu()) {
+      return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
+    }
+    if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan) && detectLlamaVulkanGpu()) {
+      return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
+    }
+    if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) {
+      return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
+    }
+    if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) {
+      return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
+    }
+    return { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" };
+  }
 }
 
 function getLlmModels() {
@@ -2007,6 +2077,56 @@ function killBackend() {
   });
 }
 
+function chooseAutoContext(modelFilename, isGpu) {
+  let modelSizeGb = 4;
+  try {
+    const modelPath = path.join(LLM_MODELS, modelFilename);
+    if (fs.existsSync(modelPath)) {
+      const stats = fs.statSync(modelPath);
+      modelSizeGb = stats.size / (1024 * 1024 * 1024);
+    }
+  } catch (_) {}
+
+  const systemRamGb = os.totalmem() / (1024 * 1024 * 1024);
+  let vramGb = 0;
+  try {
+    const gpu = getGpuInfo();
+    if (gpu && gpu.vram_gb) {
+      vramGb = gpu.vram_gb;
+    }
+  } catch (_) {}
+
+  let availableGb = 0;
+  let bufferGb = 2.0;
+
+  if (isGpu && vramGb > 0) {
+    availableGb = vramGb;
+    bufferGb = 1.5;
+  } else {
+    availableGb = systemRamGb;
+    bufferGb = 2.5;
+  }
+
+  const usableGb = Math.max(0.5, availableGb - modelSizeGb - bufferGb);
+
+  let kvPer4096Gb = 0.55;
+  if (modelSizeGb >= 5.5) {
+    kvPer4096Gb = 1.05;
+  } else if (modelSizeGb >= 4.0) {
+    kvPer4096Gb = 0.85;
+  }
+
+  const estimatedMaxCtx = (usableGb / kvPer4096Gb) * 4096;
+  const contextLadder = [32768, 24576, 16384, 12288, 8192, 4096, 2048];
+  
+  for (const limit of contextLadder) {
+    if (limit <= estimatedMaxCtx) {
+      return limit;
+    }
+  }
+  return 2048;
+}
+
 async function startLlm(settings = {}) {
   const filename = path.basename(String(settings.model || ""));
   const modelPath = path.join(LLM_MODELS, filename);
@@ -2027,11 +2147,21 @@ async function startLlm(settings = {}) {
   await killLlm();
   PORT_LLM = await findAvailableLlmPort();
   llmError = null;
+
+  let contextSize = Number(settings.contextSize);
+  if (!contextSize || contextSize <= 0) {
+    const isGpu = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.startsWith("Auto");
+    contextSize = chooseAutoContext(filename, isGpu);
+    console.log(`  [llm] Auto-selected context size: ${contextSize} tokens based on memory limits.`);
+  } else {
+    contextSize = Math.max(512, Math.min(32768, contextSize));
+  }
+
   llmSettings = {
     ...llmSettings,
     model: filename,
     threads: Math.max(1, Math.min(64, Number(settings.threads) || llmSettings.threads)),
-    contextSize: Math.max(512, Math.min(32768, Number(settings.contextSize) || 4096)),
+    contextSize: contextSize,
     gpuLayers: Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1,
     backendMode: backend.mode,
     backendBinary: path.basename(backend.path),
@@ -2093,6 +2223,7 @@ async function startLlm(settings = {}) {
     const output = data.toString();
     process.stderr.write("  [llm-err] " + output);
     if (/Vulkan\d+\s*:/i.test(output)) llmSettings.backendMode = "Vulkan GPU";
+    else if (/CUDA\d+\s*:/i.test(output)) llmSettings.backendMode = "CUDA GPU";
     else if (/Metal/i.test(output) && /GPU|device/i.test(output)) llmSettings.backendMode = "Metal GPU";
     else if (/\-\s+CPU\s+:/i.test(output) && llmSettings.backendMode.startsWith("Auto")) llmSettings.backendMode = "CPU";
     if (/error|failed/i.test(output) && !/no error/i.test(output)) {
@@ -3225,79 +3356,145 @@ const server = http.createServer(async (req, res) => {
     return json(res, 200, { ok: true });
   }
 
+function isOomError(msg) {
+  if (!msg) return false;
+  const lower = String(msg).toLowerCase();
+  return lower.includes("out of memory") ||
+         lower.includes("failed to allocate") ||
+         lower.includes("failed to initialize the context") ||
+         lower.includes("oom") ||
+         lower.includes("not enough memory") ||
+         lower.includes("allocation failed") ||
+         lower.includes("failed to create context");
+}
+
+async function retryLowerContext() {
+  const currentCtx = llmSettings.contextSize || 4096;
+  const contextLadder = [32768, 24576, 16384, 12288, 8192, 4096, 2048, 1024, 512];
+  const nextLimit = contextLadder.find(limit => limit < currentCtx);
+  if (!nextLimit) {
+    throw new Error("Cannot lower context size any further.");
+  }
+  console.log(`  [llm] Retrying with lower context limit: ${nextLimit} (was ${currentCtx})`);
+  
+  const originalModel = llmSettings.model;
+  const newSettings = {
+    model: originalModel,
+    threads: llmSettings.threads,
+    contextSize: nextLimit,
+    gpuLayers: llmSettings.gpuLayers,
+  };
+  
+  await startLlm(newSettings);
+}
+
+async function doLlmChat(req, res, body, retryCount = 0) {
+  try {
+    const isStream = body.stream === true;
+    const requestData = JSON.stringify({
+      model: llmSettings.model || "local-model",
+      messages: Array.isArray(body.messages) ? body.messages : [],
+      temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : 0.7,
+      max_tokens: Math.max(1, Math.min(4096, Number(body.max_tokens) || 512)),
+      stream: isStream,
+    });
+
+    if (isStream) {
+      const clientReq = http.request({
+        hostname: "127.0.0.1",
+        port: PORT_LLM,
+        path: "/v1/chat/completions",
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Content-Length": Buffer.byteLength(requestData),
+        }
+      }, (clientRes) => {
+        if (clientRes.statusCode < 200 || clientRes.statusCode >= 300) {
+          let errorBody = "";
+          clientRes.setEncoding("utf8");
+          clientRes.on("data", (chunk) => { errorBody += chunk; });
+          clientRes.on("end", async () => {
+            let message = `Text backend returned HTTP ${clientRes.statusCode}`;
+            try {
+              message = JSON.parse(errorBody || "{}").error?.message || message;
+            } catch (_) {}
+            
+            if (isOomError(message) && retryCount === 0) {
+              try {
+                await retryLowerContext();
+                return doLlmChat(req, res, body, retryCount + 1);
+              } catch (retryErr) {
+                console.error("Failed OOM recovery retry:", retryErr);
+              }
+            }
+            json(res, clientRes.statusCode || 500, { ok: false, error: message });
+          });
+          return;
+        }
+        res.writeHead(200, {
+          "Content-Type": "text/event-stream; charset=utf-8",
+          "Cache-Control": "no-cache, no-transform",
+          "Connection": "keep-alive",
+          "X-Accel-Buffering": "no",
+        });
+        res.flushHeaders?.();
+        clientRes.on("data", (chunk) => res.write(chunk));
+        clientRes.on("end", () => res.end());
+        clientRes.on("error", (err) => res.destroy(err));
+      });
+
+      clientReq.on("error", async (err) => {
+        console.error("LLM stream request error:", err);
+        if (isOomError(err.message) && retryCount === 0) {
+          try {
+            await retryLowerContext();
+            return doLlmChat(req, res, body, retryCount + 1);
+          } catch (retryErr) {
+            console.error("Failed OOM recovery retry:", retryErr);
+          }
+        }
+        if (!res.headersSent) {
+          res.writeHead(500, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ ok: false, error: err.message }));
+        } else {
+          res.end();
+        }
+      });
+
+      clientReq.write(requestData);
+      clientReq.end();
+      res.on("close", () => {
+        if (!res.writableEnded && !clientReq.destroyed) clientReq.destroy();
+      });
+      return;
+    } else {
+      try {
+        const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, JSON.parse(requestData), 300000);
+        return json(res, 200, result);
+      } catch (err) {
+        if (isOomError(err.message) && retryCount === 0) {
+          try {
+            await retryLowerContext();
+            return doLlmChat(req, res, body, retryCount + 1);
+          } catch (retryErr) {
+            console.error("Failed OOM recovery retry:", retryErr);
+          }
+        }
+        throw err;
+      }
+    }
+  } catch (err) {
+    return json(res, 500, { ok: false, error: err.message || String(err) });
+  }
+}
+
   if (req.url === "/api/llm/chat" && req.method === "POST") {
     const body = await readJsonBody(req, res);
     if (!body) return;
     if (!llmReady) return json(res, 409, { ok: false, error: "Load a text model before sending a message." });
-    try {
-      const isStream = body.stream === true;
-      const requestData = JSON.stringify({
-        model: llmSettings.model || "local-model",
-        messages: Array.isArray(body.messages) ? body.messages : [],
-        temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : 0.7,
-        max_tokens: Math.max(1, Math.min(4096, Number(body.max_tokens) || 512)),
-        stream: isStream,
-      });
-
-      if (isStream) {
-        const clientReq = http.request({
-          hostname: "127.0.0.1",
-          port: PORT_LLM,
-          path: "/v1/chat/completions",
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "Content-Length": Buffer.byteLength(requestData),
-          }
-        }, (clientRes) => {
-          if (clientRes.statusCode < 200 || clientRes.statusCode >= 300) {
-            let errorBody = "";
-            clientRes.setEncoding("utf8");
-            clientRes.on("data", (chunk) => { errorBody += chunk; });
-            clientRes.on("end", () => {
-              let message = `Text backend returned HTTP ${clientRes.statusCode}`;
-              try {
-                message = JSON.parse(errorBody || "{}").error?.message || message;
-              } catch (_) {}
-              json(res, clientRes.statusCode || 500, { ok: false, error: message });
-            });
-            return;
-          }
-          res.writeHead(200, {
-            "Content-Type": "text/event-stream; charset=utf-8",
-            "Cache-Control": "no-cache, no-transform",
-            "Connection": "keep-alive",
-            "X-Accel-Buffering": "no",
-          });
-          res.flushHeaders?.();
-          clientRes.on("data", (chunk) => res.write(chunk));
-          clientRes.on("end", () => res.end());
-          clientRes.on("error", (err) => res.destroy(err));
-        });
-
-        clientReq.on("error", (err) => {
-          console.error("LLM stream request error:", err);
-          if (!res.headersSent) {
-            res.writeHead(500, { "Content-Type": "application/json" });
-            res.end(JSON.stringify({ ok: false, error: err.message }));
-          } else {
-            res.end();
-          }
-        });
-
-        clientReq.write(requestData);
-        clientReq.end();
-        res.on("close", () => {
-          if (!res.writableEnded && !clientReq.destroyed) clientReq.destroy();
-        });
-        return;
-      } else {
-        const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, JSON.parse(requestData), 300000);
-        return json(res, 200, result);
-      }
-    } catch (err) {
-      return json(res, 500, { ok: false, error: err.message || String(err) });
-    }
+    await doLlmChat(req, res, body);
+    return;
   }
 
   // GET /api/hardware-specs

scripts/setup-llama.ps1

@@ -30,37 +30,23 @@ function Format-Speed {
     return "{0:N0} KB/s" -f ($bps / 1KB)
 }
 
-function Install-LlamaArchive {
-    param([string]$Variant, [string]$AssetName)
-
-    $dest = Join-Path $llmRoot $Variant
-    $server = Join-Path $dest "llama-server.exe"
-    if (Test-Path $server) {
-        Write-Host "   OK  llama.cpp $Variant backend already ready."
-        return
-    }
-
-    $archive = Join-Path $toolsDir $AssetName
-    $extract = Join-Path $toolsDir "llama-$Variant-extract"
-    $url = "https://github.com/ggml-org/llama.cpp/releases/download/$Release/$AssetName"
-
-    New-Item -ItemType Directory -Force -Path $toolsDir, $dest | Out-Null
-    Remove-Item $archive, $extract -Recurse -Force -ErrorAction SilentlyContinue
-
-    Write-Host "   >>  Downloading llama.cpp $Variant backend ($Release)..."
+function Download-File {
+    param([string]$Url, [string]$DestPath, [string]$Label)
     
     $barWidth  = 48
     $lastBytes = [long]0
     $lastTime  = [DateTime]::Now
 
+    Write-Host "   >>  Downloading $Label..."
+
     try {
         Enable-Tls12
-        $req    = [System.Net.HttpWebRequest]::Create($url)
+        $req    = [System.Net.HttpWebRequest]::Create($Url)
         $req.UserAgent = "Mozilla/5.0"
         $resp   = $req.GetResponse()
         $total  = [long]$resp.ContentLength
         $stream = $resp.GetResponseStream()
-        $out    = [System.IO.File]::Create($archive)
+        $out    = [System.IO.File]::Create($DestPath)
         $buf    = New-Object byte[] 65536
         $done   = [long]0
 
@@ -100,6 +86,26 @@ function Install-LlamaArchive {
         Write-Host ""
         throw "Download failed: $_"
     }
+}
+
+function Install-LlamaArchive {
+    param([string]$Variant, [string]$AssetName)
+
+    $dest = Join-Path $llmRoot $Variant
+    $server = Join-Path $dest "llama-server.exe"
+    if (Test-Path $server) {
+        Write-Host "   OK  llama.cpp $Variant backend already ready."
+        return
+    }
+
+    $archive = Join-Path $toolsDir $AssetName
+    $extract = Join-Path $toolsDir "llama-$Variant-extract"
+    $url = "https://github.com/ggml-org/llama.cpp/releases/download/$Release/$AssetName"
+
+    New-Item -ItemType Directory -Force -Path $toolsDir, $dest | Out-Null
+    Remove-Item $archive, $extract -Recurse -Force -ErrorAction SilentlyContinue
+
+    Download-File -Url $url -DestPath $archive -Label "llama.cpp $Variant backend ($Release)"
 
     Write-Host "   >>  Extracting llama.cpp $Variant backend..."
     Expand-Archive -Path $archive -DestinationPath $extract -Force
@@ -109,11 +115,49 @@ function Install-LlamaArchive {
     }
     Remove-Item $archive, $extract -Recurse -Force -ErrorAction SilentlyContinue
 
+    if ($Variant -eq "cuda") {
+        $cudartAsset = "cudart-llama-bin-win-cuda-12.4-x64.zip"
+        $cudartArchive = Join-Path $toolsDir $cudartAsset
+        $cudartExtract = Join-Path $toolsDir "llama-cudart-extract"
+        $cudartUrl = "https://github.com/ggml-org/llama.cpp/releases/download/$Release/$cudartAsset"
+
+        Remove-Item $cudartArchive, $cudartExtract -Recurse -Force -ErrorAction SilentlyContinue
+
+        Download-File -Url $cudartUrl -DestPath $cudartArchive -Label "llama.cpp CUDA runtime library ($Release)"
+
+        Write-Host "   >>  Extracting CUDA runtime libraries..."
+        Expand-Archive -Path $cudartArchive -DestinationPath $cudartExtract -Force
+
+        Get-ChildItem $cudartExtract -Recurse -File | ForEach-Object {
+            Copy-Item $_.FullName (Join-Path $dest $_.Name) -Force
+        }
+        Remove-Item $cudartArchive, $cudartExtract -Recurse -Force -ErrorAction SilentlyContinue
+    }
+
     if (-not (Test-Path $server)) {
         throw "llama-server.exe was not found after extracting $AssetName"
     }
     Write-Host "   OK  llama.cpp $Variant backend installed."
 }
 
+$hasNvidia = $false
+try {
+    $gpus = Get-CimInstance Win32_VideoController -ErrorAction SilentlyContinue
+    foreach ($gpu in $gpus) {
+        if ($gpu.Name -like "*NVIDIA*") {
+            $hasNvidia = $true
+        }
+    }
+} catch {}
+if (-not $hasNvidia) {
+    try {
+        & nvidia-smi *> $null
+        if ($LASTEXITCODE -eq 0) { $hasNvidia = $true }
+    } catch {}
+}
+
+if ($hasNvidia) {
+    Install-LlamaArchive -Variant "cuda" -AssetName "llama-$Release-bin-win-cuda-12.4-x64.zip"
+}
 Install-LlamaArchive -Variant "vulkan" -AssetName "llama-$Release-bin-win-vulkan-x64.zip"
 Install-LlamaArchive -Variant "cpu" -AssetName "llama-$Release-bin-win-cpu-x64.zip"

windows.bat

@@ -9,6 +9,7 @@ set DIST=%APP%\dist\index.html
 set SETUP=%~dp0scripts\setup.ps1
 set CUDA_BACKEND=%APP%\backend\win\cuda\sd-cuda.exe
 set VULKAN_BACKEND=%APP%\backend\win\vulkan\sd-vulkan.exe
+set LLM_CUDA_BACKEND=%APP%\llm-backend\win\cuda\llama-server.exe
 set LLM_VULKAN_BACKEND=%APP%\llm-backend\win\vulkan\llama-server.exe
 set LLM_CPU_BACKEND=%APP%\llm-backend\win\cpu\llama-server.exe
 set SERVE=%~dp0scripts\serve.cjs
@@ -30,7 +31,7 @@ if not exist "%DIST%" (
     set SETUP_REASON=Frontend build is missing.
     goto :run_setup
 )
-if not exist "%LLM_VULKAN_BACKEND%" if not exist "%LLM_CPU_BACKEND%" (
+if not exist "%LLM_CUDA_BACKEND%" if not exist "%LLM_VULKAN_BACKEND%" if not exist "%LLM_CPU_BACKEND%" (
     set SETUP_REASON=llama.cpp text backend is missing.
     goto :run_setup
 )