// serve.cjs — portable static file server + backend process manager // Serves app/dist/, manages sd-vulkan.exe lifecycle with correct CLI flags // serve.cjs — portable static file server + backend process manager // Serves app/dist/, manages sd-vulkan.exe lifecycle with correct CLI flags const http = require("http"); const https = require("https"); const fs = require("fs"); const net = require("net"); const os = require("os"); const path = require("path"); const { spawn, spawnSync, execSync, exec, execFile } = require("child_process"); const { comprehensiveWebSearch } = require("../search/core"); // HTTP keep-alive agent for llama-server (eliminates TCP handshake per request) const llmHttpAgent = new http.Agent({ keepAlive: true, maxSockets: 1, keepAliveMsecs: 30000, }); const HF_MODEL_CACHE_TTL_MS = 5 * 60 * 1000; const hfModelCache = new Map(); const hfProjectorCache = new Map(); function readPort(value, fallback) { const port = parseInt(value, 10); return Number.isInteger(port) && port > 0 && port < 65536 ? port : fallback; } const PORT_FRONTEND = readPort(process.env.PORT || process.env.FRONTEND_PORT, 1420); const PREFERRED_BACKEND_PORT = readPort(process.env.BACKEND_PORT || process.env.SD_BACKEND_PORT, 8080); const PREFERRED_LLM_PORT = readPort(process.env.LLM_PORT, 10086); const PREFERRED_SPEECH_PORT = readPort(process.env.SPEECH_PORT, 10088); const PREFERRED_TTS_PORT = readPort(process.env.TTS_PORT, 10089); let PORT_BACKEND = PREFERRED_BACKEND_PORT; let PORT_LLM = PREFERRED_LLM_PORT; let PORT_SPEECH = PREFERRED_SPEECH_PORT; let PORT_TTS = PREFERRED_TTS_PORT; const MAX_JSON_BODY_BYTES = 64 * 1024 * 1024; const SERVER_BUILD = "text-image-v1"; const ROOT = path.join(__dirname, "..", ".."); const DIST = path.join(ROOT, "app", "dist"); const TOOLS = path.join(ROOT, "app", "tools"); const osPlatform = process.platform; const BACKEND_PATHS = { cuda: path.join(ROOT, "app", "backend", "win", "cuda", "sd-cuda.exe"), vulkan: path.join(ROOT, "app", "backend", "win", "vulkan", "sd-vulkan.exe"), cpu: path.join(ROOT, "app", "backend", "win", "cpu", "sd-cpu.exe"), mac: path.join(ROOT, "app", "backend", "mac", "sd"), linuxCpu: path.join(ROOT, "app", "backend", "linux", "cpu", "sd-server-cpu"), linuxVulkan: path.join(ROOT, "app", "backend", "linux", "vulkan", "sd-server-vulkan"), linuxRocm: path.join(ROOT, "app", "backend", "linux", "rocm", "sd-server-rocm"), linuxCuda: path.join(ROOT, "app", "backend", "linux", "cuda", "sd-server-cuda"), }; let BACKEND_PATH = ""; const backendSupportsFlags = {}; if (osPlatform === "win32") { let hasNvidia = false; try { execSync("nvidia-smi", { stdio: "ignore" }); hasNvidia = true; } catch (_) { const commonPath = "C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe"; if (fs.existsSync(commonPath)) { hasNvidia = true; } } if (hasNvidia && fs.existsSync(BACKEND_PATHS.cuda)) { BACKEND_PATH = BACKEND_PATHS.cuda; } else if (fs.existsSync(BACKEND_PATHS.cpu)) { BACKEND_PATH = BACKEND_PATHS.cpu; } else { BACKEND_PATH = BACKEND_PATHS.vulkan; } } else if (osPlatform === "darwin") { BACKEND_PATH = BACKEND_PATHS.mac; } else { // Linux: default to Vulkan if available, otherwise CPU if (fs.existsSync(BACKEND_PATHS.linuxVulkan)) { BACKEND_PATH = BACKEND_PATHS.linuxVulkan; } else if (fs.existsSync(BACKEND_PATHS.linuxCpu)) { BACKEND_PATH = BACKEND_PATHS.linuxCpu; } else { BACKEND_PATH = BACKEND_PATHS.linuxVulkan; } } const MODELS = path.join(ROOT, "app", "models"); if (!fs.existsSync(MODELS)) { fs.mkdirSync(MODELS, { recursive: true }); } const LLM_MODELS = path.join(ROOT, "app", "llm-models"); if (!fs.existsSync(LLM_MODELS)) { fs.mkdirSync(LLM_MODELS, { recursive: true }); } const CHAT_HISTORY = path.join(ROOT, "app", "chat-history"); if (!fs.existsSync(CHAT_HISTORY)) { fs.mkdirSync(CHAT_HISTORY, { recursive: true }); } const LLM_BACKEND_PATHS = { winCuda: path.join(ROOT, "app", "llm-backend", "win", "cuda", "llama-server.exe"), winVulkan: path.join(ROOT, "app", "llm-backend", "win", "vulkan", "llama-server.exe"), winSycl: path.join(ROOT, "app", "llm-backend", "win", "sycl", "llama-server.exe"), winHip: path.join(ROOT, "app", "llm-backend", "win", "hip", "llama-server.exe"), winCpu: path.join(ROOT, "app", "llm-backend", "win", "cpu", "llama-server.exe"), linuxCuda: path.join(ROOT, "app", "llm-backend", "linux", "cuda", "llama-server"), linuxRocm: path.join(ROOT, "app", "llm-backend", "linux", "rocm", "llama-server"), linuxVulkan: path.join(ROOT, "app", "llm-backend", "linux", "vulkan", "llama-server"), linuxSycl: path.join(ROOT, "app", "llm-backend", "linux", "sycl", "llama-server"), linuxCpu: path.join(ROOT, "app", "llm-backend", "linux", "cpu", "llama-server"), macArm64: path.join(ROOT, "app", "llm-backend", "mac", "arm64", "llama-server"), macX64: path.join(ROOT, "app", "llm-backend", "mac", "x64", "llama-server"), }; const LLM_CONFIG_DIR = path.join(ROOT, "app", "config"); const LLM_MODEL_SETTINGS_PATH = path.join(LLM_CONFIG_DIR, "llm-model-settings.json"); const LLM_BENCHMARK_PATH = path.join(LLM_CONFIG_DIR, "llm-benchmarks.json"); if (!fs.existsSync(LLM_CONFIG_DIR)) { fs.mkdirSync(LLM_CONFIG_DIR, { recursive: true }); } const SPEECH_MODELS = path.join(ROOT, "app", "speech-models"); if (!fs.existsSync(SPEECH_MODELS)) { fs.mkdirSync(SPEECH_MODELS, { recursive: true }); } const TRANSCRIPTIONS = path.join(ROOT, "app", "transcriptions"); if (!fs.existsSync(TRANSCRIPTIONS)) { fs.mkdirSync(TRANSCRIPTIONS, { recursive: true }); } const TTS_MODELS = path.join(ROOT, "app", "tts-models"); if (!fs.existsSync(TTS_MODELS)) { fs.mkdirSync(TTS_MODELS, { recursive: true }); } const TTS_OUTPUTS = path.join(ROOT, "app", "tts-outputs"); if (!fs.existsSync(TTS_OUTPUTS)) { fs.mkdirSync(TTS_OUTPUTS, { recursive: true }); } const TTS_CACHE = path.join(ROOT, "app", "tts-cache"); if (!fs.existsSync(TTS_CACHE)) { fs.mkdirSync(TTS_CACHE, { recursive: true }); } const WEB_SEARCH_CACHE = path.join(ROOT, "app", "cache", "search"); if (!fs.existsSync(WEB_SEARCH_CACHE)) { fs.mkdirSync(WEB_SEARCH_CACHE, { recursive: true }); } const TTS_RUNTIME = path.join(ROOT, "app", "tts-runtime"); const TTS_WORKER = path.join(ROOT, "scripts", "workers", "tts-kokoro-worker.mjs"); const SPEECH_BACKEND_ROOT = path.join(ROOT, "app", "speech-backend"); const SPEECH_BACKEND_PATHS = { winVulkanCli: path.join(SPEECH_BACKEND_ROOT, "win", "vulkan", "whisper-cli.exe"), winVulkanServer: path.join(SPEECH_BACKEND_ROOT, "win", "vulkan", "whisper-server.exe"), winCpuCli: path.join(SPEECH_BACKEND_ROOT, "win", "cpu", "whisper-cli.exe"), winCpuServer: path.join(SPEECH_BACKEND_ROOT, "win", "cpu", "whisper-server.exe"), winCli: path.join(ROOT, "app", "speech-backend", "win", "whisper-cli.exe"), winServer: path.join(ROOT, "app", "speech-backend", "win", "whisper-server.exe"), linuxVulkanCli: path.join(SPEECH_BACKEND_ROOT, "linux", "vulkan", "whisper-cli"), linuxVulkanServer: path.join(SPEECH_BACKEND_ROOT, "linux", "vulkan", "whisper-server"), linuxCpuCli: path.join(SPEECH_BACKEND_ROOT, "linux", "cpu", "whisper-cli"), linuxCpuServer: path.join(SPEECH_BACKEND_ROOT, "linux", "cpu", "whisper-server"), linuxCli: path.join(ROOT, "app", "speech-backend", "linux", "whisper-cli"), linuxServer: path.join(ROOT, "app", "speech-backend", "linux", "whisper-server"), macMetalCli: path.join(SPEECH_BACKEND_ROOT, "mac", "metal", "whisper-cli"), macMetalServer: path.join(SPEECH_BACKEND_ROOT, "mac", "metal", "whisper-server"), macCpuCli: path.join(SPEECH_BACKEND_ROOT, "mac", "cpu", "whisper-cli"), macCpuServer: path.join(SPEECH_BACKEND_ROOT, "mac", "cpu", "whisper-server"), macCli: path.join(ROOT, "app", "speech-backend", "mac", "whisper-cli"), macServer: path.join(ROOT, "app", "speech-backend", "mac", "whisper-server"), }; const SPEECH_MODEL_CATALOG = [ { id: "tiny.en", name: "Whisper Tiny English", filename: "ggml-tiny.en.bin", size: "75 MB", language: "English", recommended: false }, { id: "tiny.en-q5_1", name: "Whisper Tiny English Q5", filename: "ggml-tiny.en-q5_1.bin", size: "31 MB", language: "English", recommended: false }, { id: "base.en", name: "Whisper Base English", filename: "ggml-base.en.bin", size: "142 MB", language: "English", recommended: true }, { id: "base.en-q5_1", name: "Whisper Base English Q5", filename: "ggml-base.en-q5_1.bin", size: "57 MB", language: "English", recommended: true }, { id: "small.en", name: "Whisper Small English", filename: "ggml-small.en.bin", size: "466 MB", language: "English", recommended: false }, { id: "small.en-q5_1", name: "Whisper Small English Q5", filename: "ggml-small.en-q5_1.bin", size: "181 MB", language: "English", recommended: false }, { id: "tiny", name: "Whisper Tiny Multilingual", filename: "ggml-tiny.bin", size: "75 MB", language: "Multilingual", recommended: false }, { id: "base", name: "Whisper Base Multilingual", filename: "ggml-base.bin", size: "142 MB", language: "Multilingual", recommended: false }, { id: "base-q5_1", name: "Whisper Base Multilingual Q5", filename: "ggml-base-q5_1.bin", size: "57 MB", language: "Multilingual", recommended: false }, { id: "small", name: "Whisper Small Multilingual", filename: "ggml-small.bin", size: "466 MB", language: "Multilingual", recommended: false }, ]; const TTS_MODEL_CATALOG = [ { id: "kokoro-onnx-q8", name: "Kokoro 82M ONNX Q8", filename: "kokoro-onnx-q8.json", modelId: "onnx-community/Kokoro-82M-v1.0-ONNX", dtype: "q8", size: "Model cache managed by kokoro-js", format: "Kokoro ONNX", recommended: true, }, { id: "kokoro-onnx-fp32", name: "Kokoro 82M ONNX FP32", filename: "kokoro-onnx-fp32.json", modelId: "onnx-community/Kokoro-82M-v1.0-ONNX", dtype: "fp32", size: "Model cache managed by kokoro-js", format: "Kokoro ONNX", recommended: false, }, ]; const TTS_VOICES = [ { id: "af_heart", name: "Heart", language: "en-us", gender: "Female", recommended: true }, { id: "af_bella", name: "Bella", language: "en-us", gender: "Female", recommended: false }, { id: "af_nicole", name: "Nicole", language: "en-us", gender: "Female", recommended: false }, { id: "af_sarah", name: "Sarah", language: "en-us", gender: "Female", recommended: false }, { id: "am_michael", name: "Michael", language: "en-us", gender: "Male", recommended: false }, { id: "am_fenrir", name: "Fenrir", language: "en-us", gender: "Male", recommended: false }, { id: "bf_emma", name: "Emma", language: "en-gb", gender: "Female", recommended: false }, { id: "bm_george", name: "George", language: "en-gb", gender: "Male", recommended: false }, ]; const OPENVINO_MODELS = path.join(ROOT, "app", "openvino-models"); if (!fs.existsSync(OPENVINO_MODELS)) { fs.mkdirSync(OPENVINO_MODELS, { recursive: true }); } const OUTPUTS = path.join(ROOT, "app", "outputs"); if (!fs.existsSync(OUTPUTS)) { fs.mkdirSync(OUTPUTS, { recursive: true }); } const OPENVINO_NPU_MODELS = [ { id: "lcm-dreamshaper-v7-fp16", name: "LCM DreamShaper v7 FP16", repo: "OpenVINO/LCM_Dreamshaper_v7-fp16-ov", folder: "LCM_Dreamshaper_v7-fp16-ov", approxSize: "2.0 GB", resolution: "512x512", notes: "OpenVINO LCM image model. Uses CPU text encoder, NPU UNet, and GPU or CPU VAE decoder.", }, ]; // ── Backend process state ───────────────────────────────────────────────────── let backendProc = null; let backendProcSeq = 0; let backendReady = false; let backendError = null; let openvinoProc = null; let openvinoReady = false; let openvinoError = null; let openvinoPort = null; let openvinoModel = null; let openvinoWidth = null; let openvinoHeight = null; let llmProc = null; let llmProcSeq = 0; let llmReady = false; let llmError = null; let llmOperationQueue = Promise.resolve(); let speechReady = false; let speechError = null; let speechOperationQueue = Promise.resolve(); let speechSettings = { model: null, language: "auto", threads: Math.max(1, Math.min(8, os.cpus().length || 4)), backendPreference: "auto", backendBinary: "", backendMode: "", }; let speechTranscriptionState = { active: false, phase: "", progress: 0, model: "", filename: "", }; let ttsReady = false; let ttsError = null; let ttsOperationQueue = Promise.resolve(); let ttsSettings = { model: null, voice: "af_heart", speed: 1, dtype: "q8", backendMode: "Kokoro ONNX", }; let ttsGenerationState = { active: false, phase: "", progress: 0, model: "", voice: "", output: "", }; function modelName(value) { return path.basename(String(value || "")); } function getActiveHeavyRuntime() { if ((backendReady || backendProc || openvinoReady || openvinoProc) && currentSettings.model) { return { type: "image", label: "Image", model: modelName(currentSettings.model) }; } if ((llmReady || llmProc) && llmSettings.model) { return { type: "text", label: "Text", model: modelName(llmSettings.model) }; } return null; } function assertNoOtherActiveRuntime(targetType, targetModel) { if (targetType !== "image" && targetType !== "text") return; const runtime = getActiveHeavyRuntime(); const targetName = modelName(targetModel); if (!runtime || (runtime.type === targetType && runtime.model === targetName)) return; const err = new Error(`"${runtime.model}" is already loaded as a ${runtime.label} model. Unload it before loading "${targetName}".`); err.statusCode = 409; err.code = "MODEL_ALREADY_ACTIVE"; err.activeRuntime = runtime; err.targetRuntime = { type: targetType, model: targetName }; throw err; } function jsonErrorStatus(err) { return Number(err?.statusCode) || (err?.code === "MODEL_ALREADY_ACTIVE" ? 409 : 500); } let llmSettings = { model: null, threads: Math.max(1, Math.min(16, os.cpus().length || 4)), contextSize: 4096, gpuLayers: -1, backendMode: "", backendBinary: "", supportsVision: false, visionMode: "none", visionStatus: "Projector not loaded", flashAttn: true, cacheTypeK: "q8_0", cacheTypeV: "q8_0", mlock: false, mmap: true, cachePrompt: true, defragThold: 0.1, batchSize: 512, ubatchSize: 512, performanceProfile: "balanced", }; function runExclusiveLlmOperation(operation) { const run = llmOperationQueue.catch(() => {}).then(operation); llmOperationQueue = run.catch(() => {}); return run; } let backendLoadState = { active: false, phase: "", progress: 0, current: 0, total: 0, speed: "", model: "", backendMode: "", backendBinary: "", device: "", }; let backendUnloadState = { active: false, phase: "", progress: 0, }; let currentSettings = { model: null, steps: 20, cfgScale: 7.0, sampler: "euler_a", threads: 8, useGpu: true, backendType: "auto", vaeTiling: true, vaeOnCpu: false, flashAttn: true, width: 512, height: 512, }; const BACKEND_RESTART_SETTING_KEYS = [ "model", "steps", "cfgScale", "sampler", "threads", "useGpu", "backendType", "vaeTiling", "vaeOnCpu", "flashAttn", "width", "height", ]; function normalizeBackendSettingValue(key, value) { if (value === undefined || value === null) return value; if (key === "model") return path.resolve(String(value)); if (["steps", "threads", "width", "height"].includes(key)) return parseInt(value); if (key === "cfgScale") return Math.round(parseFloat(value) * 1000) / 1000; if (["useGpu", "vaeTiling", "vaeOnCpu", "flashAttn"].includes(key)) return Boolean(value); return String(value); } function backendSettingsMatch(current, requested) { return BACKEND_RESTART_SETTING_KEYS.every((key) => ( normalizeBackendSettingValue(key, current[key]) === normalizeBackendSettingValue(key, requested[key]) )); } function appleNpuRuntimeMatches(current, requested) { return normalizeBackendSettingValue("model", current.model) === normalizeBackendSettingValue("model", requested.model) && current.backendType === "apple-npu" && requested.backendType === "apple-npu" && normalizeBackendSettingValue("useGpu", current.useGpu) === normalizeBackendSettingValue("useGpu", requested.useGpu) && normalizeBackendSettingValue("width", current.width || 512) === normalizeBackendSettingValue("width", requested.width || 512) && normalizeBackendSettingValue("height", current.height || 512) === normalizeBackendSettingValue("height", requested.height || 512); } function setBackendLoadStage(phase, progress, extra = {}) { if (backendReady) return; backendLoadState = { ...backendLoadState, active: true, phase, progress: Math.max(backendLoadState.progress || 0, Math.min(99, progress)), ...extra, }; } function updateCoreMLLoadProgress(output) { if (currentSettings.backendType !== "apple-npu" || backendReady) return; const cleanOutput = stripAnsi(output); if (cleanOutput.includes("Loading PyTorch reference configuration")) { setBackendLoadStage("Loading PyTorch reference configuration...", 8); } const pipelineMatch = cleanOutput.match(/Loading pipeline components.*?(\d+)%/); if (pipelineMatch) { const componentProgress = Math.max(0, Math.min(100, Number(pipelineMatch[1]) || 0)); setBackendLoadStage("Loading pipeline components...", 10 + Math.round(componentProgress * 0.25)); } if (cleanOutput.includes("Loading Core ML models from")) { setBackendLoadStage("Loading Core ML model bundle...", 40); } if (cleanOutput.includes("Loading text_encoder")) { setBackendLoadStage("Loading text encoder...", 48); } if (cleanOutput.includes("Loading unet")) { setBackendLoadStage("Loading UNet neural engine model...", 62); } if (cleanOutput.includes("Loading vae_decoder")) { setBackendLoadStage("Loading VAE decoder...", 78); } if (cleanOutput.includes("Loading safety_checker")) { setBackendLoadStage("Loading safety checker...", 88); } if (cleanOutput.includes("Initializing Core ML pipe")) { setBackendLoadStage("Initializing Core ML pipeline...", 94); } if (cleanOutput.includes("Stable Diffusion configured")) { setBackendLoadStage("Configuring image resolution...", 97); } if (cleanOutput.includes("Core ML models loaded successfully")) { setBackendLoadStage("Core ML models loaded successfully.", 99); } } let lastCpuSample = null; let cachedGpuInfo = null; let cachedBackendOptions = null; let cachedVramInfo = null; function roundGb(bytes) { return Math.round((bytes / (1024 ** 3)) * 100) / 100; } // Detect physical CPU cores (not logical/hyperthreaded cores) // llama.cpp docs: "Do NOT set threads too high — major cause of oversaturation" function getPhysicalCores() { const cpus = os.cpus(); const logicalCores = cpus.length; if (logicalCores <= 1) return 1; // Check if hyperthreading is likely: compare unique core IDs vs total // On Windows/Linux, os.cpus() lists each logical core separately // Physical cores typically have the same model but different speeds can indicate HT const uniqueModels = new Set(cpus.map(c => c.model)).size; // If we have very few unique models relative to core count, likely HT // Also check: if logical cores is exactly 2x a common physical core count const commonPhysicalCounts = [2, 4, 6, 8, 10, 12, 16, 20, 24, 32]; const isLikelyHT = commonPhysicalCounts.some(physical => logicalCores === physical * 2); if (isLikelyHT || uniqueModels < logicalCores / 2) { return Math.max(1, Math.floor(logicalCores / 2)); } return logicalCores; } // Get optimal thread count for llama.cpp // GPU mode: fewer threads to avoid CPU-GPU contention (physical/2) // CPU mode: use physical cores directly function getOptimalThreads(isGpuMode) { const physicalCores = getPhysicalCores(); if (isGpuMode) { // llama.cpp benchmark: 4 threads on 7-core CPU + GPU = 9.1 t/s (optimal) // vs 7 threads = 8.7 t/s (oversaturated) return Math.max(1, Math.floor(physicalCores / 2)); } return Math.max(1, physicalCores); } function getCpuSample() { return os.cpus().reduce((acc, cpu) => { const times = cpu.times; acc.idle += times.idle; acc.total += times.user + times.nice + times.sys + times.idle + times.irq; return acc; }, { idle: 0, total: 0 }); } function getCpuUsagePercent() { const current = getCpuSample(); if (!lastCpuSample) { lastCpuSample = current; return 0; } const idleDelta = current.idle - lastCpuSample.idle; const totalDelta = current.total - lastCpuSample.total; lastCpuSample = current; if (totalDelta <= 0) return 0; return Math.max(0, Math.min(100, Math.round((1 - idleDelta / totalDelta) * 1000) / 10)); } function hasVulkanRuntime() { if (osPlatform !== "win32") return true; try { const sysPaths = [ path.join(process.env.SystemRoot || "C:\\Windows", "System32", "vulkan-1.dll"), path.join(process.env.SystemRoot || "C:\\Windows", "SysWOW64", "vulkan-1.dll"), ]; for (const dllPath of sysPaths) { if (fs.existsSync(dllPath)) return true; } try { execSync("where vulkan-1.dll", { stdio: "ignore" }); return true; } catch (_) {} } catch (_) {} return false; } function detectLinuxGpuFromSysfs() { try { const pciDir = "/sys/bus/pci/devices"; if (!fs.existsSync(pciDir)) return null; const entries = fs.readdirSync(pciDir); for (const entry of entries) { const vendorFile = path.join(pciDir, entry, "vendor"); const deviceFile = path.join(pciDir, entry, "device"); if (!fs.existsSync(vendorFile)) continue; const vendor = fs.readFileSync(vendorFile, "utf8").trim(); let device = ""; try { device = fs.readFileSync(deviceFile, "utf8").trim(); } catch (_) {} if (vendor === "0x10de") { return { name: `NVIDIA GPU (${device || entry})` }; } if (vendor === "0x1002") { return { name: `AMD GPU (${device || entry})` }; } if (vendor === "0x8086") { return { name: `Intel GPU (${device || entry})` }; } } } catch (_) {} return null; } function isVirtualOrSoftwareGpu(name) { const lowercase = String(name || "").toLowerCase(); return lowercase.includes("virtual desktop") || lowercase.includes("remote display") || lowercase.includes("microsoft basic render") || lowercase.includes("citrix") || lowercase.includes("software rasterizer") || lowercase.includes("virtualbox") || lowercase.includes("vmware"); } let cachedPreferredVulkanBackend = null; let cachedPreferredVulkanBackendChecked = false; function getPreferredVulkanBackendName() { const explicit = String(process.env.SD_VULKAN_DEVICE || process.env.UAIS_VULKAN_DEVICE || "").trim(); if (/^vulkan\d+$/i.test(explicit)) return explicit.toLowerCase(); if (/^\d+$/.test(explicit)) return `vulkan${explicit}`; if (cachedPreferredVulkanBackendChecked) return cachedPreferredVulkanBackend || "vulkan0"; cachedPreferredVulkanBackendChecked = true; cachedPreferredVulkanBackend = "vulkan0"; if (osPlatform !== "linux") return cachedPreferredVulkanBackend; try { const result = spawnSync("vulkaninfo", ["--summary"], { encoding: "utf8", timeout: 5000, maxBuffer: 1024 * 1024, }); const output = `${result.stdout || ""}\n${result.stderr || ""}`; const devices = []; let current = null; for (const line of output.split(/\r?\n/)) { const gpuMatch = line.match(/^\s*GPU(\d+)\s*:/i); if (gpuMatch) { current = { index: Number(gpuMatch[1]), name: "", type: "" }; devices.push(current); continue; } if (!current) continue; const nameMatch = line.match(/deviceName\s*=\s*(.+)$/i); if (nameMatch) current.name = nameMatch[1].trim(); const typeMatch = line.match(/deviceType\s*=\s*(.+)$/i); if (typeMatch) current.type = typeMatch[1].trim(); } const usable = devices.filter((device) => Number.isInteger(device.index) && !isVirtualOrSoftwareGpu(device.name)); const discrete = usable.find((device) => /discrete/i.test(device.type)); const amd = usable.find((device) => /amd|radeon/i.test(device.name)); const selected = discrete || amd || usable[0]; if (selected) cachedPreferredVulkanBackend = `vulkan${selected.index}`; } catch (_) {} return cachedPreferredVulkanBackend; } let cachedLlamaCudaGpu = null; let cachedLlamaCudaGpuChecked = false; function detectLlamaCudaGpu() { if (cachedLlamaCudaGpuChecked) return cachedLlamaCudaGpu; cachedLlamaCudaGpuChecked = true; const backendPath = osPlatform === "win32" ? LLM_BACKEND_PATHS.winCuda : osPlatform === "linux" ? LLM_BACKEND_PATHS.linuxCuda : ""; if (!backendPath || !fs.existsSync(backendPath)) return null; try { const result = spawnSync(backendPath, ["--list-devices"], { encoding: "utf8", windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024, }); const output = `${result.stdout || ""}\n${result.stderr || ""}`; const devices = []; const devicePattern = /CUDA\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; let match; while ((match = devicePattern.exec(output)) !== null) { devices.push({ name: match[1].trim(), vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100, free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null, }); } let activeDevices = devices.filter(d => !isVirtualOrSoftwareGpu(d.name)); if (activeDevices.length === 0) { activeDevices = devices; } cachedLlamaCudaGpu = activeDevices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null; } catch (_) { cachedLlamaCudaGpu = null; } return cachedLlamaCudaGpu; } let cachedLlamaVulkanGpu = null; let cachedLlamaVulkanGpuChecked = false; function detectLlamaVulkanGpu() { if (cachedLlamaVulkanGpuChecked) return cachedLlamaVulkanGpu; cachedLlamaVulkanGpuChecked = true; const backendPath = osPlatform === "win32" ? LLM_BACKEND_PATHS.winVulkan : osPlatform === "linux" ? LLM_BACKEND_PATHS.linuxVulkan : ""; if (!backendPath || !fs.existsSync(backendPath)) return null; try { const result = spawnSync(backendPath, ["--list-devices"], { encoding: "utf8", windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024, }); const output = `${result.stdout || ""}\n${result.stderr || ""}`; const devices = []; const devicePattern = /Vulkan\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; let match; while ((match = devicePattern.exec(output)) !== null) { devices.push({ name: match[1].trim(), vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100, free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null, }); } let activeDevices = devices.filter(d => !isVirtualOrSoftwareGpu(d.name)); if (activeDevices.length === 0) { activeDevices = devices; } cachedLlamaVulkanGpu = activeDevices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null; } catch (_) { cachedLlamaVulkanGpu = null; } return cachedLlamaVulkanGpu; } let cachedLlamaSyclGpu = null; let cachedLlamaSyclGpuChecked = false; function detectLlamaSyclGpu() { if (cachedLlamaSyclGpuChecked) return cachedLlamaSyclGpu; cachedLlamaSyclGpuChecked = true; const backendPath = osPlatform === "win32" ? LLM_BACKEND_PATHS.winSycl : osPlatform === "linux" ? LLM_BACKEND_PATHS.linuxSycl : ""; if (!backendPath || !fs.existsSync(backendPath)) return null; try { const result = spawnSync(backendPath, ["--list-devices"], { encoding: "utf8", windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024, }); const output = `${result.stdout || ""}\n${result.stderr || ""}`; const devices = []; const devicePattern = /SYCL\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; let match; while ((match = devicePattern.exec(output)) !== null) { devices.push({ name: match[1].trim(), vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100, free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null, }); } cachedLlamaSyclGpu = devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null; } catch (_) { cachedLlamaSyclGpu = null; } return cachedLlamaSyclGpu; } function getGpuInfo() { if (cachedGpuInfo) return cachedGpuInfo; if (osPlatform === "win32" || osPlatform === "linux") { const llamaCudaGpu = detectLlamaCudaGpu(); if (llamaCudaGpu) { cachedGpuInfo = llamaCudaGpu; return cachedGpuInfo; } const llamaVulkanGpu = detectLlamaVulkanGpu(); if (llamaVulkanGpu) { cachedGpuInfo = llamaVulkanGpu; return cachedGpuInfo; } try { const stdout = execSync( 'powershell -NoProfile -Command "Get-CimInstance Win32_VideoController | ForEach-Object { [PSCustomObject]@{ Name = $_.Name; VRAM = [math]::Round($_.AdapterRAM / 1GB, 2) } } | ConvertTo-Json"', { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] } ).trim(); if (stdout) { let gpus = []; try { const parsed = JSON.parse(stdout); if (Array.isArray(parsed)) { gpus = parsed; } else if (parsed && typeof parsed === "object") { gpus = [parsed]; } } catch (_) {} if (gpus.length > 0) { const discreteKeywords = ["nvidia", "amd", "arc", "geforce", "radeon", "rtx", "gtx"]; let activeGpus = gpus.filter(gpu => !isVirtualOrSoftwareGpu(gpu.Name)); if (activeGpus.length === 0) { activeGpus = gpus; } let selectedGpu = activeGpus.find(gpu => { const name = String(gpu.Name || "").toLowerCase(); return discreteKeywords.some(kw => name.includes(kw)); }); if (!selectedGpu) { selectedGpu = activeGpus[0]; } cachedGpuInfo = { name: selectedGpu.Name || "Unknown GPU", vram_gb: typeof selectedGpu.VRAM === "number" ? selectedGpu.VRAM : 0 }; return cachedGpuInfo; } } } catch (_) {} // Fallback if Powershell fails or returns empty try { const output = execSync( "powershell -NoProfile -Command \"Get-CimInstance Win32_VideoController | Select-Object -First 1 -ExpandProperty Name\"", { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, ).trim(); if (output) { cachedGpuInfo = { name: output, vram_gb: 0 }; return cachedGpuInfo; } } catch (_) {} } if (osPlatform === "darwin") { try { const output = execSync("system_profiler SPDisplaysDataType", { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], }); const lines = output.split("\n"); let currentGpu = null; const gpus = []; for (const line of lines) { const trimmed = line.trim(); if (trimmed.startsWith("Chipset Model:")) { if (currentGpu) { gpus.push(currentGpu); } currentGpu = { name: trimmed.substring("Chipset Model:".length).trim(), vram_gb: 0, }; } else if (currentGpu && (trimmed.startsWith("VRAM (Total):") || trimmed.startsWith("VRAM (Dynamic, Max):"))) { const valStr = trimmed.split(":")[1].trim(); // e.g. "2 GB" or "1536 MB" let bytesVal = 0; const numMatch = valStr.match(/^([\d\.]+)\s*(GB|MB)/i); if (numMatch) { const val = parseFloat(numMatch[1]); const unit = numMatch[2].toUpperCase(); if (unit === "GB") { bytesVal = val; } else if (unit === "MB") { bytesVal = val / 1024; } } currentGpu.vram_gb = Math.round(bytesVal * 100) / 100; } } if (currentGpu) { gpus.push(currentGpu); } if (gpus.length > 0) { const isAppleSilicon = gpus.some(g => { const name = g.name.toLowerCase(); return name.includes("apple") || name.includes("m1") || name.includes("m2") || name.includes("m3") || name.includes("m4"); }) || (os.arch() === "arm64"); if (isAppleSilicon) { const totalRamGb = Math.round((os.totalmem() / (1024 ** 3)) * 100) / 100; let selected = gpus.find(g => g.name.toLowerCase().includes("apple")) || gpus[0]; cachedGpuInfo = { name: selected.name, vram_gb: totalRamGb }; return cachedGpuInfo; } else { const discreteKeywords = ["nvidia", "amd", "radeon", "geforce"]; let selected = gpus.find(gpu => { const name = gpu.name.toLowerCase(); return discreteKeywords.some(kw => name.includes(kw)); }); if (!selected) { selected = gpus[0]; } cachedGpuInfo = { name: selected.name, vram_gb: selected.vram_gb }; return cachedGpuInfo; } } } catch (_) {} // Fallback if system_profiler command fails const totalRamGb = Math.round((os.totalmem() / (1024 ** 3)) * 100) / 100; cachedGpuInfo = { name: "Apple Metal GPU", vram_gb: totalRamGb }; return cachedGpuInfo; } if (osPlatform === "linux") { // AMD cards: Scan /sys/class/drm/card*/device/vendor and mem_info_vram_total try { if (fs.existsSync("/sys/class/drm")) { const cards = fs.readdirSync("/sys/class/drm").filter(name => name.startsWith("card")); for (const card of cards) { const vendorPath = `/sys/class/drm/${card}/device/vendor`; const vramPath = `/sys/class/drm/${card}/device/mem_info_vram_total`; if (fs.existsSync(vendorPath) && fs.existsSync(vramPath)) { const vendor = fs.readFileSync(vendorPath, "utf8").trim(); if (vendor === "0x1002") { const vramBytesStr = fs.readFileSync(vramPath, "utf8").trim(); const vramBytes = parseFloat(vramBytesStr); if (!isNaN(vramBytes)) { cachedGpuInfo = { name: "AMD Radeon GPU", vram_gb: Math.round((vramBytes / (1024 ** 3)) * 100) / 100 }; return cachedGpuInfo; } } } } } } catch (_) {} // NVIDIA cards: read /proc/driver/nvidia/gpus/*/information or run nvidia-smi try { if (fs.existsSync("/proc/driver/nvidia/gpus")) { const gpuDirs = fs.readdirSync("/proc/driver/nvidia/gpus"); for (const gpuDir of gpuDirs) { const infoPath = `/proc/driver/nvidia/gpus/${gpuDir}/information`; if (fs.existsSync(infoPath)) { const info = fs.readFileSync(infoPath, "utf8"); const modelMatch = info.match(/Model:\s+(.*)/); if (modelMatch) { const modelName = modelMatch[1].trim(); let vram_gb = 0; try { const smiOut = execSync("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits", { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); const parsedVal = parseFloat(smiOut); if (!isNaN(parsedVal)) { vram_gb = Math.round((parsedVal / 1024) * 100) / 100; } } catch (_) {} cachedGpuInfo = { name: modelName, vram_gb: vram_gb || 8.0 }; return cachedGpuInfo; } } } } } catch (_) {} try { const smiOut = execSync("nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits", { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); if (smiOut) { const lines = smiOut.split("\n"); if (lines[0]) { const [name, totalMb] = lines[0].split(",").map(p => p.trim()); const parsedVal = parseFloat(totalMb); cachedGpuInfo = { name: name || "NVIDIA GPU", vram_gb: !isNaN(parsedVal) ? Math.round((parsedVal / 1024) * 100) / 100 : 8.0 }; return cachedGpuInfo; } } } catch (_) {} const linuxGpu = detectLinuxGpuFromSysfs(); if (linuxGpu) { cachedGpuInfo = { name: linuxGpu.name, vram_gb: 0 }; return cachedGpuInfo; } } cachedGpuInfo = { name: "Unavailable", vram_gb: 0 }; return cachedGpuInfo; } // ── NPU Detection ───────────────────────────────────────────────────────────── let cachedNpuInfo = null; let cachedHasNpuHardware = null; function hasNpuHardware() { if (cachedHasNpuHardware !== null) return cachedHasNpuHardware; if (osPlatform !== "win32" && osPlatform !== "linux") { cachedHasNpuHardware = false; return cachedHasNpuHardware; } try { if (osPlatform === "win32") { const deviceResult = spawnSync("wmic", [ "path", "win32_pnpentity", "where", "Name like '%NPU%' or Name like '%Intel%AI%' or Name like '%VPU%'", "get", "Name", "/value" ], { encoding: "utf8", timeout: 10000, stdio: ["ignore", "pipe", "pipe"] }); if (deviceResult.status === 0 && deviceResult.stdout && /NPU|AI Boost|VPU/i.test(deviceResult.stdout)) { cachedHasNpuHardware = true; return cachedHasNpuHardware; } const cpuResult = spawnSync("wmic", ["cpu", "get", "Name", "/value"], { encoding: "utf8", timeout: 10000, stdio: ["ignore", "pipe", "pipe"], }); if (cpuResult.status === 0 && cpuResult.stdout) { const cpuName = cpuResult.stdout.toLowerCase(); if (cpuName.includes("ultra") && (cpuName.match(/ultra\s*\d/) || cpuName.includes("core(tm) ultra"))) { cachedHasNpuHardware = true; return cachedHasNpuHardware; } } } else if (osPlatform === "linux") { const lspciResult = spawnSync("lspci", [], { encoding: "utf8", timeout: 10000, stdio: ["ignore", "pipe", "pipe"] }); if (lspciResult.status === 0 && lspciResult.stdout && lspciResult.stdout.toLowerCase().includes("npu")) { cachedHasNpuHardware = true; return cachedHasNpuHardware; } try { const sysPci = fs.readdirSync("/sys/bus/pci/devices"); for (const dev of sysPci) { const vendorPath = `/sys/bus/pci/devices/${dev}/vendor`; const classPath = `/sys/bus/pci/devices/${dev}/class`; if (!fs.existsSync(vendorPath) || !fs.existsSync(classPath)) continue; const vendor = fs.readFileSync(vendorPath, "utf8").trim(); const devClass = fs.readFileSync(classPath, "utf8").trim(); if (vendor === "0x8086" && (devClass.includes("1180") || devClass.includes("acce"))) { cachedHasNpuHardware = true; return cachedHasNpuHardware; } } } catch (_) {} try { const cpuInfo = fs.readFileSync("/proc/cpuinfo", "utf8").toLowerCase(); if (cpuInfo.includes("ultra") && cpuInfo.includes("intel")) { cachedHasNpuHardware = true; return cachedHasNpuHardware; } } catch (_) {} } } catch (_) {} cachedHasNpuHardware = false; return cachedHasNpuHardware; } function detectNpu() { if (cachedNpuInfo !== null) return cachedNpuInfo; cachedNpuInfo = { detected: false, vendor: null, name: null }; if (osPlatform === "win32") { try { const output = execSync( 'powershell -NoProfile -Command "Get-PnpDevice | Where-Object { $_.FriendlyName -match \'Intel.*NPU\' -or $_.FriendlyName -match \'Neural\' } | Select-Object -First 1 FriendlyName"', { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 } ).trim(); if (output && output.includes("NPU")) { cachedNpuInfo = { detected: true, vendor: "intel", name: output.trim() }; } } catch (_) {} } else if (osPlatform === "linux") { try { const output = execSync("lspci | grep -i 'neural\|npu' || true", { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }).trim(); if (output) { const isIntel = output.toLowerCase().includes("intel"); cachedNpuInfo = { detected: true, vendor: isIntel ? "intel" : "unknown", name: output.split("\n")[0].trim() }; } } catch (_) {} } // macOS: Apple Silicon Neural Engine is part of the SoC, not a separate NPU device return cachedNpuInfo; } // ── GPU Layer Auto-Tuning ───────────────────────────────────────────────────── // Calculate optimal GPU layer count from free VRAM to prevent OOM function chooseAutoLayers(modelFilename, freeVramBytes, cacheTypeK = "q8_0", cacheTypeV = "q8_0") { let modelSizeGb = 4; let layerCount = 32; try { const modelPath = path.join(LLM_MODELS, modelFilename); if (fs.existsSync(modelPath)) { const stats = fs.statSync(modelPath); modelSizeGb = stats.size / (1024 * 1024 * 1024); } } catch (_) {} // Estimate layer count from model size (rough heuristic based on common architectures) if (modelSizeGb < 2) layerCount = 24; // Small models (1-2B) else if (modelSizeGb < 4) layerCount = 32; // Medium (3-4B) else if (modelSizeGb < 8) layerCount = 40; // Large (7-8B) else if (modelSizeGb < 20) layerCount = 80; // Very large (13-20B) else if (modelSizeGb < 40) layerCount = 100; // Huge (30-40B) else layerCount = 120; // Massive (70B+) // Calculate approximate layer size const layerSizeGb = modelSizeGb / layerCount; // Account for KV cache memory with quantization // q8_0 = 0.5x, q4_0 = 0.25x of f16 const kvCacheMultiplier = (cacheTypeK === "q4_0" || cacheTypeV === "q4_0") ? 0.25 : (cacheTypeK === "q8_0" || cacheTypeV === "q8_0") ? 0.5 : 1.0; // Leave 15% VRAM headroom for KV cache + overhead + OS const usableVram = freeVramBytes * 0.85; const maxLayers = Math.floor(usableVram / (layerSizeGb * 1024 * 1024 * 1024)); // Cap at total layers, return -1 if fits fully if (maxLayers >= layerCount) return -1; // All layers fit return Math.max(0, maxLayers); } function getGgufModelSizeGb(modelFilename) { try { const modelPath = path.join(LLM_MODELS, modelFilename); if (fs.existsSync(modelPath)) { return fs.statSync(modelPath).size / (1024 * 1024 * 1024); } } catch (_) {} return 4; } function getStableSyclGpuLayers(modelFilename, requestedLayers) { if (process.env.LOCALAI_LLM_ALLOW_FULL_SYCL === "1") return requestedLayers; if (requestedLayers !== -1) return requestedLayers; const syclGpu = detectLlamaSyclGpu(); const vramGb = syclGpu?.free_vram_gb || syclGpu?.vram_gb || 0; const modelSizeGb = getGgufModelSizeGb(modelFilename); if (vramGb > 0 && vramGb <= 8.5 && modelSizeGb >= 2.5) { return 0; } if (vramGb > 0 && vramGb <= 12 && modelSizeGb >= 5) { return 0; } return requestedLayers; } let nvidiaSmiCmd = "nvidia-smi"; let hasNvidiaSmi = null; if (osPlatform === "win32") { const commonPath = "C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe"; if (fs.existsSync(commonPath)) { nvidiaSmiCmd = `"${commonPath}"`; } } let isPollingVram = false; let lastVramPollTime = 0; function pollNvidiaVram(force = false) { if (isPollingVram) return; if (hasNvidiaSmi === false) return; const now = Date.now(); // If not forced, skip if backend is running (driver queries conflict & cause display lag), // or if it has been polled too recently if (!force) { if (backendProc !== null) { return; } if (now - lastVramPollTime < 4500) { return; } } isPollingVram = true; lastVramPollTime = now; const cmd = `${nvidiaSmiCmd} --query-gpu=name,memory.used,memory.total --format=csv,noheader,nounits`; exec( cmd, { stdio: ["ignore", "pipe", "ignore"] }, (error, stdout) => { isPollingVram = false; if (error) { // Only permanently disable if we are not currently running a backend workload if (backendProc === null) { hasNvidiaSmi = false; } cachedVramInfo = null; return; } hasNvidiaSmi = true; const output = (stdout || "").trim(); const firstLine = output.split(/\r?\n/)[0]; if (!firstLine) { cachedVramInfo = null; return; } const [name, usedMb, totalMb] = firstLine.split(",").map(part => part.trim()); cachedVramInfo = { gpu_name: name || getGpuInfo().name, vram_used_gb: Math.round((parseFloat(usedMb) / 1024) * 100) / 100, vram_total_gb: Math.round((parseFloat(totalMb) / 1024) * 100) / 100, }; } ); } // Start background VRAM polling (runs every 5 seconds, only active when backend is idle) setInterval(() => pollNvidiaVram(false), 5000); pollNvidiaVram(true); function getNvidiaVram() { return cachedVramInfo; } let cachedLlamaVramInfo = null; let isPollingLlamaVram = false; let lastLlamaVramPollTime = 0; function parseLlamaDeviceMemory(output) { const devices = []; const devicePattern = /(CUDA|Vulkan|SYCL)\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; let match; while ((match = devicePattern.exec(output || "")) !== null) { const totalMiB = Number(match[3]); const freeMiB = match[4] ? Number(match[4]) : NaN; if (!Number.isFinite(totalMiB) || totalMiB <= 0) continue; const usedMiB = Number.isFinite(freeMiB) ? Math.max(0, totalMiB - freeMiB) : 0; devices.push({ gpu_name: match[2].trim(), vram_used_gb: Math.round((usedMiB / 1024) * 100) / 100, vram_total_gb: Math.round((totalMiB / 1024) * 100) / 100, }); } return devices.sort((a, b) => b.vram_total_gb - a.vram_total_gb)[0] || null; } function getLlamaTelemetryBackendPath() { if (osPlatform === "win32") { if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) return LLM_BACKEND_PATHS.winVulkan; if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) return LLM_BACKEND_PATHS.winSycl; if (fs.existsSync(LLM_BACKEND_PATHS.winHip)) return LLM_BACKEND_PATHS.winHip; if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) return LLM_BACKEND_PATHS.winCuda; } else if (osPlatform === "linux") { if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) return LLM_BACKEND_PATHS.linuxVulkan; if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl)) return LLM_BACKEND_PATHS.linuxSycl; if (fs.existsSync(LLM_BACKEND_PATHS.linuxRocm)) return LLM_BACKEND_PATHS.linuxRocm; if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) return LLM_BACKEND_PATHS.linuxCuda; } return null; } function pollLlamaVram(force = false) { if (isPollingLlamaVram) return; const now = Date.now(); if (!force && now - lastLlamaVramPollTime < 4500) { return; } const backendPath = getLlamaTelemetryBackendPath(); if (!backendPath) return; isPollingLlamaVram = true; lastLlamaVramPollTime = now; execFile( backendPath, ["--list-devices"], { windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024 }, (error, stdout, stderr) => { isPollingLlamaVram = false; if (error) { cachedLlamaVramInfo = null; return; } cachedLlamaVramInfo = parseLlamaDeviceMemory(`${stdout || ""}\n${stderr || ""}`); } ); } setInterval(() => pollLlamaVram(false), 5000); pollLlamaVram(true); function getLlamaVram() { return cachedLlamaVramInfo; } function readJsonFile(filePath, fallback) { try { if (!fs.existsSync(filePath)) return fallback; return JSON.parse(fs.readFileSync(filePath, "utf8")); } catch (_) { return fallback; } } function writeJsonFile(filePath, value) { fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.writeFileSync(filePath, JSON.stringify(value, null, 2) + "\n", "utf8"); } function getPersistedLlmModelSettings() { return readJsonFile(LLM_MODEL_SETTINGS_PATH, { models: {} }); } function savePersistedLlmModelSettings(settings) { writeJsonFile(LLM_MODEL_SETTINGS_PATH, settings && typeof settings === "object" ? settings : { models: {} }); } function getLlmModelSettings(filename) { const safeFilename = path.basename(String(filename || "")); if (!safeFilename) return {}; const settings = getPersistedLlmModelSettings(); return settings.models?.[safeFilename] || {}; } function updateLlmModelSettings(filename, patch) { const safeFilename = path.basename(String(filename || "")); if (!safeFilename) return {}; const allSettings = getPersistedLlmModelSettings(); if (!allSettings.models || typeof allSettings.models !== "object") allSettings.models = {}; const previous = allSettings.models[safeFilename] || {}; const next = { ...previous, ...patch, updatedAt: new Date().toISOString(), }; allSettings.models[safeFilename] = next; savePersistedLlmModelSettings(allSettings); return next; } function getPersistedLlmBenchmarks() { return readJsonFile(LLM_BENCHMARK_PATH, { results: [] }); } function savePersistedLlmBenchmarks(benchmarks) { writeJsonFile(LLM_BENCHMARK_PATH, benchmarks && typeof benchmarks === "object" ? benchmarks : { results: [] }); } function recordLlmBenchmark(result) { const benchmarks = getPersistedLlmBenchmarks(); const results = Array.isArray(benchmarks.results) ? benchmarks.results : []; results.unshift({ ...result, createdAt: new Date().toISOString(), }); benchmarks.results = results.slice(0, 100); savePersistedLlmBenchmarks(benchmarks); return benchmarks.results[0]; } function parseLlamaDeviceList(output) { const devices = []; const devicePattern = /(CUDA|Vulkan|SYCL|HIP|ROCm)\d*\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi; let match; while ((match = devicePattern.exec(output || "")) !== null) { const totalMiB = Number(match[3]); const freeMiB = match[4] ? Number(match[4]) : NaN; if (!Number.isFinite(totalMiB) || totalMiB <= 0) continue; devices.push({ type: match[1], name: match[2].trim(), vram_gb: Math.round((totalMiB / 1024) * 100) / 100, free_vram_gb: Number.isFinite(freeMiB) ? Math.round((freeMiB / 1024) * 100) / 100 : null, }); } return devices.sort((a, b) => b.vram_gb - a.vram_gb); } let cachedMacRamUsedGb = null; function pollMacRam() { if (osPlatform !== "darwin") return; exec("vm_stat", (err, stdout) => { if (err) return; try { const vmStat = stdout.toString(); const pageSizeMatch = vmStat.match(/page size of (\d+) bytes/); const pageSize = pageSizeMatch ? parseInt(pageSizeMatch[1], 10) : 4096; const getVal = (key) => { const match = vmStat.match(new RegExp(key + ":\\s+(\\d+)\\.")); return match ? parseInt(match[1], 10) : 0; }; const freePages = getVal("Pages free"); const inactivePages = getVal("Pages inactive"); const speculativePages = getVal("Pages speculative"); const totalFreeBytes = (freePages + inactivePages + speculativePages) * pageSize; cachedMacRamUsedGb = roundGb(os.totalmem() - totalFreeBytes); } catch (e) { // Ignore parsing errors } }); } if (osPlatform === "darwin") { pollMacRam(); setInterval(pollMacRam, 5000); } function getHardwareSpecs() { const cpus = os.cpus(); const gpu = getGpuInfo(); const ramTotalGb = roundGb(os.totalmem()); const gpuVramGb = gpu.vram_gb || 0; const physicalCores = getPhysicalCores(); const logicalCores = cpus.length; const npuInfo = detectNpu(); // Tiering logic let tier = "low"; let tierReason = ""; const isAppleSilicon = osPlatform === "darwin" && (os.arch() === "arm64" || String(cpus[0]?.model || "").toLowerCase().includes("apple")); if (isAppleSilicon) { if (ramTotalGb >= 16) { tier = "high"; tierReason = `Apple Silicon Mac with ${ramTotalGb} GB unified memory.`; } else if (ramTotalGb >= 8) { tier = "mid"; tierReason = `Apple Silicon Mac with ${ramTotalGb} GB unified memory.`; } else { tier = "low"; tierReason = `Apple Silicon Mac with ${ramTotalGb} GB unified memory.`; } } else { if (gpuVramGb >= 12) { tier = "high"; tierReason = `Discrete GPU with ${gpuVramGb} GB VRAM.`; } else if (gpuVramGb >= 6) { tier = "mid"; tierReason = `GPU with ${gpuVramGb} GB VRAM.`; } else { tier = "low"; tierReason = `GPU with ${gpuVramGb} GB VRAM and ${ramTotalGb} GB RAM.`; } } // Recommended text settings per tier const recommendedTextSettings = { low: { contextSize: 2048, threads: Math.max(1, physicalCores), gpuLayers: 0, cacheTypeK: "q4_0", cacheTypeV: "q4_0", flashAttn: true, mlock: true, mmap: true, cachePrompt: true, batchSize: 256, ubatchSize: 256, performanceProfile: "potato", }, mid: { contextSize: 4096, threads: Math.max(1, Math.floor(physicalCores / 2)), gpuLayers: -1, cacheTypeK: "q8_0", cacheTypeV: "q8_0", flashAttn: true, mlock: false, mmap: true, cachePrompt: true, batchSize: 512, ubatchSize: 512, performanceProfile: "balanced", }, high: { contextSize: 8192, threads: Math.max(1, Math.floor(physicalCores / 2)), gpuLayers: -1, cacheTypeK: "q8_0", cacheTypeV: "q8_0", flashAttn: true, mlock: false, mmap: true, cachePrompt: true, batchSize: 1024, ubatchSize: 1024, performanceProfile: "high", }, }; // Recommended models catalog matching contract exactly let recommended_models = []; if (tier === "high") { recommended_models = [ { name: "Llama 3 8B Instruct (Q8_0)", filename: "Meta-Llama-3-8B-Instruct-Q8_0.gguf", approxSize: "8.5 GB", url: "https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q8_0.gguf", notes: "Near-lossless precision for high quality responses.", reason: tierReason }, { name: "Mistral 7B Instruct v0.3 (Q8_0)", filename: "Mistral-7B-Instruct-v0.3.Q8_0.gguf", approxSize: "7.7 GB", url: "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3.Q8_0.gguf", notes: "Superb instruction-following and context window size.", reason: tierReason } ]; } else if (tier === "mid") { recommended_models = [ { name: "Llama 3 8B Instruct (Q4_K_M)", filename: "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf", approxSize: "4.8 GB", url: "https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf", notes: "Great balance of speed and performance.", reason: tierReason }, { name: "Phi-3 Mini Instruct (Q4_K_M)", filename: "Phi-3-mini-4k-instruct-q4.gguf", approxSize: "2.2 GB", url: "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf", notes: "Lightweight and fast, optimized by Microsoft.", reason: tierReason } ]; } else { recommended_models = [ { name: "Qwen 2.5 Coder 0.5B Instruct (Q4_K_M)", filename: "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf", approxSize: "491 MB", url: "https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF/resolve/main/qwen2.5-coder-0.5b-instruct-q4_k_m.gguf", notes: "Extremely fast, lightweight assistant, perfect for low RAM/VRAM machines.", reason: tierReason }, { name: "SmolLM2 1.7B Instruct (Q4_K_M)", filename: "smollm2-1.7b-instruct-q4_k_m.gguf", approxSize: "1.1 GB", url: "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/main/smollm2-1.7b-instruct-q4_k_m.gguf", notes: "Excellent lightweight assistant with strong logic, reasoning, and prompt expansion capabilities.", reason: tierReason } ]; } return { os_name: `${os.type()} ${os.release()}`, cpu_name: cpus[0]?.model || "Unavailable", cpu_cores_physical: physicalCores, cpu_cores_logical: logicalCores, ram_total_gb: ramTotalGb, gpu_name: gpu.name, gpu_vram_gb: gpuVramGb, npu: npuInfo, tier, recommended_models, recommended_text_settings: recommendedTextSettings[tier] || recommendedTextSettings.mid, }; } function getTelemetry() { const vram = getNvidiaVram(); const llamaVram = vram || getLlamaVram(); let ram_used_gb = roundGb(os.totalmem() - os.freemem()); if (osPlatform === "darwin" && cachedMacRamUsedGb !== null) { ram_used_gb = cachedMacRamUsedGb; } const gpu = getGpuInfo(); return { cpu_usage: getCpuUsagePercent(), ram_used_gb, ram_total_gb: roundGb(os.totalmem()), gpu_name: llamaVram?.gpu_name || gpu.name, vram_used_gb: Number.isFinite(Number(llamaVram?.vram_used_gb)) ? llamaVram.vram_used_gb : 0, vram_total_gb: llamaVram?.vram_total_gb || gpu.vram_gb || 0, }; } function formatBytes(bytes) { const value = Number(bytes) || 0; if (value <= 0) return "0 B"; const units = ["B", "KB", "MB", "GB", "TB"]; const idx = Math.min(units.length - 1, Math.floor(Math.log(value) / Math.log(1024))); return `${(value / (1024 ** idx)).toFixed(idx === 0 ? 0 : 1)} ${units[idx]}`; } function getPathInfo(label, targetPath, type = "file") { const exists = fs.existsSync(targetPath); return { label, path: targetPath, type, exists, ok: exists, }; } function isDirWritable(dirPath) { try { fs.mkdirSync(dirPath, { recursive: true }); const testFile = path.join(dirPath, `.uncensored-ai-studio-write-test-${Date.now()}.tmp`); fs.writeFileSync(testFile, "ok", "utf8"); fs.unlinkSync(testFile); return true; } catch (_) { return false; } } function getDirInfo(label, targetPath) { const exists = fs.existsSync(targetPath); const writable = exists ? isDirWritable(targetPath) : false; return { label, path: targetPath, type: "directory", exists, writable, ok: exists && writable, }; } function getPathSize(targetPath) { try { if (!fs.existsSync(targetPath)) return 0; const stat = fs.statSync(targetPath); if (stat.isFile()) return stat.size; if (!stat.isDirectory()) return 0; return fs.readdirSync(targetPath).reduce((sum, name) => sum + getPathSize(path.join(targetPath, name)), 0); } catch (_) { return 0; } } function getOpenVinoPythonCandidates() { const candidates = []; if (process.env.OPENVINO_PYTHON) candidates.push(process.env.OPENVINO_PYTHON); if (osPlatform === "win32") { candidates.push(path.join(ROOT, "app", "tools", "python-win", "python.exe")); } else { candidates.push(path.join(ROOT, "app", "tools", "openvino-venv-linux", "bin", "python")); candidates.push(path.join(ROOT, "app", "tools", "openvino-venv", "bin", "python")); // legacy fallback } return [...new Set(candidates)]; } function getOpenVinoPython() { for (const candidate of getOpenVinoPythonCandidates()) { try { const result = spawnSync(candidate, [ "-c", "import openvino, openvino_genai, PIL; print(openvino.__version__)", ], { encoding: "utf8", timeout: 10000, stdio: ["ignore", "pipe", "pipe"] }); if (result.status === 0) return candidate; } catch (_) {} } return null; } let cachedOpenVinoNpuInfo = null; function getOpenVinoNpuInfo() { if (cachedOpenVinoNpuInfo) return cachedOpenVinoNpuInfo; if (osPlatform !== "win32" && osPlatform !== "linux") { cachedOpenVinoNpuInfo = { supported: false, reason: "OpenVINO NPU backend is supported on Windows and Linux Intel Core Ultra systems." }; return cachedOpenVinoNpuInfo; } const python = getOpenVinoPython(); if (!python) { const setupScript = osPlatform === "win32" ? "scripts/setup/setup-openvino-npu.ps1" : "bash scripts/setup/setup-openvino-npu.sh"; cachedOpenVinoNpuInfo = { supported: false, reason: `OpenVINO GenAI runtime is not installed. Run ${setupScript} first.`, }; return cachedOpenVinoNpuInfo; } try { const script = [ "import json, openvino as ov", "core=ov.Core()", "devices=core.available_devices", "info={'devices':devices,'npu':None}", "if 'NPU' in devices:", " info['npu']={'name':core.get_property('NPU','FULL_DEVICE_NAME'),'capabilities':core.get_property('NPU','OPTIMIZATION_CAPABILITIES')}", "print(json.dumps(info))", ].join("\n"); const result = spawnSync(python, ["-c", script], { encoding: "utf8", timeout: 15000, stdio: ["ignore", "pipe", "pipe"] }); if (result.status !== 0) { cachedOpenVinoNpuInfo = { supported: false, python, reason: result.stderr.trim() || "OpenVINO NPU probe failed." }; return cachedOpenVinoNpuInfo; } const info = JSON.parse(result.stdout.trim()); if (!info.devices.includes("NPU")) { cachedOpenVinoNpuInfo = { supported: false, python, reason: `OpenVINO is installed, but NPU is not available. Devices: ${info.devices.join(", ")}` }; return cachedOpenVinoNpuInfo; } cachedOpenVinoNpuInfo = { supported: true, platform: osPlatform, python, devices: info.devices, npu: info.npu }; return cachedOpenVinoNpuInfo; } catch (err) { cachedOpenVinoNpuInfo = { supported: false, python, reason: err.message || String(err) }; return cachedOpenVinoNpuInfo; } } function getOpenVinoModelInfo() { return OPENVINO_NPU_MODELS.map((model) => { const modelPath = path.join(OPENVINO_MODELS, model.folder); const requiredFiles = [ path.join(modelPath, "model_index.json"), path.join(modelPath, "text_encoder", "openvino_model.xml"), path.join(modelPath, "unet", "openvino_model.xml"), path.join(modelPath, "vae_decoder", "openvino_model.xml"), ]; const installed = requiredFiles.every((file) => fs.existsSync(file)); return { ...model, filename: model.id, path: modelPath, installed, sizeBytes: getPathSize(modelPath), size: formatBytes(getPathSize(modelPath)), format: "OpenVINO", }; }); } function findOpenVinoModel(modelId) { return getOpenVinoModelInfo().find((model) => model.id === modelId || model.folder === modelId || model.name === modelId); } function pathInside(childPath, parentPath) { const relative = path.relative(path.resolve(parentPath), path.resolve(childPath)); return relative === "" || (!!relative && !relative.startsWith("..") && !path.isAbsolute(relative)); } function checkPort(port) { return new Promise((resolve) => { const tester = net.createServer() .once("error", () => resolve({ port, available: false, inUse: true })) .once("listening", () => { tester.close(() => resolve({ port, available: true, inUse: false })); }) .listen(port, "127.0.0.1"); }); } async function waitForPortAvailable(port, timeoutMs = 5000) { const startedAt = Date.now(); while (Date.now() - startedAt < timeoutMs) { const status = await checkPort(port); if (status.available) return true; await new Promise((resolve) => setTimeout(resolve, 150)); } return false; } async function findAvailableBackendPort() { const preferred = await checkPort(PREFERRED_BACKEND_PORT); if (preferred.available) return PREFERRED_BACKEND_PORT; for (let port = 28088; port <= 28120; port += 1) { const candidate = await checkPort(port); if (candidate.available) { console.log(` [backend] Preferred port ${PREFERRED_BACKEND_PORT} is busy; using ${port} instead.`); return port; } } throw new Error(`No free backend port found. Tried ${PREFERRED_BACKEND_PORT} and 28088-28120.`); } async function findAvailableLlmPort() { const preferred = await checkPort(PREFERRED_LLM_PORT); if (preferred.available) return PREFERRED_LLM_PORT; for (let port = 28121; port <= 28160; port += 1) { const candidate = await checkPort(port); if (candidate.available) { console.log(` [llm] Preferred port ${PREFERRED_LLM_PORT} is busy; using ${port} instead.`); return port; } } throw new Error(`No free LLM port found. Tried ${PREFERRED_LLM_PORT} and 28121-28160.`); } async function findAvailableSpeechPort() { const preferred = await checkPort(PREFERRED_SPEECH_PORT); if (preferred.available) return PREFERRED_SPEECH_PORT; for (let port = 28161; port <= 28190; port += 1) { const candidate = await checkPort(port); if (candidate.available) { console.log(` [speech] Preferred port ${PREFERRED_SPEECH_PORT} is busy; using ${port} instead.`); return port; } } throw new Error(`No free speech port found. Tried ${PREFERRED_SPEECH_PORT} and 28161-28190.`); } function getLlmBackend() { return getLlmBackendCandidates()[0] || null; } let cachedLlmBackendProbe = null; let cachedLlmBackendProbeAt = 0; function resolveSystemLlamaServer() { const configured = process.env.LOCALAI_LLM_SYSTEM_SERVER || process.env.LLAMA_SERVER_PATH || ""; if (configured && fs.existsSync(configured)) return configured; return ""; } function probeLlmBackend(backend) { if (!backend?.path || !fs.existsSync(backend.path)) { return { ...backend, installed: false, detected: false, devices: [], error: "Backend binary not installed.", }; } try { const result = spawnSync(backend.path, ["--list-devices"], { encoding: "utf8", windowsHide: true, timeout: 10000, maxBuffer: 1024 * 1024, }); const output = `${result.stdout || ""}\n${result.stderr || ""}`; const devices = parseLlamaDeviceList(output); const detected = backend.key === "cpu" || backend.key === "metal" || backend.key === "system" || devices.length > 0; return { ...backend, installed: true, detected, devices, device: devices[0] || null, error: result.status && !detected ? output.trim().slice(-600) : "", }; } catch (err) { return { ...backend, installed: true, detected: backend.key === "cpu", devices: [], device: null, error: err.message || String(err), }; } } function getAllLlmBackendCandidates() { const systemPath = resolveSystemLlamaServer(); const candidates = []; const add = (key, backendPath, mode, portable = true) => { if (!backendPath) return; if (candidates.some((item) => item.path === backendPath)) return; candidates.push({ key, path: backendPath, mode, portable }); }; if (systemPath) add("system", systemPath, "System llama.cpp", false); if (osPlatform === "win32") { add("cuda", LLM_BACKEND_PATHS.winCuda, "Auto (CUDA/CPU)"); add("hip", LLM_BACKEND_PATHS.winHip, "Auto (HIP/CPU)"); add("vulkan", LLM_BACKEND_PATHS.winVulkan, "Auto (Vulkan/CPU)"); add("sycl", LLM_BACKEND_PATHS.winSycl, "Auto (SYCL/CPU)"); add("cpu", LLM_BACKEND_PATHS.winCpu, "CPU"); } else if (osPlatform === "darwin") { add("metal", process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64, process.arch === "arm64" ? "Metal GPU" : "CPU"); } else { add("cuda", LLM_BACKEND_PATHS.linuxCuda, "Auto (CUDA/CPU)"); add("rocm", LLM_BACKEND_PATHS.linuxRocm, "Auto (ROCm/CPU)"); add("sycl", LLM_BACKEND_PATHS.linuxSycl, "Auto (SYCL/CPU)"); add("vulkan", LLM_BACKEND_PATHS.linuxVulkan, "Auto (Vulkan/CPU)"); add("cpu", LLM_BACKEND_PATHS.linuxCpu, "CPU"); } return candidates; } function getLlmBackendProbe(force = false) { const now = Date.now(); if (!force && cachedLlmBackendProbe && now - cachedLlmBackendProbeAt < 60000) return cachedLlmBackendProbe; const probed = getAllLlmBackendCandidates().map(probeLlmBackend); const priority = osPlatform === "win32" ? { system: -1, cuda: 0, hip: 1, vulkan: 2, sycl: 3, metal: 4, cpu: 9 } : osPlatform === "darwin" ? { system: -1, metal: 0, cpu: 9 } : { system: -1, cuda: 0, rocm: 1, sycl: 2, cpu: 3, vulkan: 4 }; const available = probed .filter((item) => item.installed && item.detected) .sort((a, b) => { if (osPlatform !== "linux") { if (a.key === "cpu" && b.key !== "cpu") return 1; if (b.key === "cpu" && a.key !== "cpu") return -1; } return (priority[a.key] ?? 8) - (priority[b.key] ?? 8); }); cachedLlmBackendProbe = { generatedAt: new Date().toISOString(), candidates: probed, available, selected: available[0] || null, }; cachedLlmBackendProbeAt = now; return cachedLlmBackendProbe; } function getLlmBackendCandidates() { const includeUndetectedGpu = process.env.LOCALAI_LLM_TRY_UNDETECTED_GPU === "1"; const probe = getLlmBackendProbe(); if (includeUndetectedGpu) { return probe.candidates.filter((item) => item.installed); } return probe.available; } function getLlmModels() { try { return fs.readdirSync(LLM_MODELS) .filter((filename) => filename.toLowerCase().endsWith(".gguf")) .map((filename) => { const stats = fs.statSync(path.join(LLM_MODELS, filename)); const lower = filename.toLowerCase(); return { filename, name: filename, sizeBytes: stats.size, size: formatBytes(stats.size), format: "GGUF", isProjector: lower.includes("mmproj"), }; }); } catch (_) { return []; } } function findExistingFile(candidates) { return candidates.find((candidate) => candidate && fs.existsSync(candidate)) || ""; } function getSpeechBackendCandidates() { if (osPlatform === "win32") { return [ { key: "vulkan", label: "Vulkan GPU", mode: "whisper.cpp Vulkan GPU", cli: findExistingFile([SPEECH_BACKEND_PATHS.winVulkanCli, path.join(SPEECH_BACKEND_ROOT, "win", "vulkan", "main.exe")]), server: findExistingFile([SPEECH_BACKEND_PATHS.winVulkanServer, path.join(SPEECH_BACKEND_ROOT, "win", "vulkan", "server.exe")]), pathHint: path.join(SPEECH_BACKEND_ROOT, "win", "vulkan"), }, { key: "cpu", label: "CPU", mode: "whisper.cpp CPU", cli: findExistingFile([SPEECH_BACKEND_PATHS.winCpuCli, SPEECH_BACKEND_PATHS.winCli, path.join(SPEECH_BACKEND_ROOT, "win", "cpu", "main.exe"), path.join(SPEECH_BACKEND_ROOT, "win", "main.exe")]), server: findExistingFile([SPEECH_BACKEND_PATHS.winCpuServer, SPEECH_BACKEND_PATHS.winServer, path.join(SPEECH_BACKEND_ROOT, "win", "cpu", "server.exe"), path.join(SPEECH_BACKEND_ROOT, "win", "server.exe")]), pathHint: path.join(SPEECH_BACKEND_ROOT, "win", "cpu"), }, ]; } if (osPlatform === "darwin") { return [ { key: "metal", label: "Metal GPU", mode: "whisper.cpp Metal GPU", cli: findExistingFile([SPEECH_BACKEND_PATHS.macMetalCli, path.join(SPEECH_BACKEND_ROOT, "mac", "metal", "main")]), server: findExistingFile([SPEECH_BACKEND_PATHS.macMetalServer, path.join(SPEECH_BACKEND_ROOT, "mac", "metal", "server")]), pathHint: path.join(SPEECH_BACKEND_ROOT, "mac", "metal"), }, { key: "cpu", label: "CPU", mode: "whisper.cpp CPU", cli: findExistingFile([SPEECH_BACKEND_PATHS.macCpuCli, SPEECH_BACKEND_PATHS.macCli, path.join(SPEECH_BACKEND_ROOT, "mac", "cpu", "main"), path.join(SPEECH_BACKEND_ROOT, "mac", "main")]), server: findExistingFile([SPEECH_BACKEND_PATHS.macCpuServer, SPEECH_BACKEND_PATHS.macServer, path.join(SPEECH_BACKEND_ROOT, "mac", "cpu", "server"), path.join(SPEECH_BACKEND_ROOT, "mac", "server")]), pathHint: path.join(SPEECH_BACKEND_ROOT, "mac", "cpu"), }, ]; } return [ { key: "vulkan", label: "Vulkan GPU", mode: "whisper.cpp Vulkan GPU", cli: findExistingFile([SPEECH_BACKEND_PATHS.linuxVulkanCli, path.join(SPEECH_BACKEND_ROOT, "linux", "vulkan", "main")]), server: findExistingFile([SPEECH_BACKEND_PATHS.linuxVulkanServer, path.join(SPEECH_BACKEND_ROOT, "linux", "vulkan", "server")]), pathHint: path.join(SPEECH_BACKEND_ROOT, "linux", "vulkan"), }, { key: "cpu", label: "CPU", mode: "whisper.cpp CPU", cli: findExistingFile([SPEECH_BACKEND_PATHS.linuxCpuCli, SPEECH_BACKEND_PATHS.linuxCli, path.join(SPEECH_BACKEND_ROOT, "linux", "cpu", "main"), path.join(SPEECH_BACKEND_ROOT, "linux", "main")]), server: findExistingFile([SPEECH_BACKEND_PATHS.linuxCpuServer, SPEECH_BACKEND_PATHS.linuxServer, path.join(SPEECH_BACKEND_ROOT, "linux", "cpu", "server"), path.join(SPEECH_BACKEND_ROOT, "linux", "server")]), pathHint: path.join(SPEECH_BACKEND_ROOT, "linux", "cpu"), }, ]; } function normalizeSpeechBackendPreference(value) { const raw = String(value || "auto").trim().toLowerCase(); if (["auto", "cpu", "vulkan", "metal"].includes(raw)) return raw; return "auto"; } function getSpeechBackend(preference = speechSettings.backendPreference) { const candidates = getSpeechBackendCandidates().map((candidate) => ({ ...candidate, installed: Boolean(candidate.cli), })); const preferred = normalizeSpeechBackendPreference(preference); const selected = preferred === "auto" ? candidates.find((candidate) => candidate.installed) || candidates[candidates.length - 1] : candidates.find((candidate) => candidate.key === preferred) || candidates.find((candidate) => candidate.installed) || candidates[candidates.length - 1]; return { ...(selected || { key: "cpu", label: "CPU", mode: "whisper.cpp CPU", cli: "", server: "" }), preference: preferred, candidates, }; } function getSpeechModels() { const catalog = SPEECH_MODEL_CATALOG.map((model) => { const modelPath = path.join(SPEECH_MODELS, model.filename); const installed = fs.existsSync(modelPath); return { ...model, installed, sizeBytes: installed ? getPathSize(modelPath) : 0, localSize: installed ? formatBytes(getPathSize(modelPath)) : "", url: `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${model.filename}`, }; }); const known = new Set(catalog.map((model) => model.filename.toLowerCase())); let custom = []; try { custom = fs.readdirSync(SPEECH_MODELS) .filter((filename) => filename.toLowerCase().endsWith(".bin") && !known.has(filename.toLowerCase())) .map((filename) => { const sizeBytes = getPathSize(path.join(SPEECH_MODELS, filename)); return { id: filename, name: filename, filename, size: formatBytes(sizeBytes), localSize: formatBytes(sizeBytes), sizeBytes, language: "Custom", installed: true, recommended: false, custom: true, }; }); } catch (_) {} return [...catalog, ...custom]; } function resolveSpeechModel(value) { const raw = String(value || "").trim(); const model = getSpeechModels().find((item) => item.id === raw || item.filename === raw) || getSpeechModels().find((item) => item.installed); if (!model) throw new Error("Download or import a Whisper model first."); const modelPath = path.join(SPEECH_MODELS, path.basename(model.filename)); if (!pathInside(modelPath, SPEECH_MODELS) || !fs.existsSync(modelPath)) { throw new Error(`Speech model is not installed: ${model.filename}`); } return { ...model, path: modelPath }; } function getTtsManifestPath(filename) { const manifestPath = path.join(TTS_MODELS, path.basename(String(filename || ""))); return pathInside(manifestPath, TTS_MODELS) ? manifestPath : ""; } function readTtsManifest(filename) { const manifestPath = getTtsManifestPath(filename); if (!manifestPath || !fs.existsSync(manifestPath)) return null; try { return JSON.parse(fs.readFileSync(manifestPath, "utf8")); } catch (_) { return null; } } function installTtsCatalogModel(modelIdOrFilename) { const raw = String(modelIdOrFilename || "").trim(); const catalogModel = TTS_MODEL_CATALOG.find((model) => model.id === raw || model.filename === raw) || TTS_MODEL_CATALOG[0]; if (!catalogModel) throw new Error("Unknown TTS model."); const manifestPath = getTtsManifestPath(catalogModel.filename); const manifest = { ...catalogModel, installed: true, createdAt: new Date().toISOString(), notes: "Kokoro model files are cached under app/tts-cache by kokoro-js on first generation.", }; fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), "utf8"); return manifest; } function getTtsModels() { const catalog = TTS_MODEL_CATALOG.map((model) => { const manifest = readTtsManifest(model.filename); const installed = Boolean(manifest); const manifestPath = getTtsManifestPath(model.filename); const sizeBytes = installed ? getPathSize(manifestPath) : 0; return { ...model, ...(manifest || {}), installed, sizeBytes, localSize: installed ? formatBytes(sizeBytes) : "", url: `kokoro://install/${model.id}`, }; }); const known = new Set(catalog.map((model) => model.filename.toLowerCase())); let custom = []; try { custom = fs.readdirSync(TTS_MODELS) .filter((filename) => filename.toLowerCase().endsWith(".json") && !known.has(filename.toLowerCase())) .map((filename) => { const manifest = readTtsManifest(filename) || {}; const sizeBytes = getPathSize(path.join(TTS_MODELS, filename)); return { id: manifest.id || filename, name: manifest.name || filename, filename, modelId: manifest.modelId || "onnx-community/Kokoro-82M-v1.0-ONNX", dtype: manifest.dtype || "q8", format: manifest.format || "Kokoro ONNX", size: formatBytes(sizeBytes), localSize: formatBytes(sizeBytes), sizeBytes, installed: true, recommended: false, custom: true, }; }); } catch (_) {} return [...catalog, ...custom]; } function resolveTtsModel(value) { const raw = String(value || "").trim(); const model = getTtsModels().find((item) => item.id === raw || item.filename === raw) || getTtsModels().find((item) => item.installed); if (!model) throw new Error("Download or import a Kokoro TTS model from Model Manager first."); const manifestPath = getTtsManifestPath(model.filename); if (!manifestPath || !fs.existsSync(manifestPath)) { throw new Error(`TTS model is not installed: ${model.filename}`); } return { ...model, path: manifestPath }; } function getTtsRuntimeStatus() { const nodeModules = path.join(TTS_RUNTIME, "node_modules"); const installed = fs.existsSync(path.join(nodeModules, "kokoro-js")); return { installed: installed && fs.existsSync(TTS_WORKER), worker: TTS_WORKER, runtime: TTS_RUNTIME, cache: TTS_CACHE, backendMode: "Kokoro ONNX", }; } function runExclusiveTtsOperation(operation) { const run = ttsOperationQueue.catch(() => {}).then(operation); ttsOperationQueue = run.catch(() => {}); return run; } async function startTts(settings = {}) { const runtime = getTtsRuntimeStatus(); if (!runtime.installed) { throw new Error("Kokoro TTS runtime is not installed. Run scripts/setup/setup-tts for this platform."); } const model = resolveTtsModel(settings.model || ttsSettings.model); PORT_TTS = PREFERRED_TTS_PORT; ttsError = null; ttsReady = true; ttsSettings = { ...ttsSettings, model: model.filename, voice: settings.voice || ttsSettings.voice || "af_heart", speed: Math.max(0.5, Math.min(2, Number(settings.speed) || ttsSettings.speed || 1)), dtype: model.dtype || settings.dtype || ttsSettings.dtype || "q8", backendMode: runtime.backendMode, }; } async function stopTts() { ttsReady = false; ttsError = null; ttsGenerationState = { active: false, phase: "", progress: 0, model: "", voice: "", output: "" }; } function saveTtsOutput(result) { const createdAt = new Date().toISOString(); const stamp = createdAt.replace(/[:.]/g, "-"); const base = `tts-${stamp}-${safeOutputName(result.voice || "voice")}`; const wavFilename = `${base}.wav`; const jsonFilename = `${base}.json`; const wavPath = path.join(TTS_OUTPUTS, wavFilename); fs.copyFileSync(result.wavPath, wavPath); const metadata = { text: result.text || "", model: result.model, modelName: result.modelName, voice: result.voice, voiceName: result.voiceName, speed: result.speed, durationMs: result.durationMs, sampleRate: result.sampleRate || 24000, createdAt, audioFile: wavFilename, metadata: jsonFilename, displayName: result.displayName || `${result.voiceName || result.voice} - ${createdAt}`, }; fs.writeFileSync(path.join(TTS_OUTPUTS, jsonFilename), JSON.stringify(metadata, null, 2), "utf8"); return { ...metadata, filename: jsonFilename, url: `/tts-outputs/${encodeURIComponent(wavFilename)}`, }; } function listTtsOutputs() { try { return fs.readdirSync(TTS_OUTPUTS) .filter((file) => file.toLowerCase().endsWith(".json")) .map((file) => { const filePath = path.join(TTS_OUTPUTS, file); try { const metadata = JSON.parse(fs.readFileSync(filePath, "utf8")); const stat = fs.statSync(filePath); return { ...metadata, filename: file, displayName: metadata.displayName || metadata.text?.slice(0, 48) || metadata.audioFile || file, sizeBytes: stat.size, size: formatBytes(stat.size), modifiedAt: stat.mtime.toISOString(), createdAt: metadata.createdAt || stat.mtime.toISOString(), url: metadata.audioFile ? `/tts-outputs/${encodeURIComponent(metadata.audioFile)}` : "", }; } catch (_) { return null; } }) .filter(Boolean) .sort((a, b) => String(b.createdAt || "").localeCompare(String(a.createdAt || ""))); } catch (_) { return []; } } function synthesizeTts(text, options = {}) { return new Promise((resolve, reject) => { const cleanedText = String(text || "").trim(); if (!cleanedText) { reject(new Error("Enter text to synthesize.")); return; } if (cleanedText.length > 5000) { reject(new Error("TTS text is too long. Limit is 5000 characters for V1.")); return; } const runtime = getTtsRuntimeStatus(); if (!runtime.installed) { reject(new Error("Kokoro TTS runtime is not installed.")); return; } const model = resolveTtsModel(options.model || ttsSettings.model); const voice = String(options.voice || ttsSettings.voice || "af_heart"); const voiceInfo = TTS_VOICES.find((item) => item.id === voice) || { id: voice, name: voice }; const speed = Math.max(0.5, Math.min(2, Number(options.speed) || ttsSettings.speed || 1)); const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const tempWav = path.join(TTS_OUTPUTS, `.tts-${stamp}.wav`); const payload = { text: cleanedText, modelId: model.modelId || "onnx-community/Kokoro-82M-v1.0-ONNX", dtype: model.dtype || "q8", voice, speed, output: tempWav, cacheDir: TTS_CACHE, }; ttsGenerationState = { active: true, phase: "Generating speech...", progress: -1, model: model.filename, voice, output: "", }; const startedAt = Date.now(); const proc = spawn(process.execPath, [TTS_WORKER], { cwd: ROOT, stdio: "pipe", windowsHide: true, env: { ...process.env, NODE_PATH: path.join(TTS_RUNTIME, "node_modules"), TTS_RUNTIME, }, }); let stdout = ""; let stderr = ""; proc.stdout.on("data", (data) => { stdout += data.toString(); }); proc.stderr.on("data", (data) => { stderr += data.toString(); process.stderr.write(" [tts] " + data.toString()); }); proc.on("error", (err) => { ttsGenerationState = { active: false, phase: "Failed", progress: 0, model: model.filename, voice, output: "" }; reject(err); }); proc.on("exit", (code) => { ttsGenerationState = { active: false, phase: code === 0 ? "Complete" : "Failed", progress: code === 0 ? 100 : 0, model: model.filename, voice, output: "" }; if (code !== 0) { try { fs.unlinkSync(tempWav); } catch (_) {} const message = (stderr || stdout || `Kokoro TTS exited with code ${code}`).trim().slice(-1400); ttsError = message; reject(new Error(message)); return; } try { const workerResult = JSON.parse(stdout.trim() || "{}"); if (!workerResult.ok || !fs.existsSync(tempWav)) { throw new Error(workerResult.error || "Kokoro TTS did not produce a WAV file."); } const saved = saveTtsOutput({ text: cleanedText, model: model.filename, modelName: model.name, voice, voiceName: voiceInfo.name, speed, durationMs: Date.now() - startedAt, sampleRate: workerResult.sampleRate || 24000, wavPath: tempWav, displayName: `${voiceInfo.name || voice} - ${cleanedText.slice(0, 40)}`, }); try { fs.unlinkSync(tempWav); } catch (_) {} ttsError = null; resolve(saved); } catch (err) { try { fs.unlinkSync(tempWav); } catch (_) {} ttsError = err.message || String(err); reject(err); } }); proc.stdin.end(JSON.stringify(payload)); }); } function runExclusiveSpeechOperation(operation) { const run = speechOperationQueue.catch(() => {}).then(operation); speechOperationQueue = run.catch(() => {}); return run; } async function startSpeech(settings = {}) { const backendPreference = normalizeSpeechBackendPreference(settings.backendPreference || speechSettings.backendPreference); const backend = getSpeechBackend(backendPreference); if (!backend.cli) { throw new Error(`${backend.label || "Selected"} whisper.cpp backend is not installed. Run setup or copy a compatible binary to ${backend.pathHint || "app/speech-backend"}.`); } const model = resolveSpeechModel(settings.model); PORT_SPEECH = await findAvailableSpeechPort(); speechError = null; speechReady = true; speechSettings = { ...speechSettings, model: model.filename, language: settings.language || speechSettings.language || "auto", threads: Math.max(1, Math.min(32, Number(settings.threads) || speechSettings.threads || 4)), backendPreference, backendBinary: path.basename(backend.cli), backendMode: backend.mode, }; } async function stopSpeech() { speechReady = false; speechError = null; speechTranscriptionState = { active: false, phase: "", progress: 0, model: "", filename: "" }; } function readBinaryBody(req, limitBytes = 1024 * 1024 * 1024) { return new Promise((resolve, reject) => { const chunks = []; let bytes = 0; req.on("data", (chunk) => { bytes += chunk.length; if (bytes > limitBytes) { reject(new Error(`Audio file is too large. Limit is ${Math.round(limitBytes / (1024 * 1024))} MB.`)); req.destroy(); return; } chunks.push(chunk); }); req.on("end", () => resolve(Buffer.concat(chunks))); req.on("error", reject); }); } function isWaveBuffer(buffer) { return buffer?.length > 44 && buffer.toString("ascii", 0, 4) === "RIFF" && buffer.toString("ascii", 8, 12) === "WAVE"; } function isAudioFile(buffer) { try { if (!buffer || buffer.length < 12) return false; if (isWaveBuffer(buffer)) return true; const magic = buffer.toString("ascii", 0, 3); if (magic === "ID3") return true; // MP3 with ID3 tag const ftyp = buffer.toString("ascii", 4, 8); if (ftyp === "ftyp") return true; // MP4/M4A container const ogg = buffer.toString("ascii", 0, 4); if (ogg === "OggS") return true; // OGG/OPUS const flac = buffer.toString("ascii", 0, 4); if (flac === "fLaC") return true; // FLAC return false; } catch (_) { return false; } } function saveTranscript(result) { const createdAt = new Date().toISOString(); const stamp = createdAt.replace(/[:.]/g, "-"); const base = `transcript-${stamp}-${safeOutputName(result.sourceFilename || "audio")}`; const textFilename = `${base}.txt`; const jsonFilename = `${base}.json`; fs.writeFileSync(path.join(TRANSCRIPTIONS, textFilename), result.text || "", "utf8"); const metadata = { ...result, createdAt, textFile: textFilename, metadata: jsonFilename, }; fs.writeFileSync(path.join(TRANSCRIPTIONS, jsonFilename), JSON.stringify(metadata, null, 2), "utf8"); return metadata; } function listTranscriptions() { try { return fs.readdirSync(TRANSCRIPTIONS) .filter((file) => file.toLowerCase().endsWith(".json")) .map((file) => { const filePath = path.join(TRANSCRIPTIONS, file); try { const metadata = JSON.parse(fs.readFileSync(filePath, "utf8")); const stat = fs.statSync(filePath); return { ...metadata, filename: file, displayName: metadata.sourceFilename || metadata.textFile || file, sizeBytes: stat.size, size: formatBytes(stat.size), modifiedAt: stat.mtime.toISOString(), createdAt: metadata.createdAt || stat.mtime.toISOString(), url: `/transcriptions/${encodeURIComponent(file)}`, textUrl: metadata.textFile ? `/transcriptions/${encodeURIComponent(metadata.textFile)}` : "", }; } catch (_) { return null; } }) .filter(Boolean) .sort((a, b) => String(b.createdAt || "").localeCompare(String(a.createdAt || ""))); } catch (_) { return []; } } function transcribeWavBuffer(buffer, options = {}) { return new Promise((resolve, reject) => { if (!isWaveBuffer(buffer)) { reject(new Error("Speech transcription currently accepts WAV audio only.")); return; } const backendPreference = normalizeSpeechBackendPreference(options.backendPreference || speechSettings.backendPreference); const backend = getSpeechBackend(backendPreference); if (!backend.cli) { reject(new Error(`${backend.label || "Selected"} whisper.cpp CLI backend is not installed.`)); return; } const model = resolveSpeechModel(options.model || speechSettings.model); const language = String(options.language || speechSettings.language || "auto"); const sourceFilename = safeOutputName(options.filename || "recording.wav") || "recording.wav"; const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const tempBase = path.join(TRANSCRIPTIONS, `.speech-${stamp}-${sourceFilename.replace(/\.[^.]+$/, "")}`); const wavPath = `${tempBase}.wav`; const outBase = `${tempBase}-out`; fs.writeFileSync(wavPath, buffer); speechTranscriptionState = { active: true, phase: "Transcribing audio...", progress: -1, model: model.filename, filename: sourceFilename, }; const args = [ "-m", model.path, "-f", wavPath, "-otxt", "-oj", "-of", outBase, "-t", String(Math.max(1, Math.min(32, Number(options.threads) || speechSettings.threads || 4))), ]; if (language && language !== "auto") { args.push("-l", language); } else { args.push("-l", "auto"); } if (options.translate) { args.push("-tr"); } // Suppress non-speech tokens (e.g. [Musica], [Applauso]) that appear // when the model misidentifies compressed/noisy audio as non-speech if (options.suppressNonSpeech !== false) { args.push("-sns"); } // Lower no-speech threshold (default 0.60) so the model is less likely // to label quiet or compressed speech as non-speech const noSpeechThreshold = options.noSpeechThreshold || "0.30"; args.push("-nth", String(noSpeechThreshold)); const startedAt = Date.now(); console.log(" [speech] Starting:", backend.cli, args.join(" ")); const proc = spawn(backend.cli, args, { stdio: "pipe", windowsHide: true, env: { ...process.env, ...(osPlatform === "win32" ? { PATH: path.dirname(backend.cli) + (process.env.PATH ? `;${process.env.PATH}` : "") } : {}), ...(osPlatform === "linux" ? { LD_LIBRARY_PATH: path.dirname(backend.cli) + (process.env.LD_LIBRARY_PATH ? `:${process.env.LD_LIBRARY_PATH}` : "") } : {}), ...(osPlatform === "darwin" ? { DYLD_LIBRARY_PATH: path.dirname(backend.cli) + (process.env.DYLD_LIBRARY_PATH ? `:${process.env.DYLD_LIBRARY_PATH}` : "") } : {}), }, }); let stdout = ""; let stderr = ""; proc.stdout.on("data", (data) => { stdout += data.toString(); process.stdout.write(" [speech] " + data.toString()); }); proc.stderr.on("data", (data) => { stderr += data.toString(); process.stderr.write(" [speech-err] " + data.toString()); }); proc.on("exit", (code) => { try { fs.unlinkSync(wavPath); } catch (_) {} speechTranscriptionState = { active: false, phase: code === 0 ? "Complete" : "Failed", progress: code === 0 ? 100 : 0, model: model.filename, filename: sourceFilename }; if (code !== 0) { const message = (stderr || stdout || `whisper.cpp exited with code ${code}`).trim().slice(-1200); speechError = message; reject(new Error(message)); return; } const txtPath = `${outBase}.txt`; const jsonPath = `${outBase}.json`; const text = fs.existsSync(txtPath) ? fs.readFileSync(txtPath, "utf8").trim() : stdout.trim(); let raw = null; try { raw = fs.existsSync(jsonPath) ? JSON.parse(fs.readFileSync(jsonPath, "utf8")) : null; } catch (_) {} try { fs.unlinkSync(txtPath); } catch (_) {} try { fs.unlinkSync(jsonPath); } catch (_) {} const saved = saveTranscript({ text, model: model.filename, modelName: model.name, language, backend: backend.key, backendMode: backend.mode, sourceFilename, durationMs: Date.now() - startedAt, raw, }); speechError = null; resolve(saved); }); }); } function transcribeAudioBuffer(buffer, options = {}) { if (isWaveBuffer(buffer)) { return transcribeWavBuffer(buffer, options); } return new Promise((resolve, reject) => { const backendPreference = normalizeSpeechBackendPreference(options.backendPreference || speechSettings.backendPreference); const backend = getSpeechBackend(backendPreference); if (!backend.cli) { reject(new Error(`${backend.label || "Selected"} whisper.cpp CLI backend is not installed.`)); return; } const model = resolveSpeechModel(options.model || speechSettings.model); const language = String(options.language || speechSettings.language || "auto"); const sourceFilename = safeOutputName(options.filename || "recording") || "recording"; const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const tempBase = path.join(TRANSCRIPTIONS, `.speech-${stamp}-${sourceFilename.replace(/\.[^.]+$/, "")}`); const inputPath = `${tempBase}-input`; const wavPath = `${tempBase}.wav`; const outBase = `${tempBase}-out`; fs.writeFileSync(inputPath, buffer); speechTranscriptionState = { active: true, phase: "Converting audio...", progress: -1, model: model.filename, filename: sourceFilename, }; const ffmpegArgs = ["-y", "-i", inputPath, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", wavPath]; console.log(" [speech] Converting with ffmpeg:", ffmpegArgs.join(" ")); const ffmpegProc = spawn("ffmpeg", ffmpegArgs, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, }); let ffmpegStderr = ""; ffmpegProc.stderr.on("data", (data) => { ffmpegStderr += data.toString(); }); ffmpegProc.on("exit", (code) => { try { fs.unlinkSync(inputPath); } catch (_) {} if (code !== 0) { const message = (ffmpegStderr || `ffmpeg exited with code ${code}`).trim().slice(-1200); speechError = message; try { fs.unlinkSync(wavPath); } catch (_) {} reject(new Error("Audio conversion failed: " + message)); return; } const wavBuffer = fs.readFileSync(wavPath); try { fs.unlinkSync(wavPath); } catch (_) {} transcribeWavBuffer(wavBuffer, options).then(resolve).catch((err) => { try { fs.unlinkSync(wavPath); } catch (_) {} reject(err); }); }); }); } function pingLlmReady(expectedModel = "") { return new Promise((resolve) => { const req = http.get(`http://127.0.0.1:${PORT_LLM}/v1/models`, (res) => { let body = ""; res.setEncoding("utf8"); res.on("data", (chunk) => { body += chunk; }); res.on("end", () => { if (res.statusCode < 200 || res.statusCode >= 500) { resolve(false); return; } if (!expectedModel) { resolve(true); return; } try { const parsed = JSON.parse(body || "{}"); const servedIds = Array.isArray(parsed.data) ? parsed.data.map((item) => String(item.id || item.model || "")) : []; const expectedBase = path.basename(expectedModel).toLowerCase(); resolve(servedIds.some((id) => { const lower = id.toLowerCase(); return lower === expectedBase || lower.includes(expectedBase); })); } catch (_) { resolve(false); } }); }); req.setTimeout(1000, () => { req.destroy(); resolve(false); }); req.on("error", () => resolve(false)); }); } async function waitForLlmReady(maxAttempts = 240, expectedProc = llmProc, expectedModel = "") { for (let attempt = 0; attempt < maxAttempts; attempt += 1) { if (!expectedProc || llmProc !== expectedProc) throw new Error(llmError || "llama.cpp exited during startup."); if (await pingLlmReady(expectedModel)) { llmReady = true; return; } await new Promise((resolve) => setTimeout(resolve, 500)); } throw new Error(`llama.cpp did not become ready within ${Math.round(maxAttempts / 120)} minutes.`); } function waitForChildExit(proc, timeoutMs) { return new Promise((resolve) => { if (!proc || proc.exitCode !== null || proc.signalCode !== null) { resolve(true); return; } let done = false; const finish = (exited) => { if (done) return; done = true; clearTimeout(timer); proc.off("exit", onExit); resolve(exited); }; const onExit = () => finish(true); const timer = setTimeout(() => finish(false), timeoutMs); proc.once("exit", onExit); }); } async function killLlm() { llmReady = false; llmSettings.supportsVision = false; llmSettings.visionMode = "none"; llmSettings.visionStatus = "Projector not loaded"; llmSettings.mmproj = null; if (!llmProc) { await waitForPortAvailable(PORT_LLM, 1500); return; } const proc = llmProc; const procPort = PORT_LLM; llmProc = null; try { proc.kill("SIGTERM"); } catch (_) {} const exitedAfterTerm = await waitForChildExit(proc, 2500); if (!exitedAfterTerm) { try { proc.kill("SIGKILL"); } catch (_) {} await waitForChildExit(proc, 2500); } const released = await waitForPortAvailable(procPort, 5000); if (!released) { console.warn(` [llm] Port ${procPort} is still busy after stopping llama.cpp.`); } } function getSetupPaths() { const appDir = path.join(ROOT, "app"); if (osPlatform === "win32") { return { node: path.join(appDir, "tools", "node-win", "node.exe"), npm: path.join(appDir, "tools", "node-win", "npm.cmd"), distIndex: path.join(DIST, "index.html"), cudaBackend: BACKEND_PATHS.cuda, vulkanBackend: BACKEND_PATHS.vulkan, cpuBackend: BACKEND_PATHS.cpu, models: MODELS, outputs: OUTPUTS, chatHistory: CHAT_HISTORY, transcriptions: TRANSCRIPTIONS, ttsOutputs: TTS_OUTPUTS, }; } else if (osPlatform === "darwin") { return { node: path.join(appDir, "tools", "node-mac", "bin", "node"), npm: path.join(appDir, "tools", "node-mac", "bin", "npm"), distIndex: path.join(DIST, "index.html"), macBackend: BACKEND_PATHS.mac, models: MODELS, outputs: OUTPUTS, chatHistory: CHAT_HISTORY, transcriptions: TRANSCRIPTIONS, ttsOutputs: TTS_OUTPUTS, }; } else { // Linux / WSL return { node: path.join(appDir, "tools", "node-linux", "bin", "node"), npm: path.join(appDir, "tools", "node-linux", "bin", "npm"), distIndex: path.join(DIST, "index.html"), linuxCpuBackend: BACKEND_PATHS.linuxCpu, linuxVulkanBackend: BACKEND_PATHS.linuxVulkan, linuxRocmBackend: BACKEND_PATHS.linuxRocm, models: MODELS, outputs: OUTPUTS, chatHistory: CHAT_HISTORY, transcriptions: TRANSCRIPTIONS, ttsOutputs: TTS_OUTPUTS, }; } } async function getHealth() { const paths = getSetupPaths(); const checks = [ getPathInfo("Portable Node.js", paths.node), getPathInfo("Portable npm", paths.npm), getPathInfo("Frontend build", paths.distIndex), getDirInfo("Models folder", paths.models), getDirInfo("Outputs folder", paths.outputs), getDirInfo("Chat history folder", paths.chatHistory), getDirInfo("Transcriptions folder", paths.transcriptions), getDirInfo("TTS outputs folder", paths.ttsOutputs), ]; if (osPlatform === "win32") { checks.push(getPathInfo("CUDA backend", paths.cudaBackend)); checks.push(getPathInfo("Vulkan backend", paths.vulkanBackend)); checks.push(getPathInfo("CPU backend", paths.cpuBackend)); } else if (osPlatform === "darwin") { checks.push(getPathInfo("Mac backend", paths.macBackend)); } else { checks.push(getPathInfo("Linux CPU backend", paths.linuxCpuBackend)); checks.push(getPathInfo("Linux Vulkan backend", paths.linuxVulkanBackend)); checks.push(getPathInfo("Linux ROCm backend", paths.linuxRocmBackend)); } let backendInstalled = false; if (osPlatform === "win32") { backendInstalled = checks.find((check) => check.label === "CUDA backend")?.exists || checks.find((check) => check.label === "Vulkan backend")?.exists || checks.find((check) => check.label === "CPU backend")?.exists; } else if (osPlatform === "darwin") { backendInstalled = checks.find((check) => check.label === "Mac backend")?.exists; } else { backendInstalled = checks.find((check) => check.label === "Linux CPU backend")?.exists || checks.find((check) => check.label === "Linux CPU backend")?.exists || checks.find((check) => check.label === "Linux Vulkan backend")?.exists || checks.find((check) => check.label === "Linux ROCm backend")?.exists; } const criticalOk = checks .filter((check) => !["CUDA backend", "Vulkan backend", "CPU backend", "Linux CPU backend", "Linux Vulkan backend", "Linux ROCm backend", "Mac backend"].includes(check.label)) .every((check) => check.ok) && backendInstalled; const ports = { frontend: { ...(await checkPort(PORT_FRONTEND)), expectedInUse: true }, backend: { ...(await checkPort(PORT_BACKEND)), expectedInUse: backendProc !== null || openvinoProc !== null }, }; ports.frontend.ok = !ports.frontend.available; ports.backend.preferred = PREFERRED_BACKEND_PORT; ports.backend.selected = PORT_BACKEND; ports.backend.ok = true; const issues = checks .filter((check) => !check.ok && !["CUDA backend", "Vulkan backend", "CPU backend", "Linux CPU backend", "Linux Vulkan backend", "Linux ROCm backend", "Mac backend"].includes(check.label)) .map((check) => `${check.label} is missing or not writable.`); if (!backendInstalled) { issues.push(`No ${osPlatform === "win32" ? "Windows" : osPlatform === "darwin" ? "macOS" : "Linux"} backend binary is installed.`); } return { ok: criticalOk && ports.backend.ok, build: SERVER_BUILD, root: ROOT, platform: osPlatform, checks, ports, backend: { ready: backendReady || openvinoReady, running: backendProc !== null || openvinoProc !== null, error: backendError, settings: currentSettings, options: getBackendOptions(), }, models: { count: fs.existsSync(MODELS) ? fs.readdirSync(MODELS).filter(isModelFile).length : 0, totalBytes: getPathSize(MODELS), totalSize: formatBytes(getPathSize(MODELS)), }, outputs: { count: fs.existsSync(OUTPUTS) ? fs.readdirSync(OUTPUTS).filter((file) => file.toLowerCase().endsWith(".json")).length : 0, totalBytes: getPathSize(OUTPUTS), totalSize: formatBytes(getPathSize(OUTPUTS)), }, chat: { count: fs.existsSync(CHAT_HISTORY) ? fs.readdirSync(CHAT_HISTORY).filter((file) => file.toLowerCase().endsWith(".json")).length : 0, totalBytes: getPathSize(CHAT_HISTORY), totalSize: formatBytes(getPathSize(CHAT_HISTORY)), }, speech: { transcriptions: fs.existsSync(TRANSCRIPTIONS) ? fs.readdirSync(TRANSCRIPTIONS).filter((file) => file.toLowerCase().endsWith(".json")).length : 0, totalBytes: getPathSize(TRANSCRIPTIONS), totalSize: formatBytes(getPathSize(TRANSCRIPTIONS)), }, tts: { models: fs.existsSync(TTS_MODELS) ? fs.readdirSync(TTS_MODELS).filter((file) => file.toLowerCase().endsWith(".json")).length : 0, outputs: fs.existsSync(TTS_OUTPUTS) ? fs.readdirSync(TTS_OUTPUTS).filter((file) => file.toLowerCase().endsWith(".json")).length : 0, runtimeInstalled: getTtsRuntimeStatus().installed, totalBytes: getPathSize(TTS_MODELS) + getPathSize(TTS_OUTPUTS) + getPathSize(TTS_CACHE), totalSize: formatBytes(getPathSize(TTS_MODELS) + getPathSize(TTS_OUTPUTS) + getPathSize(TTS_CACHE)), }, issues, }; } function addCleanupCandidate(candidates, id, targetPath, reason, options = {}) { if (!fs.existsSync(targetPath)) return; if (!options.allowUserData && ( pathInside(targetPath, MODELS) || pathInside(targetPath, OUTPUTS) || pathInside(targetPath, CHAT_HISTORY) || pathInside(targetPath, TRANSCRIPTIONS) || pathInside(targetPath, TTS_MODELS) || pathInside(targetPath, TTS_OUTPUTS) )) return; const sizeBytes = getPathSize(targetPath); candidates.push({ id, path: targetPath, name: path.basename(targetPath), reason, sizeBytes, size: formatBytes(sizeBytes), }); } function getCleanupCandidates() { const candidates = []; const toolsDir = path.join(ROOT, "app", "tools"); const frontendDir = path.join(ROOT, "app", "frontend"); addCleanupCandidate(candidates, "scratch-invalid", path.join(ROOT, "scratch_invalid.txt"), "Scratch/debug file."); addCleanupCandidate(candidates, "vite-cache", path.join(frontendDir, "node_modules", ".vite"), "Vite dependency cache; it can be regenerated."); if (fs.existsSync(toolsDir)) { for (const item of fs.readdirSync(toolsDir)) { const itemPath = path.join(toolsDir, item); if (item.toLowerCase().endsWith(".zip")) { addCleanupCandidate(candidates, `setup-zip-${item}`, itemPath, "Setup download archive."); } else if (/^sd-.+-temp$/i.test(item)) { addCleanupCandidate(candidates, `setup-temp-${item}`, itemPath, "Temporary backend extraction folder."); } } } if (fs.existsSync(MODELS)) { for (const item of fs.readdirSync(MODELS)) { if (item.toLowerCase().endsWith(".part") || item.toLowerCase().endsWith(".tmp")) { addCleanupCandidate(candidates, `partial-model-${item}`, path.join(MODELS, item), "Incomplete model download/import file.", { allowUserData: true }); } } } return candidates.sort((a, b) => b.sizeBytes - a.sizeBytes); } function cleanupSelected(ids = []) { const allowed = new Map(getCleanupCandidates().map((candidate) => [candidate.id, candidate])); const deleted = []; for (const id of ids) { const candidate = allowed.get(id); if (!candidate) continue; if (!fs.existsSync(candidate.path)) continue; const stat = fs.statSync(candidate.path); if (stat.isDirectory()) { fs.rmSync(candidate.path, { recursive: true, force: true }); } else { fs.unlinkSync(candidate.path); } deleted.push(candidate); } return deleted; } function hasNvidiaGpu() { const info = getGpuInfo().name.toLowerCase(); if (info.includes("nvidia")) return true; try { execSync("nvidia-smi", { stdio: "ignore" }); return true; } catch (_) { return false; } } function hasAmdGpu() { const info = getGpuInfo().name.toLowerCase(); return info.includes("amd") || info.includes("advanced micro devices") || info.includes("radeon"); } // Detect whether we are running inside WSL2 (Windows Subsystem for Linux). // WSL2's GPU paravirtualization (GPU-PV) only exposes a D3D12/DirectX interface // to Linux. Vulkan hardware passthrough is only supported for NVIDIA (via CUDA). // Intel Arc and AMD GPUs in WSL2 fall back to llvmpipe (CPU software rendering). function isRunningInWSL() { if (osPlatform !== "linux") return false; try { const procVersion = fs.readFileSync("/proc/version", "utf8").toLowerCase(); return procVersion.includes("microsoft") || procVersion.includes("wsl"); } catch (_) { return false; } } function getVulkanUnavailableReason() { if (osPlatform === "win32" && !hasVulkanRuntime()) { return "The Vulkan runtime (vulkan-1.dll) is missing. Install or update your GPU vendor driver with Vulkan support, then run setup again so the Vulkan backend folder is repaired."; } if (osPlatform === "linux" && isRunningInWSL()) { const gpuName = getGpuInfo().name; const lowerGpu = gpuName.toLowerCase(); if (lowerGpu.includes("intel") || lowerGpu.includes("arc")) { return `Vulkan GPU is not available in WSL2 for Intel ${gpuName}. WSL2's GPU paravirtualization only exposes a DirectX interface — Intel Arc Vulkan requires running natively on Windows.`; } return "Vulkan GPU is not available in WSL2. WSL2's GPU paravirtualization does not support hardware Vulkan for this GPU. Run natively on Windows or Linux for GPU acceleration."; } return "Installed, but this binary did not register a Vulkan backend on this machine."; } let cachedCoreMLPythonPath = null; let cachedCoreMLAvailable = null; function getCoreMLPythonPath() { if (cachedCoreMLAvailable !== null) { return cachedCoreMLPythonPath; } if (osPlatform !== "darwin") { cachedCoreMLAvailable = false; cachedCoreMLPythonPath = null; return null; } // 1. Check environment variable COREML_PYTHON if (process.env.COREML_PYTHON && fs.existsSync(process.env.COREML_PYTHON)) { cachedCoreMLPythonPath = process.env.COREML_PYTHON; cachedCoreMLAvailable = true; return cachedCoreMLPythonPath; } // 2. Check local/in-project venvs const localPaths = [ path.join(ROOT, "app", "backend", "mac", "coreml_venv", "bin", "python"), path.join(ROOT, "app", "backend", "mac", "venv", "bin", "python"), path.join(ROOT, "venv", "bin", "python"), ]; for (const p of localPaths) { if (fs.existsSync(p)) { cachedCoreMLPythonPath = p; cachedCoreMLAvailable = true; return p; } } // 3. Check dynamically in home directory (as fallback to user's setup) const homeDir = os.homedir(); const homePaths = [ path.join(homeDir, "workspace", "coreml_conversion", "venv", "bin", "python"), path.join(homeDir, "ml-stable-diffusion", "venv", "bin", "python"), ]; for (const p of homePaths) { if (fs.existsSync(p)) { cachedCoreMLPythonPath = p; cachedCoreMLAvailable = true; return p; } } // 4. Fallback: Probe system python3 if it has required imports try { const probeResult = spawnSync("python3", [ "-c", "import python_coreml_stable_diffusion, diffusers, transformers" ], { timeout: 2000 }); if (probeResult.status === 0) { // Find absolute path of python3 const whichResult = spawnSync("which", ["python3"], { encoding: "utf8" }); const python3Path = whichResult.stdout ? whichResult.stdout.trim() : "python3"; cachedCoreMLPythonPath = python3Path; cachedCoreMLAvailable = true; return python3Path; } } catch (err) { // python3 not in path or crashed } cachedCoreMLAvailable = false; cachedCoreMLPythonPath = null; return null; } function getCoreMLNpuInfo() { const isAppleSilicon = osPlatform === "darwin" && os.arch() === "arm64"; if (!isAppleSilicon) { return { supported: false, reason: "Apple Silicon ANE (NPU) is only available on Apple Silicon macOS devices." }; } const pythonPath = getCoreMLPythonPath(); if (!pythonPath) { return { supported: true, ready: false, reason: "CoreML Python environment is not set up. Run scripts/setup/setup-coreml-npu.sh first.", }; } return { supported: true, ready: true, python: pythonPath, }; } function getBackendOptions() { if (cachedBackendOptions) return cachedBackendOptions; const cudaInstalled = (osPlatform === "win32" && fs.existsSync(BACKEND_PATHS.cuda)) || (osPlatform === "linux" && fs.existsSync(BACKEND_PATHS.linuxCuda)); const cudaAvailable = cudaInstalled && hasNvidiaGpu() && backendAccepts( osPlatform === "win32" ? BACKEND_PATHS.cuda : BACKEND_PATHS.linuxCuda, "cuda" ); const vulkanInstalled = (osPlatform === "win32" && fs.existsSync(BACKEND_PATHS.vulkan)) || (osPlatform === "linux" && fs.existsSync(BACKEND_PATHS.linuxVulkan)); const vulkanRuntimeAvailable = osPlatform !== "win32" || hasVulkanRuntime(); const vulkanAvailable = vulkanInstalled && vulkanRuntimeAvailable && backendAccepts( osPlatform === "win32" ? BACKEND_PATHS.vulkan : BACKEND_PATHS.linuxVulkan, "vulkan" ); const rocmInstalled = osPlatform === "linux" && fs.existsSync(BACKEND_PATHS.linuxRocm); const rocmAvailable = rocmInstalled && hasAmdGpu() && backendAccepts(BACKEND_PATHS.linuxRocm, "rocm"); const cpuInstalled = (osPlatform === "linux" && fs.existsSync(BACKEND_PATHS.linuxCpu)) || (osPlatform === "win32" && fs.existsSync(BACKEND_PATHS.cpu)); const metalInstalled = osPlatform === "darwin" && fs.existsSync(BACKEND_PATHS.mac); const metalAvailable = metalInstalled; const coremlNpu = getCoreMLNpuInfo(); const openvinoNpu = getOpenVinoNpuInfo(); const openvinoModels = getOpenVinoModelInfo(); const openvinoNpuAvailable = openvinoNpu.supported && openvinoModels.some((model) => model.installed); const options = []; if (cpuInstalled) options.push({ id: "cpu", label: "CPU", available: true }); if (metalAvailable) options.push({ id: "metal", label: "Metal GPU", available: true }); if (coremlNpu.supported && coremlNpu.ready) options.push({ id: "apple-npu", label: "Apple Neural Engine (NPU)", available: true }); if (vulkanAvailable) options.push({ id: "vulkan", label: "Vulkan GPU", available: true }); if (rocmAvailable) options.push({ id: "rocm", label: "ROCm GPU (AMD)", available: true }); if (cudaAvailable) options.push({ id: "cuda", label: "CUDA GPU", available: true }); if (openvinoNpuAvailable) options.push({ id: "openvino-npu", label: "NPU (OpenVINO)", available: true }); const unavailable = []; if (vulkanInstalled && !vulkanAvailable) { unavailable.push({ id: "vulkan", label: "Vulkan GPU", reason: getVulkanUnavailableReason() }); } if (cudaInstalled && !cudaAvailable) { unavailable.push({ id: "cuda", label: "CUDA GPU", reason: "Installed, but CUDA backend validation failed." }); } if (rocmInstalled && !rocmAvailable) { unavailable.push({ id: "rocm", label: "ROCm GPU (AMD)", reason: "Installed, but ROCm backend validation failed." }); } if (metalInstalled && !metalAvailable) { unavailable.push({ id: "metal", label: "Metal GPU", reason: "Installed, but Metal backend validation failed." }); } if (coremlNpu.supported && !coremlNpu.ready) { unavailable.push({ id: "apple-npu", label: "Apple Neural Engine (NPU)", reason: coremlNpu.reason }); } if (!cpuInstalled && (osPlatform === "win32" || osPlatform === "linux")) { unavailable.push({ id: "cpu", label: "CPU", reason: osPlatform === "win32" ? "Windows CPU backend is not installed. Run scripts/setup/setup.ps1 to install app/backend/win/cpu." : "Linux CPU backend is not installed. Run setup again to install app/backend/linux/cpu.", }); } if (openvinoNpu.supported && !openvinoNpuAvailable) { unavailable.push({ id: "openvino-npu", label: "NPU (OpenVINO)", reason: "Runtime is ready, but no OpenVINO NPU model is downloaded." }); } else if (!openvinoNpu.supported && hasNpuHardware() && (osPlatform === "win32" || osPlatform === "linux")) { unavailable.push({ id: "openvino-npu", label: "NPU (OpenVINO)", reason: openvinoNpu.reason }); } let defaultBackend = "cpu"; if (coremlNpu.supported && coremlNpu.ready) { defaultBackend = "apple-npu"; } else if (metalAvailable) { defaultBackend = "metal"; } else if (cudaAvailable) { const gpuName = String(getGpuInfo().name).toLowerCase(); const isGtxCard = gpuName.includes("gtx"); if (isGtxCard && vulkanAvailable) { defaultBackend = "vulkan"; // Default to Vulkan for GTX cards because of lack of Tensor Cores } else { defaultBackend = "cuda"; } } else if (rocmAvailable) { defaultBackend = "rocm"; } else if (vulkanAvailable) { defaultBackend = "vulkan"; } cachedBackendOptions = { options, unavailable, cudaAvailable, vulkanAvailable, rocmAvailable, metalAvailable, openvinoNpuAvailable, openvinoNpu, openvinoModels, coremlNpu, defaultBackendType: defaultBackend, }; return cachedBackendOptions; } function backendAccepts(binaryPath, backendName) { if (!binaryPath || !fs.existsSync(binaryPath)) return false; try { const cliBackendName = backendName; let probeArgs = [ "--backend", cliBackendName, "--params-backend", cliBackendName, "--model", path.join(MODELS, "__backend_probe_missing__.safetensors"), "--listen-port", "18082", ]; if (cliBackendName === "metal") { probeArgs = [ "--model", path.join(MODELS, "__backend_probe_missing__.safetensors"), "--listen-port", "18082", ]; } const spawnEnv = { ...process.env }; if (osPlatform === "linux") { const dir = path.dirname(binaryPath); const existing = spawnEnv.LD_LIBRARY_PATH || ""; spawnEnv.LD_LIBRARY_PATH = dir + (existing ? ":" + existing : ""); } else if (osPlatform === "darwin") { const dir = path.dirname(binaryPath); const existing = spawnEnv.DYLD_LIBRARY_PATH || ""; spawnEnv.DYLD_LIBRARY_PATH = dir + (existing ? ":" + existing : ""); } let result = spawnSync(binaryPath, probeArgs, { env: spawnEnv, encoding: "utf8", timeout: 5000 }); let output = `${result.stdout || ""}\n${result.stderr || ""}`; let supportsFlags = true; // Some binaries do not support --backend. If we see "unknown argument", // retry without backend flags so we can still verify the binary launches. if (output.includes("unknown argument") && output.includes("--backend")) { supportsFlags = false; const fallbackArgs = [ "--model", path.join(MODELS, "__backend_probe_missing__.safetensors"), "--listen-port", "18082", ]; result = spawnSync(binaryPath, fallbackArgs, { env: spawnEnv, encoding: "utf8", timeout: 5000 }); output = `${result.stdout || ""}\n${result.stderr || ""}`; } const lower = output.toLowerCase(); if (lower.includes("backend config failed") || output.includes(`backend '${backendName}' was not found`) || output.includes(`backend '${cliBackendName}' was not found`)) { return false; } // Reject binaries that fail at the dynamic linker / glibc level. if (lower.includes("glibc") || lower.includes("libc.so") || lower.includes("libstdc++") || lower.includes("cannot open shared object")) { return false; } // A healthy binary prints project-specific log lines when it tries to load the model. const isOk = lower.includes("stable-diffusion.cpp") || lower.includes("loading model"); if (isOk) { backendSupportsFlags[binaryPath] = supportsFlags; } return isOk; } catch (_) { return false; } } function selectBackendPath(useGpu, backendType = "auto", modelPath = "") { const resolvedType = resolveBackendType(useGpu, backendType, modelPath); if (osPlatform === "win32") { if (resolvedType === "cuda" && fs.existsSync(BACKEND_PATHS.cuda)) return BACKEND_PATHS.cuda; if (resolvedType === "cpu" && fs.existsSync(BACKEND_PATHS.cpu)) return BACKEND_PATHS.cpu; if (resolvedType === "vulkan" && fs.existsSync(BACKEND_PATHS.vulkan)) return BACKEND_PATHS.vulkan; if (fs.existsSync(BACKEND_PATHS.cuda)) return BACKEND_PATHS.cuda; if (fs.existsSync(BACKEND_PATHS.cpu)) return BACKEND_PATHS.cpu; if (fs.existsSync(BACKEND_PATHS.vulkan)) return BACKEND_PATHS.vulkan; return BACKEND_PATH; } if (osPlatform === "linux") { let finalType = resolvedType; if (finalType === "cuda") { if (fs.existsSync(BACKEND_PATHS.linuxCuda)) { return BACKEND_PATHS.linuxCuda; } finalType = "vulkan"; } if (finalType === "rocm" && fs.existsSync(BACKEND_PATHS.linuxRocm)) return BACKEND_PATHS.linuxRocm; if (resolvedType === "vulkan" && fs.existsSync(BACKEND_PATHS.linuxVulkan)) return BACKEND_PATHS.linuxVulkan; if (resolvedType === "cpu" && fs.existsSync(BACKEND_PATHS.linuxCpu)) return BACKEND_PATHS.linuxCpu; if (fs.existsSync(BACKEND_PATHS.linuxVulkan)) return BACKEND_PATHS.linuxVulkan; if (fs.existsSync(BACKEND_PATHS.linuxCpu)) return BACKEND_PATHS.linuxCpu; return BACKEND_PATH; } if (osPlatform === "darwin") { if (resolvedType === "apple-npu") return getCoreMLPythonPath(); if (fs.existsSync(BACKEND_PATHS.mac)) return BACKEND_PATHS.mac; return BACKEND_PATH; } // generic fallback if (fs.existsSync(BACKEND_PATHS.mac)) return BACKEND_PATHS.mac; return BACKEND_PATH; } function resolveBackendType(useGpu, backendType = "auto", modelPath = "") { const options = getBackendOptions(); let requestedType = useGpu === false ? "cpu" : backendType === "auto" ? options.defaultBackendType : backendType; if (modelPath && osPlatform === "darwin") { const lower = modelPath.toLowerCase(); const isDir = fs.existsSync(modelPath) && fs.statSync(modelPath).isDirectory(); if (lower.endsWith(".safetensors") || lower.endsWith(".gguf")) { if (requestedType === "apple-npu") requestedType = "metal"; } else if (lower.endsWith(".coreml") || lower.endsWith(".mlpackage") || lower.endsWith(".mlmodelc") || isDir) { if (requestedType === "metal") requestedType = "apple-npu"; } } const available = new Set(options.options.map(option => option.id)); return available.has(requestedType) ? requestedType : options.defaultBackendType; } function getBackendMode(backendPath, useGpu, backendType = "auto") { if (useGpu === false || backendType === "cpu") return "CPU"; const name = path.basename(backendPath || "").toLowerCase(); if (name.includes("cuda")) return "CUDA GPU"; if (name.includes("rocm")) return "ROCm GPU"; if (name.includes("vulkan")) return "Vulkan GPU"; if (backendType === "apple-npu") return "Apple NPU"; if (osPlatform === "darwin" || backendType === "metal") return "Metal GPU"; return "GPU"; } function stripAnsi(value) { return String(value || "").replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "").replace(/\x1b\[[A-Z]/g, ""); } function markBackendReady() { if (backendReady) return; backendReady = true; backendLoadState = { ...backendLoadState, active: false, phase: "Model ready", progress: 100, }; console.log(" [backend] READY on port", PORT_BACKEND); // Force a single VRAM poll after the model settles in memory setTimeout(() => pollNvidiaVram(true), 1500); } function pingBackendReady() { return new Promise(resolve => { const req = http.get(`http://127.0.0.1:${PORT_BACKEND}/v1/models`, res => { res.resume(); resolve(res.statusCode >= 200 && res.statusCode < 500); }); req.setTimeout(1000, () => { req.destroy(); resolve(false); }); req.on("error", () => resolve(false)); }); } function requestJson(url, payload = null, timeoutMs = 120000) { return new Promise((resolve, reject) => { const parsed = new URL(url); const data = payload ? Buffer.from(JSON.stringify(payload), "utf8") : null; const req = http.request({ hostname: parsed.hostname, port: parsed.port, path: parsed.pathname + parsed.search, method: data ? "POST" : "GET", headers: data ? { "Content-Type": "application/json", "Content-Length": data.length, } : {}, timeout: timeoutMs, }, (res) => { let body = ""; res.setEncoding("utf8"); res.on("data", (chunk) => { body += chunk; }); res.on("end", () => { try { const jsonBody = JSON.parse(body || "{}"); if (res.statusCode < 200 || res.statusCode >= 300 || jsonBody.ok === false) { reject(new Error(jsonBody.error || `HTTP ${res.statusCode}`)); return; } resolve(jsonBody); } catch (err) { reject(err); } }); }); req.on("error", reject); req.on("timeout", () => req.destroy(new Error("Request timed out"))); if (data) req.write(data); req.end(); }); } function proxyImageBackendRequest(req, res) { const headers = { ...req.headers }; delete headers.host; headers.host = `127.0.0.1:${PORT_BACKEND}`; const proxyReq = http.request({ hostname: "127.0.0.1", port: PORT_BACKEND, path: req.url, method: req.method, headers, timeout: 300000, }, (proxyRes) => { res.writeHead(proxyRes.statusCode || 502, { ...proxyRes.headers, "Access-Control-Allow-Origin": "*", }); proxyRes.pipe(res); }); proxyReq.on("error", (err) => { json(res, 502, { ok: false, error: `Image backend is not reachable: ${err.message}` }); }); proxyReq.on("timeout", () => { proxyReq.destroy(new Error("Image backend request timed out")); }); req.pipe(proxyReq); } function requestHttpsJson(url, timeoutMs = 30000) { return new Promise((resolve, reject) => { const req = https.get(url, { headers: { "Accept": "application/json", "User-Agent": "Uncensored-AI-Studio/1.0", }, timeout: timeoutMs, }, (res) => { let body = ""; res.setEncoding("utf8"); res.on("data", (chunk) => { body += chunk; }); res.on("end", () => { try { const parsed = JSON.parse(body || "null"); if (res.statusCode < 200 || res.statusCode >= 300) { reject(new Error(parsed?.error || `Hugging Face returned HTTP ${res.statusCode}`)); return; } resolve(parsed); } catch (err) { reject(new Error(`Hugging Face returned invalid JSON: ${err.message}`)); } }); }); req.on("error", reject); req.on("timeout", () => req.destroy(new Error("Hugging Face request timed out"))); }); } function getModelParameterBillions(value) { const text = String(value || ""); const matches = [...text.matchAll(/(?:^|[-_/ ])(\d+(?:\.\d+)?)\s*b(?:[-_/ ]|$)/gi)]; if (matches.length === 0) return null; return Math.min(...matches.map((match) => Number(match[1])).filter(Number.isFinite)); } function selectGgufFile(siblings = [], tier = "low") { const files = siblings .map((item) => typeof item === "string" ? item : item?.rfilename) .filter((name) => name && /\.gguf$/i.test(name)) .filter((name) => !/(?:^|\/)(?:mmproj|mtp)(?:[-_/]|$)|-\d{5}-of-\d{5}\.gguf$/i.test(name)); const preferences = tier === "high" ? [/q6_k\.gguf$/i, /q5_k_m\.gguf$/i, /q4_k_m\.gguf$/i, /q8_0\.gguf$/i] : [/q4_k_m\.gguf$/i, /q4_k_s\.gguf$/i, /q4_0\.gguf$/i, /q3_k_m\.gguf$/i, /iq4_xs\.gguf$/i]; for (const pattern of preferences) { const match = files.find((name) => pattern.test(name)); if (match) return match; } return files[0] || ""; } function selectMmprojFile(siblings = [], modelFilename = "") { const files = siblings .map((item) => typeof item === "string" ? item : item?.rfilename) .filter((name) => name && /\.gguf$/i.test(name)) .filter((name) => /(?:^|[-_/])mmproj(?:[-_.\/]|$)|mmproj/i.test(name)); if (files.length === 0) return ""; const modelBase = path.basename(modelFilename).toLowerCase(); const modelStem = modelBase.replace(/\.gguf$/i, "").replace(/\.(q\d(?:_k_[ms])?|iq\d_[a-z0-9]+|bf16|f16|f32)$/i, ""); const scored = files.map((name) => { const lower = name.toLowerCase(); let score = 0; if (/bf16/i.test(lower)) score += 6; if (/f16/i.test(lower)) score += 5; if (/mmproj-model-f16/i.test(lower)) score += 4; if (modelStem && lower.includes(modelStem)) score += 10; return { name, score }; }); scored.sort((a, b) => b.score - a.score || a.name.length - b.name.length); return scored[0]?.name || ""; } function parseHuggingFaceResolveUrl(fileUrl) { try { const parsed = new URL(fileUrl); if (!/^(?:www\.)?huggingface\.co$/i.test(parsed.hostname)) return null; const parts = parsed.pathname.split("/").filter(Boolean); const resolveIndex = parts.findIndex((part) => part === "resolve" || part === "blob"); if (resolveIndex < 2 || resolveIndex + 2 >= parts.length) return null; const repoId = `${decodeURIComponent(parts[0])}/${decodeURIComponent(parts[1])}`; const revision = decodeURIComponent(parts[resolveIndex + 1] || "main"); const repositoryFilename = parts.slice(resolveIndex + 2).map(decodeURIComponent).join("/"); return { repoId, revision, repositoryFilename }; } catch (_) { return null; } } async function resolveHuggingFaceProjector(fileUrl, localModelFilename = "") { const parsed = parseHuggingFaceResolveUrl(fileUrl); if (!parsed) return null; const cacheKey = `${parsed.repoId}|${parsed.revision}|${parsed.repositoryFilename}`; const cached = hfProjectorCache.get(cacheKey); if (cached && Date.now() - cached.createdAt < HF_MODEL_CACHE_TTL_MS) return cached.projector; try { const treeUrl = `https://huggingface.co/api/models/${parsed.repoId}/tree/${encodeURIComponent(parsed.revision)}?recursive=true`; const tree = await requestHttpsJson(treeUrl); const projectorRepositoryFilename = selectMmprojFile(tree, parsed.repositoryFilename || localModelFilename); const projector = projectorRepositoryFilename ? { url: `https://huggingface.co/${parsed.repoId}/resolve/${encodeURIComponent(parsed.revision)}/${projectorRepositoryFilename.split("/").map(encodeURIComponent).join("/")}`, filename: `${parsed.repoId.replace(/\//g, "--")}--${path.basename(projectorRepositoryFilename)}`, repositoryFilename: projectorRepositoryFilename, } : null; hfProjectorCache.set(cacheKey, { createdAt: Date.now(), projector }); return projector; } catch (err) { console.warn(` [download] Could not resolve vision projector for ${parsed.repoId}: ${err.message || err}`); hfProjectorCache.set(cacheKey, { createdAt: Date.now(), projector: null }); return null; } } function classifyHuggingFaceModel(model, filename) { const searchable = `${model.id || ""} ${filename || ""} ${(model.tags || []).join(" ")}`.toLowerCase(); const parameters = getModelParameterBillions(searchable); return { potato: parameters !== null && parameters <= 3, vision: /(?:vision|llava|multimodal|(?:^|[-_/ ])vl(?:[-_/ ]|$)|moondream)/i.test(searchable), uncensored: /uncensored|abliterated|heretic/i.test(searchable), parameters, }; } function getTextModelFit(sizeBytes, specs = getHardwareSpecs()) { if (!sizeBytes) { return { recommended: false, mode: "unknown", label: "", reason: "File size is unavailable, so compatibility cannot be estimated." }; } const ramBytes = Math.max(0, Number(specs.ram_total_gb) || 0) * (1024 ** 3); const vramBytes = Math.max(0, Number(specs.gpu_vram_gb) || 0) * (1024 ** 3); const isAppleSilicon = osPlatform === "darwin" && /apple/i.test(String(specs.cpu_name || "")); const systemReserve = Math.max(4 * (1024 ** 3), ramBytes * 0.3); const usableRam = Math.max(0, ramBytes - systemReserve); const runtimeNeed = (sizeBytes * 1.25) + (1.5 * (1024 ** 3)); const fastGpuFit = vramBytes >= runtimeNeed; const cpuOrOffloadFit = usableRam >= runtimeNeed; const unifiedMemoryFit = isAppleSilicon && usableRam >= runtimeNeed; const hybridFit = !fastGpuFit && vramBytes > 0 && cpuOrOffloadFit; const recommended = fastGpuFit || cpuOrOffloadFit || unifiedMemoryFit; if (recommended) { const mode = fastGpuFit ? "GPU memory" : isAppleSilicon ? "unified memory" : hybridFit ? "combined GPU and system memory" : "system RAM"; return { recommended: true, mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : hybridFit ? "hybrid" : "ram", label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : hybridFit ? "GPU + RAM Fit" : "Fits in RAM", reason: `${formatBytes(sizeBytes)} weights fit the estimated ${mode} budget with runtime headroom.`, }; } return { recommended: false, mode: "too-large", label: "", reason: `${formatBytes(sizeBytes)} weights need about ${formatBytes(runtimeNeed)} including runtime headroom; this computer has about ${formatBytes(usableRam)} usable RAM and ${formatBytes(vramBytes)} VRAM.`, }; } async function addHuggingFaceFileSize(model) { try { const detail = await requestHttpsJson(`https://huggingface.co/api/models/${model.id}?blobs=true`); const file = (detail.siblings || []).find((item) => item?.rfilename === model.repositoryFilename); const sizeBytes = Number(file?.size || file?.lfs?.size || 0); const fit = getTextModelFit(sizeBytes); return { ...model, sizeBytes, size: sizeBytes > 0 ? formatBytes(sizeBytes) : "Unknown", approxSize: sizeBytes > 0 ? formatBytes(sizeBytes) : model.approxSize, recommendedFit: fit.recommended, fitMode: fit.mode, fitLabel: fit.label, fitReason: fit.reason, }; } catch (_) { return { ...model, sizeBytes: 0, size: "Unknown", recommendedFit: false, fitMode: "unknown", fitLabel: "", fitReason: "File size is unavailable, so compatibility cannot be estimated.", }; } } async function searchHuggingFaceModels(query, filters) { const specs = getHardwareSpecs(); const tier = specs.gpu_vram_gb >= 10 || specs.ram_total_gb >= 32 ? "high" : specs.gpu_vram_gb >= 5 || specs.ram_total_gb >= 16 ? "mid" : "low"; const defaultSearch = tier === "low" ? "1B instruct" : tier === "mid" ? "7B instruct" : "8B instruct"; let searchTerms = ""; if (query.trim()) { searchTerms = [ query.trim(), filters.includes("vision") ? "vision" : "", filters.includes("uncensored") ? "uncensored" : "", ].filter(Boolean).join(" "); } else { if (filters.includes("vision") && filters.includes("uncensored")) { searchTerms = "vision uncensored gguf"; } else if (filters.includes("vision")) { searchTerms = "vision gguf"; } else if (filters.includes("uncensored")) { searchTerms = "uncensored gguf"; } else { searchTerms = defaultSearch; } } const cacheKey = `${tier}|${searchTerms}|${filters.sort().join(",")}`; const cached = hfModelCache.get(cacheKey); if (cached && Date.now() - cached.createdAt < HF_MODEL_CACHE_TTL_MS) return cached.models; const params = new URLSearchParams({ filter: "gguf", search: searchTerms, sort: "downloads", direction: "-1", limit: "50", full: "true", }); const results = await requestHttpsJson(`https://huggingface.co/api/models?${params.toString()}`); const models = []; for (const model of Array.isArray(results) ? results : []) { if (model.private || model.gated || model.disabled) continue; const filename = selectGgufFile(model.siblings, tier); if (!filename) continue; const projectorFilename = selectMmprojFile(model.siblings, filename); const traits = classifyHuggingFaceModel(model, filename); if (filters.includes("vision") && !traits.vision) continue; if (filters.includes("uncensored") && !traits.uncensored) continue; const isLikelyChatModel = /instruct|chat|assistant|coder|uncensored|abliterated|vision|llava|\bvl\b/i.test(`${model.id} ${filename}`); if (!query.trim() && !isLikelyChatModel) continue; const normalizedQuery = query.trim().toLowerCase().replace(/[-_/]+/g, " ").replace(/\s+/g, " "); const normalizedModelName = `${model.id} ${filename}`.toLowerCase().replace(/[-_/]+/g, " ").replace(/\s+/g, " "); const queryWords = normalizedQuery.split(" ").filter(Boolean); const exactPhraseMatch = normalizedQuery && normalizedModelName.includes(normalizedQuery); const matchedQueryWords = queryWords.filter((word) => normalizedModelName.includes(word)).length; const score = Number(model.downloads || 0) + (traits.parameters && traits.parameters <= 3 && tier === "low" ? 10000000 : 0) + (traits.parameters && tier === "mid" && traits.parameters >= 4 && traits.parameters <= 9 ? 10000000 : 0) + (traits.parameters && tier === "high" && traits.parameters >= 7 && traits.parameters <= 14 ? 10000000 : 0) + (exactPhraseMatch ? 100000000 : 0) + (matchedQueryWords * 5000000); models.push({ id: model.id, name: String(model.id || "").split("/").pop().replace(/[-_]+/g, " "), filename: `${model.id.replace(/\//g, "--")}--${path.basename(filename)}`, repositoryFilename: filename, format: "GGUF", approxSize: traits.parameters ? `~${traits.parameters}B parameters` : "Size shown before download", resolution: "N/A", notes: `Community GGUF from ${String(model.id || "").split("/")[0]}. ${Number(model.downloads || 0).toLocaleString()} Hugging Face downloads.`, url: `https://huggingface.co/${model.id}/resolve/main/${filename.split("/").map(encodeURIComponent).join("/")}`, projectorUrl: projectorFilename ? `https://huggingface.co/${model.id}/resolve/main/${projectorFilename.split("/").map(encodeURIComponent).join("/")}` : "", projectorFilename: projectorFilename ? `${model.id.replace(/\//g, "--")}--${path.basename(projectorFilename)}` : "", pageUrl: `https://huggingface.co/${model.id}`, downloads: Number(model.downloads || 0), likes: Number(model.likes || 0), tags: Object.entries(traits).filter(([key, value]) => key !== "parameters" && value).map(([key]) => key), score, }); } const ranked = models.sort((a, b) => b.score - a.score); hfModelCache.set(cacheKey, { createdAt: Date.now(), models: ranked }); return ranked; } async function killOpenVinoWorker() { if (!openvinoProc) { openvinoReady = false; return; } console.log(" [openvino-npu] Stopping worker..."); try { openvinoProc.kill("SIGTERM"); } catch (_) {} await new Promise((resolve) => setTimeout(resolve, 800)); try { openvinoProc.kill("SIGKILL"); } catch (_) {} openvinoProc = null; openvinoReady = false; } function getOpenVinoResolution(settings = {}) { const width = Number(settings.width) || 512; const height = Number(settings.height) || 512; const supported = (width === 512 && height === 512) || (width === 1024 && height === 1024); if (!supported) { throw new Error("OpenVINO NPU supports 512x512 generation or 1024x1024 HD upscale."); } return { width, height }; } async function startOpenVinoWorker(settings = {}) { const npuInfo = getOpenVinoNpuInfo(); if (!npuInfo.supported) throw new Error(npuInfo.reason || "OpenVINO NPU is not available."); const requestedModel = settings.model ? findOpenVinoModel(settings.model) : null; if (settings.model && !requestedModel) { throw new Error("OpenVINO NPU requires a downloaded OpenVINO model. Standard .safetensors/.ckpt weights must use Vulkan, CUDA, or CPU."); } const model = requestedModel || getOpenVinoModelInfo().find((item) => item.installed); if (!model || !model.installed) { throw new Error("OpenVINO NPU model is not downloaded. Download an OpenVINO NPU model from Model Manager first."); } const { width, height } = getOpenVinoResolution(settings); if (openvinoProc && openvinoReady && openvinoModel === model.id) { currentSettings = { ...currentSettings, ...settings, model: model.id, width, height }; return; } await killBackend(); await killOpenVinoWorker(); openvinoError = null; openvinoReady = false; openvinoModel = model.id; openvinoWidth = 512; openvinoHeight = 512; openvinoPort = await findAvailableBackendPort(); PORT_BACKEND = openvinoPort; currentSettings = { ...currentSettings, ...settings }; currentSettings.backendType = "openvino-npu"; currentSettings.useGpu = true; currentSettings.backendMode = "NPU (OpenVINO)"; currentSettings.backendBinary = "openvino_npu_worker.py"; currentSettings.backendDevice = npuInfo.npu?.name || "Intel AI Boost"; currentSettings.model = model.id; currentSettings.width = width; currentSettings.height = height; backendError = null; backendLoadState = { active: true, phase: "Compiling OpenVINO NPU pipeline (512x512)...", progress: 5, current: 0, total: 0, speed: "", model: model.name, backendMode: "NPU (OpenVINO)", backendBinary: "openvino_npu_worker.py", device: currentSettings.backendDevice, }; const workerPath = path.join(ROOT, "scripts", "workers", "openvino_npu_worker.py"); const cacheDir = path.join(ROOT, "app", "tools", "openvino-cache", "512x512"); console.log(` [openvino-npu] Starting 512x512 worker on port ${openvinoPort}`); openvinoProc = spawn(npuInfo.python, [ workerPath, "--model-dir", model.path, "--port", String(openvinoPort), "--width", "512", "--height", "512", "--cache-dir", cacheDir, ], { stdio: "pipe" }); let openvinoStdoutBuffer = ""; openvinoProc.stdout.on("data", (data) => { const text = data.toString(); process.stdout.write(" " + text); openvinoStdoutBuffer += text; const lines = openvinoStdoutBuffer.split(/\r?\n/); openvinoStdoutBuffer = lines.pop() || ""; for (const line of lines) { if (line.includes("READY")) { openvinoReady = true; backendReady = true; backendLoadState = { ...backendLoadState, active: false, phase: "OpenVINO NPU model ready", progress: 100 }; } const progressMatch = line.match(/PROGRESS\s+(\d+)\/(\d+)\s+(.+)$/); if (progressMatch) { generationState.active = true; generationState.step = Number(progressMatch[1]); generationState.steps = Number(progressMatch[2]); generationState.speed = progressMatch[3].trim(); generationState.decoding = false; } else if (line.includes("DECODING") && generationState.active) { generationState.step = generationState.steps; generationState.decoding = true; generationState.speed = ""; } } }); openvinoProc.stderr.on("data", (data) => { const text = data.toString(); process.stderr.write(" " + text); const cleanText = stripAnsi(text).trim(); if (cleanText && (cleanText.includes("ERROR") || cleanText.includes("Traceback") || cleanText.includes("Exception"))) { openvinoError = cleanText; backendError = openvinoError; } }); openvinoProc.on("exit", (code) => { console.log(" [openvino-npu] worker exited with code", code); openvinoProc = null; openvinoReady = false; backendReady = false; if (code !== 0 && code !== null && !backendError) backendError = `OpenVINO NPU worker exited with code ${code}`; backendLoadState.active = false; }); const startedAt = Date.now(); while (Date.now() - startedAt < 360000) { if (openvinoReady) return; if (backendError) throw new Error(backendError); if (!openvinoProc) throw new Error("OpenVINO NPU worker exited during startup."); const elapsed = Math.round((Date.now() - startedAt) / 1000); backendLoadState.progress = Math.min(95, 5 + Math.round(elapsed / 2)); await new Promise((resolve) => setTimeout(resolve, 1000)); } await killOpenVinoWorker(); throw new Error("OpenVINO NPU 512x512 compile timed out after 6 minutes."); } async function generateWithOpenVino(body) { if (!openvinoReady || !openvinoPort) throw new Error("OpenVINO NPU worker is not ready. Load an OpenVINO NPU model first."); const steps = Math.max(1, Math.min(8, Number(body.steps) || 4)); generationState = { active: true, step: 0, steps, speed: "", decoding: false }; try { const result = await requestJson(`http://127.0.0.1:${openvinoPort}/generate`, { prompt: body.prompt, negative_prompt: body.negative_prompt || "", width: Number(body.width) || currentSettings.width || 512, height: Number(body.height) || currentSettings.height || 512, steps, cfg_scale: Number(body.cfg_scale) || 1.0, seed: body.seed, }, 300000); resetGenerationState(); return result; } catch (err) { resetGenerationState(); throw err; } } function startBackendReadyPoll() { let attempts = 0; const isNpu = currentSettings.backendType === "apple-npu" || currentSettings.backendType === "openvino-npu"; const maxAttempts = isNpu ? 1200 : 240; const interval = setInterval(async () => { attempts += 1; if (!backendProc || backendReady || attempts > maxAttempts) { clearInterval(interval); return; } if (await pingBackendReady()) { markBackendReady(); clearInterval(interval); } }, 500); } function startCoreMLLoadHeartbeat(proc) { if (currentSettings.backendType !== "apple-npu") return null; const startedAt = Date.now(); const interval = setInterval(() => { if (backendProc !== proc || backendReady || !backendLoadState.active) { clearInterval(interval); return; } const elapsedSec = (Date.now() - startedAt) / 1000; const targetProgress = Math.min(96, Math.round(8 + 88 * (1 - Math.exp(-elapsedSec / 55)))); if (targetProgress > (backendLoadState.progress || 0)) { backendLoadState = { ...backendLoadState, active: true, phase: backendLoadState.phase || "Loading Core ML model...", progress: targetProgress, }; } }, 1000); return interval; } function getDefaultModel() { try { const files = fs.readdirSync(MODELS).filter(isModelFile); return files.length ? path.join(MODELS, files[0]) : null; } catch (_) { return null; } } function killBackend() { return new Promise(resolve => { resetGenerationState(); const proc = backendProc; if (!proc) { backendUnloadState = { active: false, phase: "", progress: 0 }; resolve(); return; } backendUnloadState = { active: true, phase: "Stopping backend process...", progress: 10 }; backendReady = false; try { proc.kill("SIGTERM"); } catch (_) {} backendUnloadState = { active: true, phase: "Waiting for process exit...", progress: 50 }; setTimeout(() => { try { proc.kill("SIGKILL"); } catch (_) {} if (backendProc === proc) backendProc = null; backendUnloadState = { active: false, phase: "Backend unloaded", progress: 100 }; resolve(); }, 2000); }); } function chooseAutoContext(modelFilename, isGpu) { let modelSizeGb = 4; try { const modelPath = path.join(LLM_MODELS, modelFilename); if (fs.existsSync(modelPath)) { const stats = fs.statSync(modelPath); modelSizeGb = stats.size / (1024 * 1024 * 1024); } } catch (_) {} const lowerFilename = String(modelFilename || "").toLowerCase(); const isVisionModel = isVisionModelFilename(lowerFilename); let projectorSizeGb = 0; if (isVisionModel) { try { const files = fs.readdirSync(LLM_MODELS); const projector = files.find((file) => { const lower = file.toLowerCase(); return lower.endsWith(".gguf") && lower.includes("mmproj"); }); if (projector) { projectorSizeGb = fs.statSync(path.join(LLM_MODELS, projector)).size / (1024 * 1024 * 1024); } } catch (_) {} } const systemRamGb = os.totalmem() / (1024 * 1024 * 1024); let vramGb = 0; try { const gpu = getGpuInfo(); if (gpu && gpu.vram_gb) { vramGb = gpu.vram_gb; } } catch (_) {} let availableGb = 0; let bufferGb = 2.0; if (isGpu && vramGb > 0) { availableGb = vramGb; bufferGb = 1.5; } else { availableGb = systemRamGb; bufferGb = 2.5; } if (isGpu && vramGb > 0) { const gpuBudgetGb = Math.max(0.5, vramGb - modelSizeGb - projectorSizeGb - bufferGb); const maxFastContext = vramGb < 10 ? 8192 : vramGb < 16 ? 16384 : 32768; if (isVisionModel) { return Math.min(maxFastContext, gpuBudgetGb >= 2.5 ? 8192 : 4096); } if (gpuBudgetGb < 1.5) return 4096; if (gpuBudgetGb < 3.0) return Math.min(maxFastContext, 8192); return Math.min(maxFastContext, 16384); } const usableGb = Math.max(0.5, availableGb - modelSizeGb - projectorSizeGb - bufferGb); let kvPer4096Gb = 0.55; if (modelSizeGb >= 5.5) { kvPer4096Gb = 1.05; } else if (modelSizeGb >= 4.0) { kvPer4096Gb = 0.85; } const estimatedMaxCtx = (usableGb / kvPer4096Gb) * 4096; const contextLadder = [32768, 24576, 16384, 12288, 8192, 4096, 2048]; for (const limit of contextLadder) { if (limit <= estimatedMaxCtx) { return limit; } } return 2048; } function isVisionModelFilename(filename = "") { const lower = String(filename || "").toLowerCase(); return lower.includes("llava") || lower.includes("vision") || lower.includes("qwen2vl") || lower.includes("qwen2-vl") || lower.includes("qwen3.5-4b-vision") || lower.includes("gemma-4") || lower.includes("gemma4") || lower.includes("-e2b-") || lower.includes("e2b-it") || lower.includes("paligemma") || lower.includes("minicpm-v") || lower.includes("internvl") || lower.includes("phi-3-vision") || lower.includes("phi3-vision") || lower.includes("smolvlm") || lower.includes("moondream") || lower === "ggml-model-q4_k.gguf"; } function buildLlmLoadProfiles(settings = {}, backend = {}) { const requestedContext = Number(settings.contextSize); const safeContext = requestedContext && requestedContext > 0 ? Math.max(512, Math.min(4096, requestedContext)) : 4096; const isCpu = backend.key === "cpu" || backend.mode === "CPU"; const profiles = [ { name: "requested", gpuLayers: isCpu ? 0 : undefined }, { name: "safe", contextSize: safeContext, gpuLayers: isCpu ? 0 : 0, flashAttn: false, cacheTypeK: "q4_0", cacheTypeV: "q4_0", batchSize: 256, ubatchSize: 256, enableThinking: false, }, ]; return profiles; } async function startLlm(settings = {}) { const filename = path.basename(String(settings.model || "")); const modelPath = path.join(LLM_MODELS, filename); if (!filename || !pathInside(modelPath, LLM_MODELS) || !fs.existsSync(modelPath)) { throw new Error("Select a downloaded GGUF text model first."); } if (!filename.toLowerCase().endsWith(".gguf")) { throw new Error("Text generation requires a .gguf model."); } const candidates = getLlmBackendCandidates(); if (candidates.length === 0) { throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend."); } const persistedSettings = getLlmModelSettings(filename); const preferredBackend = String(settings.preferredBackend || persistedSettings.preferredBackend || "").toLowerCase(); const sortedCandidates = preferredBackend ? [...candidates].sort((a, b) => { if (a.key === preferredBackend && b.key !== preferredBackend) return -1; if (b.key === preferredBackend && a.key !== preferredBackend) return 1; return 0; }) : candidates; const failures = []; for (const backend of sortedCandidates) { for (const profile of buildLlmLoadProfiles(settings, backend)) { try { await startLlmWithBackend({ ...settings, __loadProfile: profile }, backend); llmSettings.backendFallbacks = failures; updateLlmModelSettings(filename, { preferredBackend: backend.key, lastBackendMode: llmSettings.backendMode, lastBackendBinary: llmSettings.backendBinary, lastSettings: { threads: llmSettings.threads, contextSize: llmSettings.contextSize, gpuLayers: llmSettings.gpuLayers, cacheTypeK: llmSettings.cacheTypeK, cacheTypeV: llmSettings.cacheTypeV, batchSize: llmSettings.batchSize, ubatchSize: llmSettings.ubatchSize, flashAttn: llmSettings.flashAttn, }, }); return; } catch (err) { const message = err.message || String(err); failures.push({ backend: backend.mode, binary: path.basename(backend.path), profile: profile.name, error: message.slice(-500), }); console.warn(` [llm] Load failed on ${backend.mode} (${profile.name}): ${message}`); await killLlm(); } } } const last = failures[failures.length - 1]; llmSettings.backendFallbacks = failures; throw new Error(`Text model failed on all available llama.cpp backends. Last failure: ${last?.error || "unknown error"}`); } async function startLlmWithBackend(settings = {}, backend) { const filename = path.basename(String(settings.model || "")); const modelPath = path.join(LLM_MODELS, filename); if (!filename || !pathInside(modelPath, LLM_MODELS) || !fs.existsSync(modelPath)) { throw new Error("Select a downloaded GGUF text model first."); } if (!filename.toLowerCase().endsWith(".gguf")) { throw new Error("Text generation requires a .gguf model."); } if (!backend) { throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend."); } assertNoOtherActiveRuntime("text", filename); await killBackend(); await killOpenVinoWorker(); await killLlm(); PORT_LLM = await findAvailableLlmPort(); llmError = null; const loadProfile = settings.__loadProfile || {}; let contextSize = Number(loadProfile.contextSize ?? settings.contextSize); if (!contextSize || contextSize <= 0) { const isGpu = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.startsWith("Auto"); contextSize = chooseAutoContext(filename, isGpu); console.log(` [llm] Auto-selected context size: ${contextSize} tokens based on memory limits.`); } else { contextSize = Math.max(512, Math.min(32768, contextSize)); } const isSyclBackend = backend.mode.includes("SYCL") || path.basename(path.dirname(backend.path)).toLowerCase() === "sycl"; const requestedGpuLayers = Number.isFinite(Number(loadProfile.gpuLayers ?? settings.gpuLayers)) ? Number(loadProfile.gpuLayers ?? settings.gpuLayers) : -1; const effectiveGpuLayers = isSyclBackend ? getStableSyclGpuLayers(filename, requestedGpuLayers) : requestedGpuLayers; const effectiveFlashAttn = isSyclBackend ? false : (loadProfile.flashAttn ?? (settings.flashAttn !== false)); const effectiveCacheTypeK = isSyclBackend ? "q4_0" : loadProfile.cacheTypeK || settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0"; const effectiveCacheTypeV = isSyclBackend ? "q4_0" : loadProfile.cacheTypeV || settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0"; const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512) : Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512; const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512; const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : (loadProfile.enableThinking ?? (settings.enableThinking === true)); llmSettings = { ...llmSettings, model: filename, threads: Math.max(1, Math.min(64, Number(settings.threads) || llmSettings.threads)), contextSize: contextSize, gpuLayers: effectiveGpuLayers, requestedGpuLayers, backendMode: backend.mode, backendBinary: path.basename(backend.path), loadProfile: loadProfile.name || "requested", enableThinking: effectiveEnableThinking, supportsThinking: /deepseek|qwen3|gemma-4|gemma4|think|r1|e2b|reasoning/i.test(filename), // New performance settings flashAttn: effectiveFlashAttn, cacheTypeK: effectiveCacheTypeK, cacheTypeV: effectiveCacheTypeV, mlock: settings.mlock || llmSettings.mlock || false, mmap: settings.mmap !== false && llmSettings.mmap !== false, cachePrompt: settings.cachePrompt !== false && llmSettings.cachePrompt !== false, defragThold: Number(settings.defragThold) || llmSettings.defragThold || 0.1, batchSize: Math.max(128, Math.min(2048, effectiveBatchSize)), ubatchSize: Math.max(128, Math.min(2048, effectiveUbatchSize)), performanceProfile: settings.performanceProfile || llmSettings.performanceProfile || "balanced", }; let mmprojPath = null; const lowerFilename = filename.toLowerCase(); const isMultimodal = isVisionModelFilename(lowerFilename); if (isMultimodal) { if (settings.mmproj) { const customProjPath = path.join(LLM_MODELS, path.basename(settings.mmproj)); if (fs.existsSync(customProjPath)) { mmprojPath = customProjPath; } } if (!mmprojPath) { try { const files = fs.readdirSync(LLM_MODELS); let bestMatch = null; // 1. Match by Hugging Face repository prefix (e.g., "unsloth--gemma-4-E2B-it-qat-GGUF--") if (filename.includes("--")) { const lastIndex = filename.lastIndexOf("--"); const repoPrefix = filename.substring(0, lastIndex + 2); bestMatch = files.find(file => { const lower = file.toLowerCase(); return file.startsWith(repoPrefix) && lower.includes("mmproj") && lower.endsWith(".gguf"); }); } // 2. Match by model family keyword (e.g. matching model containing "gemma-4" with projector containing "gemma-4") if (!bestMatch) { const keywords = ["gemma-4", "gemma4", "llava", "qwen2vl", "qwen2-vl", "paligemma", "minicpm", "internvl", "phi-3", "phi3", "smolvlm", "moondream"]; const matchedKeyword = keywords.find(kw => lowerFilename.includes(kw)); if (matchedKeyword) { bestMatch = files.find(file => { const lower = file.toLowerCase(); return lower.endsWith(".gguf") && lower.includes("mmproj") && lower.includes(matchedKeyword); }); } } // 3. General fallback if (!bestMatch) { bestMatch = files.find(file => { const lower = file.toLowerCase(); return lower.endsWith(".gguf") && (lower === "mmproj-model-f16.gguf" || lower.includes("mmproj")); }); } if (bestMatch) { mmprojPath = path.join(LLM_MODELS, bestMatch); } } catch (_) {} } } const args = [ "--model", modelPath, "--host", "127.0.0.1", "--port", String(PORT_LLM), "-lv", "1", "--ctx-size", String(llmSettings.contextSize), "--threads", String(llmSettings.threads), "--n-gpu-layers", String(llmSettings.gpuLayers), "--parallel", "1", "--cache-type-k", String(llmSettings.cacheTypeK), // ✅ KV cache quantization K "--cache-type-v", String(llmSettings.cacheTypeV), // ✅ KV cache quantization V "--cache-prompt", // ✅ Prompt caching for multi-turn (major UX win) "--ctx-checkpoints", "8", // ✅ Context shifting recovery "--poll", "30", // ✅ More aggressive polling "--metrics", // ✅ Metrics endpoint for tuning "--batch-size", String(llmSettings.batchSize), // ✅ Prompt processing batch "--ubatch-size", String(llmSettings.ubatchSize), // ✅ Micro-batch size ]; if (llmSettings.flashAttn) { args.push("--flash-attn", "on"); } if (isSyclBackend) { args.push("--no-warmup"); } // CPU mode: lock memory to prevent OS paging (critical for consistent speed) const isGpuMode = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("HIP") || backend.mode.includes("ROCm") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.includes("SYCL"); if (!isGpuMode && llmSettings.mlock) { args.push("--mlock"); // ✅ Prevent OS swapping on CPU } // All modes: memory-mapped loading for faster startup if (llmSettings.mmap) { args.push("--mmap"); // ✅ Faster model loading } if (llmSettings.enableThinking === false) { args.push("--reasoning", "off"); // ✅ Disable reasoning/thinking completely } else if (llmSettings.supportsThinking) { args.push("--reasoning", "on"); // ✅ Enable reasoning for supported models } if (llmSettings.enableThinking === true && llmSettings.supportsThinking) { args.push("--reasoning-format", "deepseek"); // ✅ Extract thoughts into reasoning_content } if (mmprojPath) { args.push("--mmproj", mmprojPath); } llmSettings.supportsVision = Boolean(mmprojPath); llmSettings.mmproj = mmprojPath ? path.basename(mmprojPath) : null; llmSettings.visionMode = mmprojPath ? "mmproj" : "none"; llmSettings.visionStatus = mmprojPath ? `Using projector ${path.basename(mmprojPath)}` : (isMultimodal ? "Matching mmproj projector not found" : "Model is text-only"); const spawnEnv = { ...process.env }; const backendDir = path.dirname(backend.path); // Platform library paths if (osPlatform === "linux") { spawnEnv.LD_LIBRARY_PATH = backendDir + (spawnEnv.LD_LIBRARY_PATH ? `:${spawnEnv.LD_LIBRARY_PATH}` : ""); } else if (osPlatform === "darwin") { spawnEnv.DYLD_LIBRARY_PATH = backendDir + (spawnEnv.DYLD_LIBRARY_PATH ? `:${spawnEnv.DYLD_LIBRARY_PATH}` : ""); } // CUDA-specific optimizations if (backend.mode.includes("CUDA")) { // Multi-GPU: increase command buffer size (reduces CPU stalls) spawnEnv.CUDA_SCALE_LAUNCH_QUEUES = spawnEnv.CUDA_SCALE_LAUNCH_QUEUES || "4x"; // Linux: allow swapping to system RAM when VRAM exhausted if (osPlatform === "linux") { spawnEnv.GGML_CUDA_ENABLE_UNIFIED_MEMORY = spawnEnv.GGML_CUDA_ENABLE_UNIFIED_MEMORY || "1"; } } // SYCL-specific optimizations (Intel Arc/Graphics) if (backend.mode.includes("SYCL")) { spawnEnv.ZES_ENABLE_SYSMAN = "1"; } console.log(" [llm] Starting:", backend.path, args.join(" ")); const proc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv }); const procSeq = ++llmProcSeq; llmProc = proc; proc.stdout.on("data", (data) => { if (llmProc !== proc) return; process.stdout.write(" [llm] " + data.toString()); }); proc.stderr.on("data", (data) => { if (llmProc !== proc) return; const output = data.toString(); process.stderr.write(" [llm-err] " + output); if (/Vulkan\d+\s*:/i.test(output)) llmSettings.backendMode = "Vulkan GPU"; else if (/CUDA\d+\s*:/i.test(output)) llmSettings.backendMode = "CUDA GPU"; else if (/(HIP|ROCm)\d*\s*:/i.test(output)) llmSettings.backendMode = "ROCm GPU"; else if (/SYCL\d+\s*:/i.test(output)) llmSettings.backendMode = "SYCL GPU"; else if (/Metal/i.test(output) && /GPU|device/i.test(output)) llmSettings.backendMode = "Metal GPU"; else if (/\-\s+CPU\s+:/i.test(output) && llmSettings.backendMode.startsWith("Auto")) llmSettings.backendMode = "CPU"; if (/error|failed/i.test(output) && !/no error/i.test(output)) { llmError = output.trim().slice(-1200); } }); proc.on("exit", (code) => { if (llmProc !== proc) { console.log(" [llm] stale process exited with code", code); return; } llmReady = false; llmProc = null; if (code !== 0 && code !== null && !llmError) llmError = `llama.cpp exited with code ${code}`; console.log(" [llm] exited with code", code, `(process ${procSeq})`); }); await waitForLlmReady(backend.key === "cpu" ? 720 : 360, proc, filename); } async function startBackend(settings = {}) { const requestedSettings = { ...currentSettings, ...settings }; if (!requestedSettings.model) requestedSettings.model = getDefaultModel(); assertNoOtherActiveRuntime("image", requestedSettings.model); await killLlm(); if (settings.backendType === "openvino-npu") { await startOpenVinoWorker(settings); return; } await killOpenVinoWorker(); backendError = null; currentSettings = requestedSettings; if (!currentSettings.model) { console.log(" [backend] No model found in app/models/ — backend not started"); return; } const modelLoadIssue = getModelLoadIssue(currentSettings.model); if (modelLoadIssue) { backendError = modelLoadIssue; backendLoadState = { active: false, phase: "Model load blocked", progress: 0, current: 0, total: 0, speed: "", model: path.basename(currentSettings.model), backendMode: "", backendBinary: "", device: "", }; throw new Error(modelLoadIssue); } PORT_BACKEND = Number(settings.backendPort) > 0 ? Number(settings.backendPort) : await findAvailableBackendPort(); const resolvedBackendType = resolveBackendType(currentSettings.useGpu, currentSettings.backendType, currentSettings.model); currentSettings.backendType = resolvedBackendType; currentSettings.useGpu = resolvedBackendType !== "cpu"; const backendPath = selectBackendPath(currentSettings.useGpu, currentSettings.backendType, currentSettings.model); if (!backendPath) { const setupHint = resolvedBackendType === "apple-npu" ? "CoreML Python environment is not set up. Run scripts/setup/setup-coreml-npu.sh first." : resolvedBackendType === "cpu" && osPlatform === "win32" ? "Windows CPU backend is not installed. Run scripts/setup.ps1 again so app/backend/win/cpu is created." : "No compatible backend executable was found."; throw new Error(setupHint); } const backendMode = getBackendMode(backendPath, currentSettings.useGpu, currentSettings.backendType); currentSettings.backendMode = backendMode; currentSettings.backendBinary = path.basename(backendPath); backendLoadState = { active: true, phase: "Starting backend...", progress: 0, current: 0, total: 0, speed: "", model: path.basename(currentSettings.model), backendMode, backendBinary: path.basename(backendPath), device: "", }; let runThreads = parseInt(currentSettings.threads) || 4; if (currentSettings.useGpu) { runThreads = Math.min(4, runThreads); } let args = []; const requestedBackend = resolveBackendType(currentSettings.useGpu, currentSettings.backendType, currentSettings.model); const selectedVulkanBackend = requestedBackend === "vulkan" ? getPreferredVulkanBackendName() : ""; const supportsFlags = backendSupportsFlags[backendPath] !== false; if (requestedBackend === "apple-npu") { args = [ path.join(ROOT, "scripts", "workers", "coreml_server.py"), "--listen-port", String(PORT_BACKEND), "--model", currentSettings.model, "--steps", String(currentSettings.steps), "--cfg-scale", String(currentSettings.cfgScale), ]; } else { args = [ "--listen-port", String(PORT_BACKEND), "--model", currentSettings.model, "--steps", String(currentSettings.steps), "--cfg-scale", String(currentSettings.cfgScale), "--sampling-method", currentSettings.sampler, "--threads", String(runThreads), ]; if (requestedBackend === "cpu") { if (supportsFlags) { args.push("--backend", "cpu", "--params-backend", "cpu"); } args.push("--rng", "cpu", "--sampler-rng", "cpu"); } else if (requestedBackend === "vulkan") { if (supportsFlags) { args.push("--backend", selectedVulkanBackend, "--params-backend", selectedVulkanBackend); } args.push("--rng", "cpu", "--sampler-rng", "cpu"); } else if (requestedBackend === "cuda") { if (supportsFlags) { args.push("--backend", "cuda0", "--params-backend", "cuda0"); } args.push("--rng", "cuda", "--sampler-rng", "cuda"); } else if (requestedBackend === "rocm") { if (supportsFlags) { args.push("--backend", "rocm0", "--params-backend", "rocm0"); } args.push("--rng", "cpu", "--sampler-rng", "cpu"); } else if (requestedBackend === "metal") { args.push("--rng", "cpu", "--sampler-rng", "cpu"); } } if (requestedBackend === "vulkan" && selectedVulkanBackend) { currentSettings.backendDevice = selectedVulkanBackend; backendLoadState.device = selectedVulkanBackend; } if (requestedBackend !== "apple-npu") { if (currentSettings.vaeTiling) { args.push("--vae-tiling"); } if (currentSettings.vaeOnCpu) { args.push("--vae-on-cpu"); } if (currentSettings.flashAttn) { if (requestedBackend === "cuda") { args.push("--diffusion-fa"); } else { args.push("--fa"); } } } // Build environment for Linux backends so bundled .so libraries are found. // Official Linux releases ship libstable-diffusion.so next to the executable. const spawnEnv = { ...process.env }; if (osPlatform === "linux") { const extraLibs = []; if (requestedBackend === "rocm") { extraLibs.push(path.dirname(BACKEND_PATHS.linuxRocm)); } else if (requestedBackend === "vulkan") { extraLibs.push(path.dirname(BACKEND_PATHS.linuxVulkan)); } else if (requestedBackend === "cpu") { extraLibs.push(path.dirname(BACKEND_PATHS.linuxCpu)); } else if (requestedBackend === "cuda") { extraLibs.push(path.dirname(BACKEND_PATHS.linuxCuda)); } if (extraLibs.length > 0) { const existing = spawnEnv.LD_LIBRARY_PATH || ""; spawnEnv.LD_LIBRARY_PATH = extraLibs.join(":") + (existing ? ":" + existing : ""); } } else if (osPlatform === "darwin" && requestedBackend !== "apple-npu") { const existing = spawnEnv.DYLD_LIBRARY_PATH || ""; const backendDir = path.dirname(BACKEND_PATHS.mac); spawnEnv.DYLD_LIBRARY_PATH = backendDir + (existing ? ":" + existing : ""); } console.log(" [backend] Starting:", path.basename(backendPath), args.join(" ")); backendReady = false; const procSeq = ++backendProcSeq; const proc = spawn(backendPath, args, { stdio: "pipe", env: spawnEnv }); backendProc = proc; startBackendReadyPoll(); const coremlLoadHeartbeat = startCoreMLLoadHeartbeat(proc); proc.stdout.on("data", d => { const output = d.toString(); process.stdout.write(" [sd] " + output); updateCoreMLLoadProgress(output); const cleanOutput = stripAnsi(output); if (cleanOutput.includes("listening on")) { markBackendReady(); } const lines = output.split(/[\r\n]+/); for (const line of lines) { const cleanLine = stripAnsi(line); if (!cleanLine.trim()) continue; if (cleanLine.includes("listening on")) { markBackendReady(); } if (cleanLine.includes("loading model from")) { backendLoadState.active = true; backendLoadState.phase = "Loading model weights..."; backendLoadState.progress = Math.max(backendLoadState.progress, 1); } if (cleanLine.includes("model files processing completed")) { backendLoadState.active = true; backendLoadState.phase = "Initializing model..."; backendLoadState.progress = Math.max(backendLoadState.progress, 95); } const loadMatch = cleanLine.match(/\|\s*(\d+)\/(\d+)\s*-\s*([^|]+)$/); if (loadMatch && !cleanLine.includes("it/s") && !cleanLine.includes("s/it")) { const current = parseInt(loadMatch[1], 10); const total = parseInt(loadMatch[2], 10); const progress = total > 0 ? Math.round((current / total) * 100) : 0; const isCoreML = currentSettings.backendType === "apple-npu" || total <= 10; const phaseDesc = isCoreML ? stripAnsi(loadMatch[3]).trim() : "Loading model weights..."; const speedDesc = isCoreML ? "" : stripAnsi(loadMatch[3]).trim(); backendLoadState = { ...backendLoadState, active: !backendReady, phase: phaseDesc, progress: Math.max(backendLoadState.progress, Math.min(99, progress)), current, total, speed: speedDesc, }; } if (cleanLine.includes("generate_image") || cleanLine.includes("generating image")) { generationState.active = true; generationState.step = 0; generationState.steps = 0; generationState.speed = ""; generationState.decoding = false; } if (cleanLine.includes("decoding") && generationState.active) { generationState.decoding = true; } const match = cleanLine.match(/\|\s*[^|]*\s*\|\s*(\d+)\/(\d+)\s*-\s*([\d.]+\s*(?:it\/s|s\/it))/); if (match) { generationState.active = true; if (!generationState.decoding) { generationState.step = parseInt(match[1], 10); generationState.steps = parseInt(match[2], 10); generationState.speed = match[3].trim(); } } if (cleanLine.includes("generate_image completed")) { generationState.active = false; generationState.step = 0; generationState.steps = 0; generationState.speed = ""; generationState.decoding = false; } } }); proc.stderr.on("data", d => { const output = d.toString(); process.stderr.write(" [sd-err] " + output); updateCoreMLLoadProgress(output); const cleanOutput = stripAnsi(output); const runtimeLinkerError = describeLinuxRuntimeLinkerError(cleanOutput); if (runtimeLinkerError) { backendError = runtimeLinkerError; backendLoadState.phase = "Linux runtime is too old for this backend"; } const deviceMatch = cleanOutput.match(/Device\s+\d+:\s*([^,\r\n]+)/); if (deviceMatch) { backendLoadState.device = deviceMatch[1].trim(); currentSettings.backendDevice = backendLoadState.device; } const metalDeviceMatch = cleanOutput.match(/GPU name:\s*([^\r\n]+)/); if (metalDeviceMatch) { backendLoadState.device = metalDeviceMatch[1].trim(); currentSettings.backendDevice = backendLoadState.device; } if (cleanOutput.includes("ggml_cuda_init")) { backendLoadState.backendMode = "CUDA GPU"; currentSettings.backendMode = "CUDA GPU"; } if (cleanOutput.includes("ggml_vulkan")) { backendLoadState.backendMode = "Vulkan GPU"; currentSettings.backendMode = "Vulkan GPU"; } if (cleanOutput.includes("ggml_hip") || cleanOutput.includes("ggml_rocm")) { backendLoadState.backendMode = "ROCm GPU"; currentSettings.backendMode = "ROCm GPU"; } if (cleanOutput.includes("ggml_metal")) { backendLoadState.backendMode = "Metal GPU"; currentSettings.backendMode = "Metal GPU"; } if (cleanOutput.includes("[ERROR]")) { const nextError = describeBackendError(cleanOutput.trim(), currentSettings.model); const hasSpecificFileError = String(backendError || "").includes("incomplete or corrupted"); const isGenericContextError = cleanOutput.includes("new_sd_ctx_t failed"); if (!(hasSpecificFileError && isGenericContextError)) { backendError = nextError; } } }); proc.on("exit", (code, signal) => { if (coremlLoadHeartbeat) clearInterval(coremlLoadHeartbeat); if (backendProc !== proc) { console.log(" [backend] stale process exited with code", code, signal ? `(signal ${signal})` : "", `(process ${procSeq})`); return; } backendReady = false; backendProc = null; console.log(" [backend] exited with code", code, signal ? `(signal ${signal})` : "", `(process ${procSeq})`); if (code !== null && code !== 0) { if (!backendError) { backendError = describeBackendExitCode(code, currentSettings.backendBinary || BACKEND_PATH); } } backendLoadState.active = false; generationState.active = false; generationState.step = 0; generationState.steps = 0; generationState.speed = ""; generationState.decoding = false; // Force a VRAM poll to update free memory space setTimeout(() => pollNvidiaVram(true), 1000); }); } // ── Model Downloader ───────────────────────────────────────────────────────── let downloadState = { active: false, filename: "", progress: 0, speed: "0 MB/s", eta: 0, totalBytes: 0, downloadedBytes: 0, error: null }; let activeDownload = null; function startOpenVinoModelDownload(modelId) { if (downloadState.active) return; const npuInfo = getOpenVinoNpuInfo(); if (!npuInfo.supported) throw new Error(npuInfo.reason || "OpenVINO NPU runtime is not available."); const model = OPENVINO_NPU_MODELS.find((item) => item.id === modelId); if (!model) throw new Error("Unknown OpenVINO model."); const destDir = path.join(OPENVINO_MODELS, model.folder); fs.mkdirSync(destDir, { recursive: true }); downloadState = { active: true, filename: model.name, progress: -1, speed: "Downloading snapshot", eta: 0, totalBytes: 0, downloadedBytes: 0, error: null, }; const script = [ "from huggingface_hub import snapshot_download", `snapshot_download(repo_id=${JSON.stringify(model.repo)}, local_dir=${JSON.stringify(destDir)}, ignore_patterns=['safety_checker/*','feature_extractor/*','*.safetensors'])`, "print('DONE')", ].join("\n"); console.log(` [openvino-download] Downloading ${model.repo} -> ${destDir}`); const proc = spawn(npuInfo.python, ["-c", script], { stdio: "pipe" }); activeDownload = { process: proc, destPath: destDir, openvino: true }; proc.stdout.on("data", (data) => process.stdout.write(" [openvino-download] " + data.toString())); proc.stderr.on("data", (data) => process.stderr.write(" [openvino-download] " + data.toString())); proc.on("exit", (code) => { activeDownload = null; if (String(downloadState.error || "").toLowerCase().includes("cancelled")) { return; } downloadState.active = false; if (code === 0) { downloadState.progress = 100; downloadState.downloadedBytes = getPathSize(destDir); downloadState.totalBytes = downloadState.downloadedBytes; downloadState.speed = "Complete"; cachedBackendOptions = null; } else { downloadState.error = `OpenVINO model download failed with code ${code}`; } }); } const IMAGE_BACKEND_DOWNLOADS = { "cpu": { id: "cpu", label: "CPU", url: "https://github.com/leejet/stable-diffusion.cpp/releases/download/master-721-8caa3f9/sd-master-8caa3f9-bin-win-avx2-x64.zip", destDir: path.join(ROOT, "app", "backend", "win", "cpu"), exeName: "sd-cpu.exe", requiredDll: "stable-diffusion.dll", }, "vulkan": { id: "vulkan", label: "Vulkan GPU", url: "https://github.com/leejet/stable-diffusion.cpp/releases/download/master-669-2d40a8b/sd-master-2d40a8b-bin-win-vulkan-x64.zip", destDir: path.join(ROOT, "app", "backend", "win", "vulkan"), exeName: "sd-vulkan.exe", requiredDll: "stable-diffusion.dll", }, "cuda": { id: "cuda", label: "CUDA GPU", url: "https://github.com/leejet/stable-diffusion.cpp/releases/download/master-721-8caa3f9/sd-master-8caa3f9-bin-win-cuda12-x64.zip", destDir: path.join(ROOT, "app", "backend", "win", "cuda"), exeName: "sd-cuda.exe", requiredDll: "stable-diffusion.dll", }, }; function listFilesRecursive(dir) { const out = []; if (!fs.existsSync(dir)) return out; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const full = path.join(dir, entry.name); if (entry.isDirectory()) { out.push(...listFilesRecursive(full)); } else if (entry.isFile()) { out.push(full); } } return out; } function installImageBackendArchive(zipPath, backend) { const tempDir = path.join(TOOLS, `image-backend-${backend.id}-extract-${Date.now()}`); try { fs.rmSync(tempDir, { recursive: true, force: true }); fs.mkdirSync(tempDir, { recursive: true }); fs.mkdirSync(backend.destDir, { recursive: true }); const tar = spawnSync("tar.exe", ["-xf", zipPath, "-C", tempDir], { encoding: "utf8" }); if (tar.status !== 0) { throw new Error((tar.stderr || tar.stdout || "Could not extract backend archive.").trim()); } const files = listFilesRecursive(tempDir); const exeSource = files.find((file) => ["sd-server.exe", "sd.exe"].includes(path.basename(file).toLowerCase())); const dllSource = files.find((file) => path.basename(file).toLowerCase() === backend.requiredDll.toLowerCase()); if (!exeSource || !dllSource) { throw new Error("Backend archive did not contain the expected stable-diffusion executable and DLL."); } fs.copyFileSync(exeSource, path.join(backend.destDir, backend.exeName)); fs.copyFileSync(dllSource, path.join(backend.destDir, backend.requiredDll)); for (const file of files) { const ext = path.extname(file).toLowerCase(); if (ext !== ".dll" && ext !== ".exe") continue; if (file === exeSource) continue; fs.copyFileSync(file, path.join(backend.destDir, path.basename(file))); } if (!fs.existsSync(path.join(backend.destDir, backend.exeName)) || !fs.existsSync(path.join(backend.destDir, backend.requiredDll))) { throw new Error(`Backend install did not create ${backend.exeName} and ${backend.requiredDll}.`); } cachedBackendOptions = null; } finally { try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch (_) {} } } function startImageBackendDownload(backendId, redirectCount = 0, redirectUrl = "") { if (downloadState.active && redirectCount === 0) { throw new Error("Another download is already active."); } if (osPlatform !== "win32") { throw new Error("In-app backend downloads are currently available for Windows image backends."); } const backend = IMAGE_BACKEND_DOWNLOADS[String(backendId || "").toLowerCase()]; if (!backend) { throw new Error("This backend cannot be installed from the app yet. Use the platform setup script."); } fs.mkdirSync(TOOLS, { recursive: true }); const url = redirectUrl || backend.url; const filename = `${backend.id}-backend.zip`; const destPath = path.join(TOOLS, filename); const tempPath = `${destPath}.part`; if (redirectCount === 0) { try { fs.unlinkSync(tempPath); } catch (_) {} downloadState = { active: true, filename: backend.label, progress: 0, speed: "0 MB/s", eta: 0, totalBytes: 0, downloadedBytes: 0, error: null, kind: "backend", backendId: backend.id, }; } console.log(` [backend-download] Downloading ${backend.label} backend from ${url}`); let downloadFinalized = false; const fileStream = fs.createWriteStream(tempPath); const failDownload = (message, err = null) => { if (downloadFinalized) return; downloadFinalized = true; downloadState.active = false; downloadState.error = message; if (err) console.error(" [backend-download]", message, err); else console.error(" [backend-download] Failed:", message); try { fileStream.close(); } catch (_) {} try { fs.unlinkSync(tempPath); } catch (_) {} activeDownload = null; }; fileStream.on("error", (err) => failDownload(`Could not write ${filename}: ${err.message}`, err)); const client = url.startsWith("https") ? https : http; const request = client.get(url, { headers: { "User-Agent": "Uncensored-AI-Studio/1.0 (+https://github.com/techjarves/Uncensored-AI-Studio)", "Accept": "application/zip, application/octet-stream, */*", }, }, (response) => { activeDownload = { request, fileStream, destPath, tempPath }; if ([301, 302, 303, 307, 308].includes(response.statusCode)) { const nextUrl = response.headers.location ? new URL(response.headers.location, url).toString() : ""; if (!nextUrl) return failDownload("Redirect response did not include a Location header."); if (redirectCount > 10) return failDownload("Too many redirects."); downloadState.speed = "Following redirect"; request.removeAllListeners("error"); request.destroy(); response.resume(); fileStream.close(); try { fs.unlinkSync(tempPath); } catch (_) {} activeDownload = null; startImageBackendDownload(backend.id, redirectCount + 1, nextUrl); return; } if (response.statusCode !== 200) { return failDownload(describeDownloadHttpError(response.statusCode, url, response.headers)); } const totalBytes = parseInt(response.headers["content-length"], 10) || 0; downloadState.totalBytes = totalBytes; let downloadedBytes = 0; let lastTime = Date.now(); let lastDownloaded = 0; response.on("data", (chunk) => { downloadedBytes += chunk.length; downloadState.downloadedBytes = downloadedBytes; fileStream.write(chunk); const now = Date.now(); const elapsed = (now - lastTime) / 1000; if (elapsed >= 0.5) { const chunkSpeed = (downloadedBytes - lastDownloaded) / elapsed; lastDownloaded = downloadedBytes; lastTime = now; downloadState.speed = `${(chunkSpeed / (1024 * 1024)).toFixed(1)} MB/s`; if (totalBytes > 0) { downloadState.progress = Math.round((downloadedBytes / totalBytes) * 100); downloadState.eta = Math.round((totalBytes - downloadedBytes) / Math.max(1, chunkSpeed)); } else { downloadState.progress = -1; downloadState.eta = -1; } } }); response.on("aborted", () => failDownload(`Download interrupted before ${backend.label} backend finished.`)); response.on("error", (err) => failDownload(`Download stream failed before ${backend.label} backend finished.`, err)); response.on("end", () => { if (downloadFinalized) return; if (totalBytes > 0 && downloadedBytes !== totalBytes) { failDownload(`Download incomplete for ${backend.label}: received ${formatBytes(downloadedBytes)} of ${formatBytes(totalBytes)}.`); return; } fileStream.end(() => { try { try { fs.unlinkSync(destPath); } catch (_) {} fs.renameSync(tempPath, destPath); downloadState.speed = "Installing"; downloadState.progress = 95; installImageBackendArchive(destPath, backend); try { fs.unlinkSync(destPath); } catch (_) {} downloadFinalized = true; downloadState.active = false; downloadState.progress = 100; downloadState.downloadedBytes = downloadedBytes; downloadState.error = null; downloadState.speed = "Complete"; activeDownload = null; console.log(` [backend-download] Installed ${backend.label} backend`); } catch (err) { failDownload(`Could not install ${backend.label} backend: ${err.message}`, err); } }); }); }); request.on("error", (err) => failDownload(err.message, err)); activeDownload = { request, fileStream, destPath, tempPath }; } // ── Generation State (Real-time progress parser) ───────────────────────────── let generationState = { active: false, step: 0, steps: 0, speed: "", decoding: false, backendMode: "", backendDevice: "", }; function resetGenerationState() { generationState = { active: false, step: 0, steps: 0, speed: "", decoding: false, backendMode: "", backendDevice: "", }; } function describeDownloadHttpError(statusCode, url, headers = {}) { const host = (() => { try { return new URL(url).hostname; } catch (_) { return ""; } })(); const isHuggingFace = host.includes("huggingface.co") || host.includes("hf.co"); const hfErrorCode = headers["x-error-code"]; const hfErrorMessage = headers["x-error-message"]; if (isHuggingFace && hfErrorCode === "GatedRepo") { return `HTTP ${statusCode}: Hugging Face says this model is gated. Open the model page in your browser, accept access/login, then use a Hugging Face token or download the file manually.`; } if (isHuggingFace && hfErrorMessage) { return `HTTP ${statusCode}: Hugging Face rejected the download request: ${hfErrorMessage}`; } if (statusCode === 401 || statusCode === 403) { return isHuggingFace ? `HTTP ${statusCode}: Hugging Face rejected the download request. If this model is public, retry after restarting the app; otherwise open the model page in your browser and accept any license/login requirement.` : `HTTP ${statusCode}: This download URL requires authorization or permission. Use a public direct download URL.`; } if (statusCode === 404) { return `HTTP 404: Model file not found. Check that the URL points directly to a .safetensors, .gguf, or .ckpt file.`; } return `HTTP ${statusCode}`; } function startModelDownload(url, overrideFilename = null, targetDir = MODELS, kind = "image", redirectCount = 0) { if (downloadState.active && !overrideFilename) { console.log(" [download] Already downloading a model"); return; } // Convert HuggingFace viewer URL (/blob/) to direct download URL (/resolve/) if (url.includes("huggingface.co") && url.includes("/blob/")) { url = url.replace("/blob/", "/resolve/"); } let filename = overrideFilename; if (!filename) { filename = "model.gguf"; try { const parsed = new URL(url); filename = path.basename(parsed.pathname); } catch (e) { console.error("Failed to parse URL filename:", e); } } const destPath = path.join(targetDir, filename); const tempPath = `${destPath}.part`; try { fs.unlinkSync(tempPath); } catch (_) {} downloadState = { active: true, filename: filename, progress: 0, speed: "0 MB/s", eta: 0, totalBytes: 0, downloadedBytes: 0, error: null, kind, }; if (redirectCount > 0) { console.log(` [download] Continuing redirected download of ${filename}`); } else { console.log(` [download] Starting download of ${filename} from ${url}`); } let downloadFinalized = false; const failDownload = (message, err = null) => { if (downloadFinalized) return; downloadFinalized = true; downloadState.active = false; downloadState.error = message; if (err) { console.error(" [download]", message, err); } else { console.error(" [download] Failed:", message); } try { fileStream.close(); } catch (_) {} try { fs.unlinkSync(tempPath); } catch (_) {} activeDownload = null; }; const fileStream = fs.createWriteStream(tempPath); fileStream.on("error", (err) => { failDownload(`Could not write ${filename}: ${err.message}`, err); }); const client = url.startsWith("https") ? https : http; const request = client.get(url, { headers: { "User-Agent": "Uncensored-AI-Studio/1.0 (+https://github.com/techjarves/Uncensored-AI-Studio)", "Accept": "application/octet-stream, application/x-safetensors, */*", "Referer": "https://huggingface.co/", }, }, (response) => { activeDownload = { request, fileStream, destPath, tempPath }; // Handle redirects (HuggingFace resolve URLs redirect to Cloudfront/S3) if ([301, 302, 303, 307, 308].includes(response.statusCode)) { const redirectUrl = response.headers.location ? new URL(response.headers.location, url).toString() : ""; if (!redirectUrl) { failDownload("Redirect response did not include a Location header"); return; } if (redirectCount > 10) { failDownload("Too many redirects"); return; } console.log(` [download] Following redirect for ${filename}`); downloadState.speed = "Following redirect"; // Clean up redirected request to avoid triggering error handlers later request.removeAllListeners("error"); request.destroy(); response.resume(); fileStream.close(); try { fs.unlinkSync(tempPath); } catch (_) {} activeDownload = null; startModelDownload(redirectUrl, filename, targetDir, kind, redirectCount + 1); return; } if (response.statusCode !== 200) { failDownload(describeDownloadHttpError(response.statusCode, url, response.headers)); return; } const totalBytes = parseInt(response.headers["content-length"], 10) || 0; downloadState.totalBytes = totalBytes; let downloadedBytes = 0; let startTime = Date.now(); let lastTime = startTime; let lastDownloaded = 0; response.on("data", (chunk) => { downloadedBytes += chunk.length; downloadState.downloadedBytes = downloadedBytes; fileStream.write(chunk); const now = Date.now(); const elapsed = (now - lastTime) / 1000; if (elapsed >= 0.5) { const chunkSpeed = (downloadedBytes - lastDownloaded) / elapsed; // bytes/sec lastDownloaded = downloadedBytes; lastTime = now; const speedMb = (chunkSpeed / (1024 * 1024)).toFixed(1); downloadState.speed = `${speedMb} MB/s`; if (totalBytes > 0) { downloadState.progress = Math.round((downloadedBytes / totalBytes) * 100); const remainingBytes = totalBytes - downloadedBytes; downloadState.eta = Math.round(remainingBytes / chunkSpeed); } else { downloadState.progress = -1; // Indeterminate downloadState.eta = -1; } } }); response.on("aborted", () => { failDownload(`Download interrupted before ${filename} finished. Delete and retry the model download.`); }); response.on("error", (err) => { failDownload(`Download stream failed before ${filename} finished. Delete and retry the model download.`, err); }); response.on("end", () => { if (downloadFinalized) return; if (totalBytes > 0 && downloadedBytes !== totalBytes) { failDownload(`Download incomplete for ${filename}: received ${formatBytes(downloadedBytes)} of ${formatBytes(totalBytes)}. Delete and retry the model download.`); return; } fileStream.end(() => { try { if (totalBytes > 0) { const writtenBytes = fs.statSync(tempPath).size; if (writtenBytes !== totalBytes) { failDownload(`Download incomplete for ${filename}: wrote ${formatBytes(writtenBytes)} of ${formatBytes(totalBytes)}. Delete and retry the model download.`); return; } } try { fs.unlinkSync(destPath); } catch (_) {} fs.renameSync(tempPath, destPath); if (destPath.toLowerCase().endsWith(".zip")) { try { const { execSync } = require("child_process"); execSync(`unzip -o "${destPath}" -d "${path.dirname(destPath)}"`, { stdio: "ignore" }); fs.unlinkSync(destPath); } catch(err) { console.error("Failed to unzip", err); } } downloadState.active = false; downloadFinalized = true; downloadState.progress = 100; downloadState.downloadedBytes = downloadedBytes; downloadState.error = null; activeDownload = null; console.log(` [download] Completed download of ${filename}`); } catch (err) { failDownload(`Could not finalize ${filename}: ${err.message}`); } }); }); }); request.on("error", (err) => { failDownload(err.message, err); }); activeDownload = { request, fileStream, destPath, tempPath }; } function cancelModelDownload() { if (!downloadState.active) { return false; } const filename = downloadState.filename; if (activeDownload) { if (activeDownload.process) { try { activeDownload.process.kill("SIGTERM"); } catch (_) {} setTimeout(() => { try { activeDownload?.process?.kill("SIGKILL"); } catch (_) {} }, 1000); } else { try { activeDownload.request.destroy(new Error("Download cancelled by user")); } catch (_) {} try { activeDownload.fileStream.destroy(); } catch (_) {} try { fs.unlinkSync(activeDownload.tempPath || activeDownload.destPath); } catch (_) {} } } else if (filename) { const targetDir = downloadState.kind === "text" ? LLM_MODELS : downloadState.kind === "speech" ? SPEECH_MODELS : downloadState.kind === "tts" ? TTS_MODELS : MODELS; try { fs.unlinkSync(`${path.join(targetDir, filename)}.part`); } catch (_) {} } activeDownload = null; downloadState = { active: false, filename, progress: 0, speed: "0 MB/s", eta: 0, totalBytes: 0, downloadedBytes: 0, error: "Download cancelled", kind: downloadState.kind || "image", }; console.log(` [download] Cancelled download of ${filename}`); return true; } // ── MIME types ──────────────────────────────────────────────────────────────── const MIME = { ".html": "text/html", ".js": "application/javascript", ".css": "text/css", ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".svg": "image/svg+xml", ".ico": "image/x-icon", ".json": "application/json", ".wav": "audio/wav", ".txt": "text/plain", ".woff2":"font/woff2", ".woff": "font/woff", ".ttf": "font/ttf", }; function isModelFile(filename) { const lower = String(filename || "").toLowerCase(); if (lower.endsWith(".safetensors") || lower.endsWith(".gguf") || lower.endsWith(".ckpt") || lower.endsWith(".coreml") || lower.endsWith(".coreml.zip")) { return true; } const fullPath = path.join(MODELS, filename); try { const stat = fs.statSync(fullPath); if (stat.isDirectory()) { const files = fs.readdirSync(fullPath).map(f => f.toLowerCase()); const hasTextEncoder = files.includes("text_encoder.mlpackage") || files.includes("text_encoder.mlmodelc") || files.includes("textencoder.mlpackage") || files.includes("textencoder.mlmodelc"); const hasUnet = files.includes("unet.mlpackage") || files.includes("unet.mlmodelc"); if (hasTextEncoder && hasUnet) return true; const checkSubdirs = ["split_einsum/packages", "split_einsum/compiled", "original/packages", "original/compiled", "split_einsum", "original"]; for (const subdir of checkSubdirs) { const subPath = path.join(fullPath, subdir); if (fs.existsSync(subPath)) { const subFiles = fs.readdirSync(subPath).map(f => f.toLowerCase()); const subTextEncoder = subFiles.includes("text_encoder.mlpackage") || subFiles.includes("text_encoder.mlmodelc") || subFiles.includes("textencoder.mlpackage") || subFiles.includes("textencoder.mlmodelc"); const subUnet = subFiles.includes("unet.mlpackage") || subFiles.includes("unet.mlmodelc"); if (subTextEncoder && subUnet) return true; } } } } catch (_) {} return false; } function formatBytes(bytes) { const value = Number(bytes) || 0; if (value <= 0) return "0 B"; const units = ["B", "KB", "MB", "GB", "TB"]; const idx = Math.min(units.length - 1, Math.floor(Math.log(value) / Math.log(1024))); return `${(value / (1024 ** idx)).toFixed(idx === 0 ? 0 : 1)} ${units[idx]}`; } function getModelLoadIssue(modelPath) { const filename = path.basename(modelPath || ""); const lower = filename.toLowerCase(); const ext = path.extname(lower); if (!modelPath || !fs.existsSync(modelPath)) { return `Model file not found: ${filename || "unknown model"}`; } const size = getPathSize(modelPath); if (size < 128 * 1024 * 1024) { return `${filename} is too small to be a complete image model (${formatBytes(size)}). Delete it and download/import it again.`; } if (ext === ".gguf") { const knownDiffusionOnlyGguf = lower === "stable-diffusion-xl-base-1.0-q4_0.gguf" || lower.includes("stable-diffusion-xl-base-1.0"); const requiresSeparateComponents = knownDiffusionOnlyGguf || lower.includes("z_image") || lower.includes("z-image") || lower.includes("zimage") || lower.includes("qwen") || lower.includes("hidream") || lower.includes("hunyuan") || lower.includes("wan") || lower.includes("flux"); if (requiresSeparateComponents) { if (knownDiffusionOnlyGguf) { return `${filename} is not supported as a one-click model in this app. This SDXL GGUF is a diffusion-only component and needs matching VAE/text encoder files instead of being loaded with --model. Use one of the recommended Safetensors SDXL/SD 1.5 checkpoints, or import a complete single-file GGUF checkpoint.`; } return `${filename} looks like a multi-file diffusion GGUF. This app currently loads single-file SD 1.5/SDXL checkpoints directly. Models like Z-Image, Qwen, Flux, HiDream, Hunyuan, and Wan usually need extra files such as VAE/text encoders and must be launched with --diffusion-model instead of --model.`; } } return null; } function describeBackendError(rawError, modelPath) { const raw = String(rawError || "").trim(); const filename = path.basename(modelPath || ""); const lower = filename.toLowerCase(); if (raw.includes("read tensor data failed") || raw.includes("load tensors from file failed")) { return `${raw}\n\n${filename || "The selected model"} is present but appears incomplete or corrupted. Delete it from Local Models and download/import it again.`; } if (!raw.includes("new_sd_ctx_t failed")) return raw; if (lower.endsWith(".gguf")) { if (lower === "stable-diffusion-xl-base-1.0-q4_0.gguf" || lower.includes("stable-diffusion-xl-base-1.0")) { return `${raw}\n\n${filename} is not supported as a one-click model in this app. This SDXL GGUF is a diffusion-only component and needs matching VAE/text encoder files instead of being loaded with --model. Use one of the recommended Safetensors SDXL/SD 1.5 checkpoints, or import a complete single-file GGUF checkpoint.`; } return `${raw}\n\n${filename} could not be loaded as a single checkpoint. Some GGUF files are only the diffusion part of a larger workflow and need separate VAE/text encoder files. Try a recommended SD 1.5/SDXL model, or re-download/import the file if this is meant to be a single-file SD/SDXL GGUF.`; } return `${raw}\n\nThe backend could not create the model context. Common causes are a corrupt/incomplete model file, unsupported checkpoint type, or not enough free RAM/VRAM. Delete and re-download the model, then try CPU or Vulkan mode at 512x512.`; } function describeLinuxRuntimeLinkerError(rawError) { const raw = String(rawError || "").trim(); const lower = raw.toLowerCase(); if (osPlatform !== "linux") return null; if (!lower.includes("glibc") && !lower.includes("libstdc++") && !lower.includes("libc.so.6")) return null; const needsGlibc = raw.match(/GLIBC_([0-9.]+)/)?.[1]; const needsGlibcxx = raw.match(/GLIBCXX_([0-9.]+)/)?.[1]; const requirements = []; if (needsGlibc) requirements.push(`glibc ${needsGlibc}+`); if (needsGlibcxx) requirements.push(`GLIBCXX_${needsGlibcxx}+`); const requirementText = requirements.length ? requirements.join(" and ") : "newer glibc/libstdc++ runtime libraries"; return `${raw}\n\nThe selected model is not the problem. The Linux backend binary cannot start because this OS is missing ${requirementText}. The bundled Linux backends are built for Ubuntu 24.04-era systems. Use Ubuntu 24.04+, Fedora 40+, another glibc 2.38+ distro, or build stable-diffusion.cpp from source on this machine.`; } function describeBackendExitCode(code, backendPath) { const numericCode = Number(code); if (osPlatform === "win32" && numericCode === 3221225781) { const backendName = path.basename(backendPath || BACKEND_PATH || "backend"); const lowerBackend = backendName.toLowerCase(); const isVulkan = lowerBackend.includes("vulkan"); const isCuda = lowerBackend.includes("cuda"); const likelyMissing = isVulkan ? "the Vulkan runtime/driver DLL, such as vulkan-1.dll" : isCuda ? "a CUDA runtime DLL or NVIDIA driver component" : "a required backend DLL"; const driverHint = isVulkan ? "Install or update the GPU vendor driver with Vulkan support, then run setup again so the Vulkan backend folder is repaired." : isCuda ? "Install or update the NVIDIA driver, then run setup again so the CUDA backend folder is repaired." : "Update the GPU driver, then run setup again so the backend folder is repaired."; return `exited with code ${code} (0xC0000135: required DLL not found).\n\nWindows could not start ${backendName} because ${likelyMissing} is missing or not loadable. ${driverHint}\n\nIf you are using an AMD/Intel GPU, update the AMD/Intel graphics driver first. If the GPU is too old for the current Vulkan backend, switch the backend to CPU.`; } return `exited with code ${code}`; } function getModelInfo(filename) { const safeFilename = path.basename(filename || ""); const fullPath = path.join(MODELS, safeFilename); const stats = fs.statSync(fullPath); if (stats.isDirectory()) { const dirSize = getPathSize(fullPath); return { filename: safeFilename, sizeBytes: dirSize, size: formatBytes(dirSize), format: "CoreML", }; } return { filename: safeFilename, sizeBytes: stats.size, size: formatBytes(stats.size), }; } function readBody(req) { return new Promise((resolve, reject) => { let data = ""; let bytes = 0; req.on("data", chunk => { bytes += chunk.length; if (bytes > MAX_JSON_BODY_BYTES) { reject(new Error(`Request body is too large. Limit is ${Math.round(MAX_JSON_BODY_BYTES / (1024 * 1024))} MB.`)); req.destroy(); return; } data += chunk; }); req.on("end", () => { try { resolve(JSON.parse(data || "{}")); } catch(e) { reject(new Error("Invalid JSON request body")); } }); req.on("error", reject); }); } async function readJsonBody(req, res) { try { return await readBody(req); } catch (err) { json(res, err.message.includes("too large") ? 413 : 400, { ok: false, error: err.message }); return null; } } function safeOutputName(value) { return String(value || "") .replace(/[^a-z0-9._-]/gi, "_") .replace(/_+/g, "_") .slice(0, 120); } function sanitizeChatMessageForStorage(message) { if (!message || typeof message !== "object") return null; const role = String(message.role || "").trim(); if (!["system", "user", "assistant"].includes(role)) return null; let content = message.content; if (Array.isArray(content)) { content = content.map((item) => { if (!item || typeof item !== "object") return item; if (item.type === "image_url") { return { type: "text", text: "[Attached image omitted from saved chat history]" }; } return item; }); } else if (typeof content !== "string") { content = content === undefined || content === null ? "" : String(content); } return { ...message, role, content, }; } function sanitizeChatConversationForStorage(conversation = {}) { const now = Date.now(); const id = safeOutputName(conversation.id || `chat_${now}`) || `chat_${now}`; const messages = Array.isArray(conversation.messages) ? conversation.messages.map(sanitizeChatMessageForStorage).filter(Boolean) : []; const timestamp = Number(conversation.timestamp) || now; return { id, title: String(conversation.title || "Chat Session").slice(0, 160), model: String(conversation.model || ""), messages, timestamp, createdAt: conversation.createdAt || new Date(timestamp).toISOString(), updatedAt: new Date().toISOString(), }; } function getChatConversationPath(id) { const safeId = safeOutputName(id); if (!safeId) throw new Error("A chat id is required."); const filePath = path.join(CHAT_HISTORY, `${safeId}.json`); if (!pathInside(filePath, CHAT_HISTORY)) { throw new Error("Invalid chat id."); } return filePath; } function saveChatConversation(conversation = {}) { const saved = sanitizeChatConversationForStorage(conversation); const filePath = getChatConversationPath(saved.id); fs.writeFileSync(filePath, JSON.stringify(saved, null, 2), "utf8"); return saved; } function listChatConversations() { try { return fs.readdirSync(CHAT_HISTORY) .filter((file) => file.toLowerCase().endsWith(".json")) .map((file) => { try { const filePath = path.join(CHAT_HISTORY, file); if (!pathInside(filePath, CHAT_HISTORY)) return null; const conversation = JSON.parse(fs.readFileSync(filePath, "utf8")); const stat = fs.statSync(filePath); return { ...conversation, filename: file, modifiedAt: stat.mtime.toISOString(), sizeBytes: stat.size, size: formatBytes(stat.size), }; } catch (_) { return null; } }) .filter(Boolean) .sort((a, b) => Number(b.timestamp || 0) - Number(a.timestamp || 0)); } catch (_) { return []; } } function deleteChatConversation(id) { const filePath = getChatConversationPath(id); if (fs.existsSync(filePath)) { fs.unlinkSync(filePath); return true; } return false; } function saveGeneratedOutput(imageDataUrl, metadata = {}) { const match = String(imageDataUrl || "").match(/^data:(image\/(?:png|jpeg|jpg|webp));base64,(.+)$/); if (!match) { throw new Error("Expected a real base64 PNG, JPEG, or WebP image data URL"); } const mime = match[1]; const extByMime = { "image/png": ".png", "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/webp": ".webp", }; const ext = extByMime[mime] || ".png"; const imageBuffer = Buffer.from(match[2], "base64"); const isPng = imageBuffer.length > 8 && imageBuffer[0] === 0x89 && imageBuffer[1] === 0x50 && imageBuffer[2] === 0x4e && imageBuffer[3] === 0x47; const isJpeg = imageBuffer.length > 3 && imageBuffer[0] === 0xff && imageBuffer[1] === 0xd8 && imageBuffer[2] === 0xff; const isWebp = imageBuffer.length > 12 && imageBuffer.toString("ascii", 0, 4) === "RIFF" && imageBuffer.toString("ascii", 8, 12) === "WEBP"; if ((mime === "image/png" && !isPng) || ((mime === "image/jpeg" || mime === "image/jpg") && !isJpeg) || (mime === "image/webp" && !isWebp)) { throw new Error("Generated image payload did not match its declared image format."); } const createdAt = new Date().toISOString(); const stamp = createdAt.replace(/[:.]/g, "-"); const seed = metadata.seed !== undefined && metadata.seed !== null ? `-${safeOutputName(metadata.seed)}` : ""; const baseName = `output-${stamp}${seed}`; const imageFilename = `${baseName}${ext}`; const metadataFilename = `${baseName}.json`; const imagePath = path.join(OUTPUTS, imageFilename); const metadataPath = path.join(OUTPUTS, metadataFilename); fs.writeFileSync(imagePath, imageBuffer); const savedMetadata = { ...metadata, createdAt, image: imageFilename, metadata: metadataFilename, }; fs.writeFileSync(metadataPath, JSON.stringify(savedMetadata, null, 2), "utf8"); console.log(` [api] Saved generated output: ${imageFilename}`); return savedMetadata; } function listGeneratedOutputs() { try { return fs.readdirSync(OUTPUTS) .filter(file => file.toLowerCase().endsWith(".json")) .map(file => { try { const metadata = JSON.parse(fs.readFileSync(path.join(OUTPUTS, file), "utf8")); return { ...metadata, url: `/api/output-file?filename=${encodeURIComponent(metadata.image)}`, }; } catch (_) { return null; } }) .filter(Boolean) .sort((a, b) => String(b.createdAt || "").localeCompare(String(a.createdAt || ""))); } catch (_) { return []; } } function deleteGeneratedOutputs(outputs = []) { const deleted = []; for (const output of outputs) { const imageFilename = path.basename(output?.image || output?.filename || ""); const metadataFilename = path.basename(output?.metadata || ""); if (!imageFilename && !metadataFilename) continue; const targets = new Set(); if (imageFilename) targets.add(imageFilename); if (metadataFilename) targets.add(metadataFilename); if (!metadataFilename && imageFilename) { const stem = imageFilename.replace(/\.[^.]+$/, ""); targets.add(`${stem}.json`); } for (const filename of targets) { const filePath = path.join(OUTPUTS, filename); if (fs.existsSync(filePath)) { fs.unlinkSync(filePath); deleted.push(filename); } } } return deleted; } function streamModelUpload(req, filename, targetDir = MODELS, mode = "image") { return new Promise((resolve, reject) => { const safeFilename = path.basename(filename || ""); const lowerName = safeFilename.toLowerCase(); const valid = mode === "text" ? lowerName.endsWith(".gguf") : mode === "speech" ? lowerName.endsWith(".bin") : mode === "tts" ? lowerName.endsWith(".json") : isModelFile(lowerName); if (!safeFilename || !valid) { reject(new Error(mode === "text" ? "Filename must end with .gguf" : mode === "speech" ? "Filename must end with .bin" : mode === "tts" ? "Filename must end with .json" : "Filename must end with .gguf, .safetensors, or .ckpt")); return; } const destPath = path.join(targetDir, safeFilename); const tempPath = `${destPath}.part`; const out = fs.createWriteStream(tempPath); let finished = false; const cleanupPartial = () => { if (finished) return; out.destroy(); try { fs.unlinkSync(tempPath); } catch (_) {} }; req.pipe(out); req.on("error", err => { cleanupPartial(); reject(err); }); req.on("aborted", () => { cleanupPartial(); reject(new Error("Import cancelled by user")); }); req.on("close", () => { if (!finished && req.aborted) { cleanupPartial(); } }); out.on("error", err => { cleanupPartial(); reject(err); }); out.on("finish", () => { try { finished = true; fs.renameSync(tempPath, destPath); console.log(` [api] Imported model file: ${safeFilename}`); if (mode === "text" || mode === "speech") { const stats = fs.statSync(destPath); resolve({ filename: safeFilename, name: safeFilename, sizeBytes: stats.size, size: formatBytes(stats.size), format: mode === "speech" ? "Whisper GGML" : "GGUF" }); } else { resolve(getModelInfo(safeFilename)); } } catch (err) { try { fs.unlinkSync(tempPath); } catch (_) {} reject(err); } }); }); } function json(res, code, obj) { const body = JSON.stringify(obj); res.writeHead(code, { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }); res.end(body); } // ── HTTP Server ─────────────────────────────────────────────────────────────── const server = http.createServer(async (req, res) => { // CORS preflight if (req.method === "OPTIONS") { res.writeHead(204, { "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "Content-Type", "Access-Control-Allow-Methods": "GET,POST" }); res.end(); return; } if ((req.url.startsWith("/v1/") || req.url.startsWith("/sdapi/")) && ["GET", "POST"].includes(req.method)) { return proxyImageBackendRequest(req, res); } // ── Management API ──────────────────────────────────────────────────────── // GET /api/health if (req.url === "/api/health" && req.method === "GET") { return json(res, 200, await getHealth()); } // GET /api/diagnostics if (req.url === "/api/diagnostics" && req.method === "GET") { return json(res, 200, { generatedAt: new Date().toISOString(), health: await getHealth(), cleanupCandidates: getCleanupCandidates(), download: downloadState, generation: generationState, llm: { ready: llmReady, running: llmProc !== null, error: llmError, settings: llmSettings, port: PORT_LLM, }, speech: { ready: speechReady, error: speechError, settings: speechSettings, backend: getSpeechBackend(), transcription: speechTranscriptionState, }, hardware: getHardwareSpecs(), telemetry: getTelemetry(), }); } // GET /api/cleanup-candidates if (req.url === "/api/cleanup-candidates" && req.method === "GET") { return json(res, 200, { candidates: getCleanupCandidates() }); } // POST /api/cleanup if (req.url === "/api/cleanup" && req.method === "POST") { try { const body = await readJsonBody(req, res); if (!body) return; const deleted = cleanupSelected(Array.isArray(body.ids) ? body.ids : []); return json(res, 200, { ok: true, deleted }); } catch (err) { console.error(" [api] Cleanup failed:", err); return json(res, 500, { ok: false, error: err.message }); } } // GET /api/backend-status if (req.url === "/api/backend-status" && req.method === "GET") { return json(res, 200, { ready: backendReady || openvinoReady, running: backendProc !== null || openvinoProc !== null, port: PORT_BACKEND, preferredPort: PREFERRED_BACKEND_PORT, error: backendError, loading: backendLoadState, unloading: backendUnloadState, settings: currentSettings, build: SERVER_BUILD, }); } if (req.url === "/api/llm/status" && req.method === "GET") { const backend = getLlmBackend(); const probe = getLlmBackendProbe(); return json(res, 200, { ready: llmReady, running: llmProc !== null, port: PORT_LLM, preferredPort: PREFERRED_LLM_PORT, error: llmError, settings: llmSettings, backendInstalled: Boolean(backend), backendMode: backend?.mode || "", backendPath: backend?.path || "", selectedBackend: probe.selected, availableBackends: probe.available, }); } if (req.url.startsWith("/api/llm/backends") && req.method === "GET") { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); return json(res, 200, { ok: true, ...getLlmBackendProbe(parsed.searchParams.get("refresh") === "1") }); } if (req.url === "/api/llm/stats" && req.method === "GET") { return json(res, 200, getLlmRuntimeStats()); } if (req.url === "/api/llm/model-settings" && req.method === "GET") { return json(res, 200, { ok: true, ...getPersistedLlmModelSettings() }); } if (req.url === "/api/llm/model-settings" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const filename = path.basename(String(body.model || "")); if (!filename) return json(res, 400, { ok: false, error: "model is required" }); const updated = updateLlmModelSettings(filename, { preferredBackend: body.preferredBackend ? String(body.preferredBackend).toLowerCase() : undefined, }); return json(res, 200, { ok: true, model: filename, settings: updated }); } if (req.url === "/api/llm/conversations" && req.method === "GET") { return json(res, 200, { ok: true, conversations: listChatConversations() }); } if (req.url === "/api/llm/save-conversation" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { const conversation = saveChatConversation(body.conversation || body); return json(res, 200, { ok: true, conversation }); } catch (err) { console.error(" [api] Failed to save chat conversation:", err); return json(res, 500, { ok: false, error: err.message || String(err) }); } } if (req.url === "/api/llm/delete-conversation" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { const deleted = deleteChatConversation(body.id); return json(res, 200, { ok: true, deleted }); } catch (err) { console.error(" [api] Failed to delete chat conversation:", err); return json(res, 500, { ok: false, error: err.message || String(err) }); } } if (req.url === "/api/llm/models" && req.method === "GET") { return json(res, 200, { models: getLlmModels() }); } if (req.url.startsWith("/api/huggingface/models") && req.method === "GET") { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); const query = String(parsed.searchParams.get("query") || "").slice(0, 120); const filters = String(parsed.searchParams.get("filters") || "") .split(",") .map((value) => value.trim().toLowerCase()) .filter((value) => ["vision", "uncensored"].includes(value)); const page = Math.max(1, Math.min(20, Number(parsed.searchParams.get("page")) || 1)); const pageSize = 9; try { const rankedModels = await searchHuggingFaceModels(query, [...new Set(filters)]); const start = (page - 1) * pageSize; const pageModels = rankedModels.slice(start, start + pageSize); const models = await Promise.all(pageModels.map(addHuggingFaceFileSize)); return json(res, 200, { ok: true, source: "huggingface", models, page, hasMore: start + models.length < rankedModels.length, }); } catch (err) { return json(res, 502, { ok: false, error: err.message || "Hugging Face search failed." }); } } if (req.url === "/api/llm/start" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { await runExclusiveLlmOperation(() => startLlm(body)); return json(res, 200, { ok: true, ready: llmReady, port: PORT_LLM, settings: llmSettings }); } catch (err) { llmError = err.message || String(err); if (err.code !== "MODEL_ALREADY_ACTIVE") { await runExclusiveLlmOperation(() => killLlm()); } return json(res, jsonErrorStatus(err), { ok: false, error: llmError, code: err.code || "", activeRuntime: err.activeRuntime || null }); } } if (req.url === "/api/llm/stop" && req.method === "POST") { await runExclusiveLlmOperation(() => killLlm()); return json(res, 200, { ok: true }); } if (req.url === "/api/llm/benchmark" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { const result = await runExclusiveLlmOperation(() => benchmarkLlmModel(body)); return json(res, 200, result); } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); } } if (req.url === "/api/speech/status" && req.method === "GET") { const backend = getSpeechBackend(); return json(res, 200, { ok: true, ready: speechReady, running: speechReady, port: PORT_SPEECH, preferredPort: PREFERRED_SPEECH_PORT, backendInstalled: Boolean(backend.cli), backendPath: backend.cli || "", serverPath: backend.server || "", backendMode: backend.mode, backendPreference: backend.preference, backends: backend.candidates, error: speechError, settings: speechSettings, transcription: speechTranscriptionState, }); } if (req.url === "/api/speech/models" && req.method === "GET") { return json(res, 200, { ok: true, models: getSpeechModels() }); } if (req.url === "/api/speech/start" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { await runExclusiveSpeechOperation(() => startSpeech(body)); return json(res, 200, { ok: true, ready: speechReady, settings: speechSettings, port: PORT_SPEECH }); } catch (err) { speechError = err.message || String(err); return json(res, jsonErrorStatus(err), { ok: false, error: speechError, code: err.code || "", activeRuntime: err.activeRuntime || null }); } } if (req.url === "/api/speech/stop" && req.method === "POST") { await stopSpeech(); return json(res, 200, { ok: true }); } if (req.url.startsWith("/api/speech/transcribe") && req.method === "POST") { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); try { const audio = await readBinaryBody(req); const suppressNonSpeech = parsed.searchParams.get("suppressNonSpeech"); const noSpeechThreshold = parsed.searchParams.get("noSpeechThreshold"); const transcribeOptions = { model: parsed.searchParams.get("model") || speechSettings.model, language: parsed.searchParams.get("language") || "auto", filename: parsed.searchParams.get("filename") || req.headers["x-filename"] || "recording", threads: parsed.searchParams.get("threads") || speechSettings.threads, backendPreference: parsed.searchParams.get("backendPreference") || speechSettings.backendPreference, translate: parsed.searchParams.get("translate") === "true", }; if (suppressNonSpeech !== null) transcribeOptions.suppressNonSpeech = suppressNonSpeech !== "false"; if (noSpeechThreshold !== null) transcribeOptions.noSpeechThreshold = noSpeechThreshold; const result = await runExclusiveSpeechOperation(() => transcribeAudioBuffer(audio, transcribeOptions)); return json(res, 200, { ok: true, transcription: result }); } catch (err) { speechError = err.message || String(err); return json(res, err.message?.includes("too large") ? 413 : 500, { ok: false, error: speechError }); } } // API endpoint that returns transcription as downloadable file (txt, json, md) if (req.url.startsWith("/api/speech/transcribe-file") && req.method === "POST") { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); try { const audio = await readBinaryBody(req); const outputFormat = parsed.searchParams.get("format") || "txt"; const suppressNonSpeech = parsed.searchParams.get("suppressNonSpeech"); const noSpeechThreshold = parsed.searchParams.get("noSpeechThreshold"); const transcribeOptions = { model: parsed.searchParams.get("model") || speechSettings.model, language: parsed.searchParams.get("language") || "auto", filename: parsed.searchParams.get("filename") || req.headers["x-filename"] || "recording", threads: parsed.searchParams.get("threads") || speechSettings.threads, backendPreference: parsed.searchParams.get("backendPreference") || speechSettings.backendPreference, translate: parsed.searchParams.get("translate") === "true", }; if (suppressNonSpeech !== null) transcribeOptions.suppressNonSpeech = suppressNonSpeech !== "false"; if (noSpeechThreshold !== null) transcribeOptions.noSpeechThreshold = noSpeechThreshold; const result = await runExclusiveSpeechOperation(() => transcribeAudioBuffer(audio, transcribeOptions)); const filename = parsed.searchParams.get("output_filename") || `transcription-${Date.now()}`; let fileContent = ""; let contentType = "text/plain;charset=utf-8"; let extension = "txt"; if (outputFormat === "json") { fileContent = JSON.stringify({ text: result.text, model: result.model, modelName: result.modelName, language: result.language, backend: result.backend, backendMode: result.backendMode, sourceFilename: result.sourceFilename, durationMs: result.durationMs, raw: result.raw, createdAt: result.createdAt }, null, 2); contentType = "application/json;charset=utf-8"; extension = "json"; } else if (outputFormat === "md") { fileContent = `# Trascrizione\n\n${result.text}\n\n---\n*Modello: ${result.modelName || result.model || "unknown"}\n*Lingua: ${result.language || "auto"}\n*Backend: ${result.backend || "unknown"} (${result.backendMode || ""})\n*File sorgente: ${result.sourceFilename || "unknown"}\n*Durata elaborazione: ${result.durationMs ? (result.durationMs / 1000).toFixed(1) : "?"}s\n*Generato il: ${new Date().toISOString()}`; contentType = "text/markdown;charset=utf-8"; extension = "md"; } else { fileContent = result.text; contentType = "text/plain;charset=utf-8"; extension = "txt"; } res.writeHead(200, { "Content-Type": contentType, "Content-Disposition": `attachment; filename="${filename}.${extension}"`, "Content-Length": Buffer.byteLength(fileContent, "utf8") }); res.end(fileContent); } catch (err) { speechError = err.message || String(err); return json(res, err.message?.includes("too large") ? 413 : 500, { ok: false, error: speechError }); } } if (req.url === "/api/speech/download-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; if (downloadState.active) return json(res, 409, { ok: false, error: "Another model download is already active." }); const modelId = String(body.modelId || body.model_id || body.model || ""); const catalogModel = SPEECH_MODEL_CATALOG.find((model) => model.id === modelId || model.filename === modelId); const url = body.url ? String(body.url) : (catalogModel ? `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${catalogModel.filename}` : ""); const filename = path.basename(String(body.filename || catalogModel?.filename || "")); if (!url || !filename || !filename.toLowerCase().endsWith(".bin")) { return json(res, 400, { ok: false, error: "A speech model .bin URL or catalog model id is required." }); } startModelDownload(url, filename, SPEECH_MODELS, "speech"); return json(res, 200, { ok: true, message: "Speech model download started", filename }); } if (req.url.startsWith("/api/speech/import-model") && req.method === "POST") { try { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); const result = await streamModelUpload(req, parsed.searchParams.get("filename"), SPEECH_MODELS, "speech"); return json(res, 200, { ok: true, message: `Imported ${result.filename}`, model: result }); } catch (err) { console.error(" [api] Failed to import speech model:", err); return json(res, 500, { ok: false, error: err.message }); } } if (req.url === "/api/speech/delete-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const filename = path.basename(String(body.filename || "")); const modelPath = path.join(SPEECH_MODELS, filename); if (!filename || !pathInside(modelPath, SPEECH_MODELS)) { return json(res, 400, { ok: false, error: "Invalid filename" }); } if (speechSettings.model === filename) await stopSpeech(); try { fs.unlinkSync(modelPath); return json(res, 200, { ok: true }); } catch (err) { return json(res, err.code === "ENOENT" ? 404 : 500, { ok: false, error: err.code === "ENOENT" ? "Speech model not found" : err.message }); } } if (req.url === "/api/speech/transcriptions" && req.method === "GET") { return json(res, 200, { ok: true, transcriptions: listTranscriptions() }); } if (req.url === "/api/speech/delete-transcription" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const filename = path.basename(String(body.filename || "")); if (!filename || !filename.endsWith(".json")) { return json(res, 400, { ok: false, error: "Invalid filename" }); } const jsonPath = path.join(TRANSCRIPTIONS, filename); if (!pathInside(jsonPath, TRANSCRIPTIONS)) { return json(res, 400, { ok: false, error: "Invalid path" }); } try { if (!fs.existsSync(jsonPath)) { return json(res, 404, { ok: false, error: "Transcription metadata not found" }); } const metadata = JSON.parse(fs.readFileSync(jsonPath, "utf8")); const textFile = metadata.textFile ? path.basename(String(metadata.textFile)) : ""; fs.unlinkSync(jsonPath); if (textFile) { const textPath = path.join(TRANSCRIPTIONS, textFile); if (pathInside(textPath, TRANSCRIPTIONS) && fs.existsSync(textPath)) { fs.unlinkSync(textPath); } } return json(res, 200, { ok: true }); } catch (err) { console.error(" [api] Failed to delete transcription:", err); return json(res, 500, { ok: false, error: err.message }); } } if (req.url === "/api/tts/status" && req.method === "GET") { const runtime = getTtsRuntimeStatus(); return json(res, 200, { ok: true, ready: ttsReady, running: ttsReady, port: PORT_TTS, preferredPort: PREFERRED_TTS_PORT, runtimeInstalled: runtime.installed, runtimePath: runtime.runtime, workerPath: runtime.worker, cachePath: runtime.cache, backendMode: runtime.backendMode, error: ttsError, settings: ttsSettings, generation: ttsGenerationState, voices: TTS_VOICES, }); } if (req.url === "/api/tts/models" && req.method === "GET") { return json(res, 200, { ok: true, models: getTtsModels(), voices: TTS_VOICES }); } if (req.url === "/api/tts/start" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { await runExclusiveTtsOperation(() => startTts(body)); return json(res, 200, { ok: true, ready: ttsReady, settings: ttsSettings, port: PORT_TTS }); } catch (err) { ttsError = err.message || String(err); return json(res, jsonErrorStatus(err), { ok: false, error: ttsError, code: err.code || "", activeRuntime: err.activeRuntime || null }); } } if (req.url === "/api/tts/stop" && req.method === "POST") { await stopTts(); return json(res, 200, { ok: true }); } if (req.url === "/api/tts/speak" && req.method === "POST") { const body = await readJsonBody(req, res, 512 * 1024); if (!body) return; try { if (!ttsReady || (body.model && ttsSettings.model !== body.model)) { await runExclusiveTtsOperation(() => startTts(body)); } const output = await runExclusiveTtsOperation(() => synthesizeTts(body.text, body)); return json(res, 200, { ok: true, output }); } catch (err) { ttsError = err.message || String(err); return json(res, jsonErrorStatus(err), { ok: false, error: ttsError, code: err.code || "", activeRuntime: err.activeRuntime || null }); } } if (req.url === "/api/tts/download-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { const model = installTtsCatalogModel(body.modelId || body.model_id || body.model || body.filename); return json(res, 200, { ok: true, message: "TTS model manifest installed", filename: model.filename, model }); } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); } } if (req.url.startsWith("/api/tts/import-model") && req.method === "POST") { try { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); const result = await streamModelUpload(req, parsed.searchParams.get("filename"), TTS_MODELS, "tts"); return json(res, 200, { ok: true, message: `Imported ${result.filename}`, model: result }); } catch (err) { console.error(" [api] Failed to import TTS model:", err); return json(res, 500, { ok: false, error: err.message }); } } if (req.url === "/api/tts/delete-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const filename = path.basename(String(body.filename || "")); const modelPath = path.join(TTS_MODELS, filename); if (!filename || !filename.endsWith(".json") || !pathInside(modelPath, TTS_MODELS)) { return json(res, 400, { ok: false, error: "Invalid filename" }); } if (ttsSettings.model === filename) await stopTts(); try { fs.unlinkSync(modelPath); return json(res, 200, { ok: true }); } catch (err) { return json(res, err.code === "ENOENT" ? 404 : 500, { ok: false, error: err.code === "ENOENT" ? "TTS model not found" : err.message }); } } if (req.url === "/api/tts/outputs" && req.method === "GET") { return json(res, 200, { ok: true, outputs: listTtsOutputs() }); } if (req.url === "/api/tts/delete-output" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const filename = path.basename(String(body.filename || "")); if (!filename || !filename.endsWith(".json")) { return json(res, 400, { ok: false, error: "Invalid filename" }); } const jsonPath = path.join(TTS_OUTPUTS, filename); if (!pathInside(jsonPath, TTS_OUTPUTS)) { return json(res, 400, { ok: false, error: "Invalid path" }); } try { if (!fs.existsSync(jsonPath)) { return json(res, 404, { ok: false, error: "TTS output metadata not found" }); } const metadata = JSON.parse(fs.readFileSync(jsonPath, "utf8")); const audioFile = metadata.audioFile ? path.basename(String(metadata.audioFile)) : ""; fs.unlinkSync(jsonPath); if (audioFile) { const audioPath = path.join(TTS_OUTPUTS, audioFile); if (pathInside(audioPath, TTS_OUTPUTS) && fs.existsSync(audioPath)) { fs.unlinkSync(audioPath); } } return json(res, 200, { ok: true }); } catch (err) { console.error(" [api] Failed to delete TTS output:", err); return json(res, 500, { ok: false, error: err.message }); } } function isOomError(msg) { if (!msg) return false; const lower = String(msg).toLowerCase(); return lower.includes("out of memory") || lower.includes("failed to allocate") || lower.includes("failed to initialize the context") || lower.includes("oom") || lower.includes("not enough memory") || lower.includes("allocation failed") || lower.includes("failed to create context"); } async function retryLowerContext() { const currentCtx = llmSettings.contextSize || 4096; const contextLadder = [32768, 24576, 16384, 12288, 8192, 4096, 2048, 1024, 512]; const nextLimit = contextLadder.find(limit => limit < currentCtx); if (!nextLimit) { throw new Error("Cannot lower context size any further."); } console.log(` [llm] Retrying with lower context limit: ${nextLimit} (was ${currentCtx})`); const originalModel = llmSettings.model; const newSettings = { model: originalModel, threads: llmSettings.threads, contextSize: nextLimit, gpuLayers: llmSettings.gpuLayers, enableThinking: llmSettings.enableThinking, }; await runExclusiveLlmOperation(() => startLlm(newSettings)); } function getMessageText(content) { if (typeof content === "string") return content; if (Array.isArray(content)) { return content.map((item) => { if (typeof item === "string") return item; if (typeof item?.text === "string") return item.text; if (typeof item?.content === "string") return item.content; return ""; }).join("\n"); } if (content && typeof content === "object") { if (typeof content.text === "string") return content.text; if (typeof content.content === "string") return content.content; } return ""; } function getLastUserQuery(messages) { for (let i = messages.length - 1; i >= 0; i -= 1) { if (messages[i]?.role === "user") { return getMessageText(messages[i].content).trim(); } } return ""; } async function augmentMessagesWithWebSearch(messages, body) { if (body.useWeb !== true && body.use_web !== true) { return { messages, webSources: [], webContext: "" }; } const query = String(body.webQuery || body.query || getLastUserQuery(messages) || "").trim(); if (!query) return { messages, webSources: [], webContext: "" }; const result = await comprehensiveWebSearch(query, { timeFilter: body.timeFilter || body.time_filter || "any", resultLimit: body.webResultLimit || 5, fetchLimit: body.webFetchLimit || 3, cacheDir: WEB_SEARCH_CACHE, }); if (!result.context || !result.sources.length) { return { messages, webSources: [], webContext: "" }; } let insertAt = 0; while (insertAt < messages.length && messages[insertAt]?.role === "system") { insertAt += 1; } const augmentedMessages = [ ...messages.slice(0, insertAt), { role: "user", content: result.context }, ...messages.slice(insertAt), ]; return { messages: augmentedMessages, webSources: result.sources, webContext: result.context, }; } async function doLlmChat(req, res, body, retryCount = 0) { try { const isStream = body.stream === true; const rawMessages = Array.isArray(body.messages) ? body.messages : []; const webAugmentation = await augmentMessagesWithWebSearch(rawMessages, body); const requestData = JSON.stringify({ model: llmSettings.model || "local-model", messages: webAugmentation.messages, temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : 0.7, max_tokens: Math.max(1, Math.min(4096, Number(body.max_tokens) || Number(body.maxTokens) || 1024)), stream: isStream, // New sampling parameters top_p: Number.isFinite(Number(body.top_p)) ? Number(body.top_p) : 0.95, top_k: Number.isFinite(Number(body.top_k)) ? Number(body.top_k) : 40, min_p: Number.isFinite(Number(body.min_p)) ? Number(body.min_p) : 0.05, repeat_penalty: Number.isFinite(Number(body.repeat_penalty)) ? Number(body.repeat_penalty) : 1.1, frequency_penalty: Number.isFinite(Number(body.frequency_penalty)) ? Number(body.frequency_penalty) : 0.0, presence_penalty: Number.isFinite(Number(body.presence_penalty)) ? Number(body.presence_penalty) : 0.0, seed: Number.isInteger(Number(body.seed)) ? Number(body.seed) : undefined, stop: Array.isArray(body.stop) ? body.stop : (body.stop ? [body.stop] : undefined), }); if (isStream) { const clientReq = http.request({ hostname: "127.0.0.1", port: PORT_LLM, path: "/v1/chat/completions", method: "POST", headers: { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(requestData), }, agent: llmHttpAgent, // ✅ HTTP keep-alive: eliminates TCP handshake per turn }, (clientRes) => { if (clientRes.statusCode < 200 || clientRes.statusCode >= 300) { let errorBody = ""; clientRes.setEncoding("utf8"); clientRes.on("data", (chunk) => { errorBody += chunk; }); clientRes.on("end", async () => { let message = `Text backend returned HTTP ${clientRes.statusCode}`; try { message = JSON.parse(errorBody || "{}").error?.message || message; } catch (_) {} if (isOomError(message) && retryCount === 0) { try { await retryLowerContext(); return doLlmChat(req, res, body, retryCount + 1); } catch (retryErr) { console.error("Failed OOM recovery retry:", retryErr); } } json(res, clientRes.statusCode || 500, { ok: false, error: message }); }); return; } res.writeHead(200, { "Content-Type": "text/event-stream; charset=utf-8", "Cache-Control": "no-cache, no-transform", "Connection": "keep-alive", "X-Accel-Buffering": "no", "X-Content-Type-Options": "nosniff", }); const socket = res.socket || res.connection; socket?.setNoDelay?.(true); res.flushHeaders?.(); if (webAugmentation.webSources.length) { res.write(`event: web_sources\ndata: ${JSON.stringify({ sources: webAugmentation.webSources })}\n\n`); } clientRes.on("data", (chunk) => res.write(chunk)); clientRes.on("end", () => res.end()); clientRes.on("error", (err) => res.destroy(err)); }); clientReq.on("error", async (err) => { console.error("LLM stream request error:", err); if (isOomError(err.message) && retryCount === 0) { try { await retryLowerContext(); return doLlmChat(req, res, body, retryCount + 1); } catch (retryErr) { console.error("Failed OOM recovery retry:", retryErr); } } if (!res.headersSent) { res.writeHead(500, { "Content-Type": "application/json" }); res.end(JSON.stringify({ ok: false, error: err.message })); } else { res.end(); } }); clientReq.write(requestData); clientReq.end(); res.on("close", () => { if (!res.writableEnded && !clientReq.destroyed) clientReq.destroy(); }); return; } else { try { const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, JSON.parse(requestData), 300000); if (webAugmentation.webSources.length) { result.web_sources = webAugmentation.webSources; } return json(res, 200, result); } catch (err) { if (isOomError(err.message) && retryCount === 0) { try { await retryLowerContext(); return doLlmChat(req, res, body, retryCount + 1); } catch (retryErr) { console.error("Failed OOM recovery retry:", retryErr); } } throw err; } } } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); } } function getLlmRuntimeStats() { const benchmarks = getPersistedLlmBenchmarks(); return { ok: true, ready: llmReady, running: llmProc !== null, selectedBackend: getLlmBackendProbe().selected, settings: llmSettings, modelSettings: getPersistedLlmModelSettings(), benchmarks: (benchmarks.results || []).slice(0, 20), }; } async function benchmarkLlmBackend(model, backend, baseSettings = {}) { const prompt = String(baseSettings.prompt || "Reply with one short sentence about local AI performance."); const startedAt = Date.now(); await startLlmWithBackend({ ...baseSettings, model, contextSize: Math.min(2048, Number(baseSettings.contextSize) || 2048), gpuLayers: Number.isFinite(Number(baseSettings.gpuLayers)) ? Number(baseSettings.gpuLayers) : -1, enableThinking: false, }, backend); const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, { model: llmSettings.model || "local-model", messages: [ { role: "system", content: "You are a concise local benchmark assistant." }, { role: "user", content: prompt }, ], temperature: 0.2, max_tokens: 64, stream: false, }, 180000); const finishedAt = Date.now(); const timings = result.timings || {}; return { backendKey: backend.key, backendMode: llmSettings.backendMode || backend.mode, backendBinary: llmSettings.backendBinary || path.basename(backend.path), model, ok: true, total_ms: finishedAt - startedAt, prompt_ms: Number(timings.prompt_ms) || null, predicted_ms: Number(timings.predicted_ms) || null, predicted_n: Number(timings.predicted_n) || null, predicted_per_second: Number(timings.predicted_per_second) || null, prompt_per_second: Number(timings.prompt_per_second) || null, }; } async function benchmarkLlmModel(settings = {}) { const filename = path.basename(String(settings.model || llmSettings.model || "")); const modelPath = path.join(LLM_MODELS, filename); if (!filename || !pathInside(modelPath, LLM_MODELS) || !fs.existsSync(modelPath)) { throw new Error("Select a downloaded GGUF text model before benchmarking."); } const previous = { ready: llmReady, model: llmSettings.model, settings: { ...llmSettings }, }; const requestedBackends = Array.isArray(settings.backends) ? settings.backends.map((item) => String(item).toLowerCase()) : []; const candidates = getLlmBackendCandidates() .filter((backend) => backend.key !== "cpu" || settings.includeCpu !== false) .filter((backend) => requestedBackends.length === 0 || requestedBackends.includes(backend.key)); if (candidates.length === 0) { throw new Error("No available text backends were detected for benchmarking."); } const results = []; for (const backend of candidates) { try { const result = await benchmarkLlmBackend(filename, backend, settings); results.push(recordLlmBenchmark(result)); } catch (err) { results.push(recordLlmBenchmark({ backendKey: backend.key, backendMode: backend.mode, backendBinary: path.basename(backend.path), model: filename, ok: false, error: (err.message || String(err)).slice(-800), })); await killLlm(); } } const successful = results .filter((item) => item.ok && Number(item.predicted_per_second) > 0) .sort((a, b) => Number(b.predicted_per_second) - Number(a.predicted_per_second)); const winner = successful[0] || null; if (winner) { updateLlmModelSettings(filename, { preferredBackend: winner.backendKey, benchmarkWinner: { backendMode: winner.backendMode, predicted_per_second: winner.predicted_per_second, createdAt: winner.createdAt, }, }); } if (previous.ready && previous.model && previous.model === filename && winner) { try { await startLlm({ ...previous.settings, model: filename, preferredBackend: winner.backendKey }); } catch (err) { console.warn(" [llm] Failed to restore text model after benchmark:", err.message || err); } } return { ok: true, model: filename, winner, results }; } // ── llmfit Integration ────────────────────────────────────────────────────────── let cachedLlmfitResults = new Map(); function getHardwareHash() { const cpus = os.cpus(); const gpu = getGpuInfo(); const ram = os.totalmem(); return `${cpus.length}-${cpus[0]?.model || 'unknown'}-${gpu.name}-${ram}`; } async function getLlmfitRecommendations(useCase = "chat", limit = 10) { const hardwareHash = getHardwareHash(); const cacheKey = `${hardwareHash}:${useCase}:${limit}`; const cached = cachedLlmfitResults.get(cacheKey); if (cached && (Date.now() - cached.timestamp) < 3600000) { // 1 hour cache return cached.data; } // Try to find llmfit binary const llmfitPaths = [ path.join(ROOT, "app", "tools", "llmfit", osPlatform === "win32" ? "llmfit.exe" : "llmfit"), "llmfit", // PATH ]; let llmfitPath = null; for (const p of llmfitPaths) { try { if (fs.existsSync(p)) { llmfitPath = p; break; } if (osPlatform !== "win32") { const whichResult = spawnSync("which", [p], { stdio: "ignore" }); if (whichResult.status === 0) { llmfitPath = p; break; } } } catch (_) {} } if (!llmfitPath) return null; // Fallback to tier table try { const result = spawnSync(llmfitPath, [ "recommend", "--json", "--limit", String(limit), "--use-case", useCase, "--force-runtime", "llamacpp" ], { encoding: "utf8", timeout: 8000, maxBuffer: 1024 * 1024, windowsHide: true, }); if (result.status !== 0) return null; const data = JSON.parse(result.stdout); cachedLlmfitResults.set(cacheKey, { timestamp: Date.now(), data }); return data; } catch (_) { return null; } } if (req.url === "/api/web-search/config" && req.method === "GET") { return json(res, 200, { ok: true, provider: "duckduckgo", providers: [{ key: "duckduckgo", name: "DuckDuckGo", requiresApiKey: false, active: true }], cachePath: WEB_SEARCH_CACHE, defaults: { resultLimit: 5, fetchLimit: 3, timeFilter: "any" }, }); } if (req.url === "/api/web-search" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { const result = await comprehensiveWebSearch(body.query || body.q || "", { timeFilter: body.timeFilter || body.time_filter || "any", resultLimit: body.resultLimit || 5, fetchLimit: body.fetchLimit || 3, cacheDir: WEB_SEARCH_CACHE, }); return json(res, 200, { ok: true, ...result }); } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); } } if (req.url === "/api/llm/chat" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; if (!llmReady) return json(res, 409, { ok: false, error: "Load a text model before sending a message." }); await doLlmChat(req, res, body); return; } // GET /api/llm/recommend — llmfit recommendations if (req.url.startsWith("/api/llm/recommend") && req.method === "GET") { const url = new URL(req.url, `http://${req.headers.host || "localhost"}`); const useCase = url.searchParams.get("useCase") || "chat"; const limit = Math.min(20, Math.max(1, Number(url.searchParams.get("limit")) || 10)); const recommendations = await getLlmfitRecommendations(useCase, limit); return json(res, 200, { ok: true, recommendations, source: recommendations ? "llmfit" : "fallback" }); } // GET /api/hardware-specs if (req.url === "/api/hardware-specs" && req.method === "GET") { return json(res, 200, getHardwareSpecs()); } // GET /api/backend-options if (req.url === "/api/backend-options" && req.method === "GET") { return json(res, 200, getBackendOptions()); } // GET /api/telemetry if (req.url === "/api/telemetry" && req.method === "GET") { return json(res, 200, getTelemetry()); } // GET /api/models — list available model files if (req.url === "/api/models" && req.method === "GET") { try { let files = fs.readdirSync(MODELS).filter(isModelFile).map(getModelInfo); if (downloadState.active && downloadState.filename) { files = files.filter(model => model.filename !== downloadState.filename); } return json(res, 200, { models: files }); } catch (_) { return json(res, 200, { models: [] }); } } if (req.url === "/api/openvino-models" && req.method === "GET") { const npuInfo = getOpenVinoNpuInfo(); return json(res, 200, { supported: npuInfo.supported, reason: npuInfo.reason || "", npu: npuInfo.npu || null, python: npuInfo.python || "", models: npuInfo.supported ? getOpenVinoModelInfo() : [], }); } // POST /api/restart-backend — restart with new settings if (req.url === "/api/restart-backend" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; console.log(" [api] Restart backend request:", body); const newSettings = {}; if (body.model) newSettings.model = body.backend_type === "openvino-npu" ? String(body.model) : path.join(MODELS, body.model); if (body.steps) newSettings.steps = parseInt(body.steps); if (body.cfgScale) newSettings.cfgScale = parseFloat(body.cfgScale); if (body.sampler) newSettings.sampler = body.sampler; if (body.threads) newSettings.threads = parseInt(body.threads); if (body.width) newSettings.width = parseInt(body.width); if (body.height) newSettings.height = parseInt(body.height); if (typeof body.use_gpu === "boolean") newSettings.useGpu = body.use_gpu; if (body.backend_type) { newSettings.backendType = String(body.backend_type); newSettings.useGpu = body.backend_type !== "cpu"; } if (typeof body.vae_tiling === "boolean") newSettings.vaeTiling = body.vae_tiling; if (typeof body.vae_on_cpu === "boolean") newSettings.vaeOnCpu = body.vae_on_cpu; if (typeof body.flash_attn === "boolean") newSettings.flashAttn = body.flash_attn; try { const targetModel = newSettings.model || currentSettings.model || getDefaultModel(); assertNoOtherActiveRuntime("image", targetModel); const requestedSettings = { ...currentSettings, ...newSettings, model: targetModel }; const imageBackendReady = (backendReady && backendProc) || (openvinoReady && openvinoProc); if (imageBackendReady && backendSettingsMatch(currentSettings, requestedSettings)) { return json(res, 200, { ok: true, message: "Backend already running with requested settings.", settings: currentSettings, port: PORT_BACKEND }); } if (backendReady && backendProc && appleNpuRuntimeMatches(currentSettings, requestedSettings)) { currentSettings = requestedSettings; return json(res, 200, { ok: true, message: "Apple NPU backend already running; updated generation defaults.", settings: currentSettings, port: PORT_BACKEND }); } await killBackend(); await new Promise(r => setTimeout(r, 500)); if (newSettings.backendType === "openvino-npu") { startBackend(newSettings).catch((err) => { backendError = err.message || String(err); backendLoadState = { ...backendLoadState, active: false, phase: "OpenVINO NPU model load failed", }; console.error(" [openvino-npu] Startup failed:", backendError); }); return json(res, 200, { ok: true, message: "OpenVINO NPU backend starting...", settings: currentSettings, port: PORT_BACKEND }); } if (newSettings.backendType === "apple-npu") { const backendPort = await findAvailableBackendPort(); PORT_BACKEND = backendPort; newSettings.backendPort = backendPort; currentSettings = { ...currentSettings, ...newSettings }; backendLoadState = { active: true, phase: "Starting Apple NPU backend...", progress: 1, current: 0, total: 0, speed: "", model: path.basename(newSettings.model || targetModel), backendMode: "Apple NPU", backendBinary: path.basename(getCoreMLPythonPath()), device: "", }; startBackend(newSettings).catch((err) => { backendError = err.message || String(err); backendLoadState = { ...backendLoadState, active: false, phase: "Apple NPU model load failed", }; console.error(" [coreml-npu] Startup failed:", backendError); }); return json(res, 200, { ok: true, message: "Apple NPU backend starting...", settings: { ...currentSettings, ...newSettings }, port: backendPort }); } await startBackend(newSettings); return json(res, 200, { ok: true, message: "Backend restarting...", settings: currentSettings, port: PORT_BACKEND }); } catch (err) { backendError = err.message || String(err); return json(res, jsonErrorStatus(err), { ok: false, error: backendError, code: err.code || "", activeRuntime: err.activeRuntime || null, port: PORT_BACKEND }); } } // POST /api/stop-backend if (req.url === "/api/stop-backend" && req.method === "POST") { await killBackend(); await killOpenVinoWorker(); return json(res, 200, { ok: true }); } // POST /api/download-backend if (req.url === "/api/download-backend" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; if (downloadState.active) return json(res, 409, { ok: false, error: "Another download is already active." }); try { const backendId = String(body.backend_id || body.backendId || body.id || ""); startImageBackendDownload(backendId); return json(res, 200, { ok: true, message: "Backend download started", backendId }); } catch (err) { return json(res, 400, { ok: false, error: err.message || String(err) }); } } // POST /api/download-model if (req.url === "/api/download-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const { url } = body; if (!url) return json(res, 400, { error: "URL is required" }); startModelDownload(url); return json(res, 200, { ok: true, message: "Download started" }); } if (req.url === "/api/llm/download-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const url = String(body.url || ""); if (!url) return json(res, 400, { ok: false, error: "URL is required" }); if (downloadState.active) return json(res, 409, { ok: false, error: "Another model download is already active." }); try { const directUrl = url.includes("huggingface.co") ? url.replace("/blob/", "/resolve/") : url; const overrideFilename = body.filename ? path.basename(String(body.filename)) : null; const filename = overrideFilename || path.basename(new URL(directUrl).pathname); if (!filename.toLowerCase().endsWith(".gguf")) { return json(res, 400, { ok: false, error: "Text model URL must point to a .gguf file." }); } const projector = body.projectorUrl && body.projectorFilename ? { url: String(body.projectorUrl), filename: path.basename(String(body.projectorFilename)) } : await resolveHuggingFaceProjector(directUrl, filename); startModelDownload(directUrl, filename, LLM_MODELS, "text"); return json(res, 200, { ok: true, message: "Text model download started", filename, projectorUrl: projector?.url || "", projectorFilename: projector?.filename || "", }); } catch (err) { return json(res, 400, { ok: false, error: err.message || String(err) }); } } if (req.url === "/api/download-openvino-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const modelId = String(body.model_id || body.modelId || ""); if (!modelId) return json(res, 400, { ok: false, error: "model_id is required" }); try { startOpenVinoModelDownload(modelId); return json(res, 200, { ok: true, message: "OpenVINO model download started" }); } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); } } // GET /api/download-progress if (req.url === "/api/download-progress" && req.method === "GET") { if (!downloadState.active && downloadState.kind === "backend" && downloadState.backendId) { const backend = IMAGE_BACKEND_DOWNLOADS[downloadState.backendId]; const installed = backend && fs.existsSync(path.join(backend.destDir, backend.exeName)) && fs.existsSync(path.join(backend.destDir, backend.requiredDll)); if (installed && !downloadState.error) { cachedBackendOptions = null; downloadState = { active: false, filename: "", progress: 0, speed: "0 MB/s", eta: 0, totalBytes: 0, downloadedBytes: 0, error: null, }; } } return json(res, 200, downloadState); } // POST /api/cancel-download if (req.url === "/api/cancel-download" && req.method === "POST") { const cancelled = cancelModelDownload(); return json(res, 200, { ok: true, cancelled }); } // GET /api/generation-progress if (req.url === "/api/generation-progress" && req.method === "GET") { return json(res, 200, { ...generationState, backendMode: generationState.backendMode || currentSettings.backendMode || "", backendDevice: generationState.backendDevice || currentSettings.backendDevice || "", }); } if (req.url === "/api/openvino-generate" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; try { return json(res, 200, await generateWithOpenVino(body)); } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); } } // GET /api/outputs if (req.url === "/api/outputs" && req.method === "GET") { return json(res, 200, { outputs: listGeneratedOutputs() }); } // GET /api/output-file?filename=... if (req.url.startsWith("/api/output-file") && req.method === "GET") { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); const filename = path.basename(parsed.searchParams.get("filename") || ""); const filePath = path.join(OUTPUTS, filename); if (!filename || !fs.existsSync(filePath)) { return json(res, 404, { error: "Output file not found" }); } const ext = path.extname(filePath).toLowerCase(); fs.readFile(filePath, (err, data) => { if (err) return json(res, 500, { error: err.message }); res.writeHead(200, { "Content-Type": MIME[ext] || "application/octet-stream", "Access-Control-Allow-Origin": "*" }); res.end(data); }); return; } if (req.url.startsWith("/tts-outputs/") && req.method === "GET") { const filename = path.basename(decodeURIComponent(req.url.replace(/^\/tts-outputs\//, "").split("?")[0] || "")); const filePath = path.join(TTS_OUTPUTS, filename); if (!filename || !pathInside(filePath, TTS_OUTPUTS) || !fs.existsSync(filePath)) { return json(res, 404, { ok: false, error: "TTS output not found" }); } const ext = path.extname(filePath).toLowerCase(); fs.readFile(filePath, (err, data) => { if (err) return json(res, 500, { ok: false, error: err.message }); res.writeHead(200, { "Content-Type": MIME[ext] || "application/octet-stream", "Access-Control-Allow-Origin": "*" }); res.end(data); }); return; } // POST /api/save-output if (req.url === "/api/save-output" && req.method === "POST") { try { const body = await readJsonBody(req, res); if (!body) return; const saved = saveGeneratedOutput(body.image, body.metadata || {}); return json(res, 200, { ok: true, output: { ...saved, url: `/api/output-file?filename=${encodeURIComponent(saved.image)}` } }); } catch (err) { console.error(" [api] Failed to save output:", err); return json(res, 500, { ok: false, error: err.message }); } } // POST /api/delete-outputs if (req.url === "/api/delete-outputs" && req.method === "POST") { try { const body = await readJsonBody(req, res); if (!body) return; const deleted = deleteGeneratedOutputs(body.outputs || []); return json(res, 200, { ok: true, deleted }); } catch (err) { console.error(" [api] Failed to delete outputs:", err); return json(res, 500, { ok: false, error: err.message }); } } // POST /api/import-model?filename=... if (req.url.startsWith("/api/import-model") && req.method === "POST") { try { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); const result = await streamModelUpload(req, parsed.searchParams.get("filename")); return json(res, 200, { ok: true, message: `Imported ${result.filename}`, model: result, filename: result.filename }); } catch (err) { console.error(" [api] Failed to import model:", err); return json(res, 500, { ok: false, error: err.message }); } } if (req.url.startsWith("/api/llm/import-model") && req.method === "POST") { try { const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); const result = await streamModelUpload(req, parsed.searchParams.get("filename"), LLM_MODELS, "text"); return json(res, 200, { ok: true, message: `Imported ${result.filename}`, model: result }); } catch (err) { console.error(" [api] Failed to import text model:", err); return json(res, 500, { ok: false, error: err.message }); } } if (req.url === "/api/llm/delete-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const filename = path.basename(String(body.filename || "")); const modelPath = path.join(LLM_MODELS, filename); if (!filename || !pathInside(modelPath, LLM_MODELS)) { return json(res, 400, { ok: false, error: "Invalid filename" }); } if (llmSettings.model === filename && llmProc) await killLlm(); try { fs.unlinkSync(modelPath); return json(res, 200, { ok: true }); } catch (err) { return json(res, err.code === "ENOENT" ? 404 : 500, { ok: false, error: err.code === "ENOENT" ? "Text model not found" : err.message, }); } } // POST /api/delete-model if (req.url === "/api/delete-model" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; const { filename } = body; if (!filename) return json(res, 400, { error: "Filename is required" }); const openvinoModel = findOpenVinoModel(filename); const safeFilename = path.basename(filename); const modelPath = path.join(MODELS, safeFilename); try { if (openvinoModel?.installed && openvinoModel.path && pathInside(openvinoModel.path, OPENVINO_MODELS)) { fs.rmSync(openvinoModel.path, { recursive: true, force: true }); cachedBackendOptions = null; console.log(` [api] Deleted OpenVINO model: ${openvinoModel.id}`); return json(res, 200, { ok: true, message: `Deleted ${openvinoModel.name}` }); } else if (fs.existsSync(modelPath)) { fs.unlinkSync(modelPath); console.log(` [api] Deleted model file: ${safeFilename}`); return json(res, 200, { ok: true, message: `Deleted ${safeFilename}` }); } else { return json(res, 404, { error: "Model file not found" }); } } catch (err) { console.error(` [api] Failed to delete model ${safeFilename}:`, err); return json(res, 500, { error: err.message }); } } if (req.url.startsWith("/api/")) { return json(res, 404, { ok: false, error: "Unknown API endpoint" }); } // ── Static frontend files ───────────────────────────────────────────────── let filePath = path.join(DIST, req.url === "/" ? "index.html" : req.url); filePath = filePath.split("?")[0]; if (!fs.existsSync(filePath) || fs.statSync(filePath).isDirectory()) { filePath = path.join(DIST, "index.html"); } const ext = path.extname(filePath).toLowerCase(); const mime = MIME[ext] || "application/octet-stream"; fs.readFile(filePath, (err, data) => { if (err) { res.writeHead(404); res.end("Not found"); return; } res.writeHead(200, { "Content-Type": mime, "Access-Control-Allow-Origin": "*" }); res.end(data); }); }); server.timeout = 0; // Disable socket timeout for large model uploads/downloads server.listen(PORT_FRONTEND, "0.0.0.0", () => { console.log(""); console.log(" ============================================================"); console.log(" UNCENSORED AI STUDIO | Running"); console.log(" Server Build: " + SERVER_BUILD); console.log(" Frontend : http://localhost:" + PORT_FRONTEND); console.log(" Image API: http://127.0.0.1:" + PORT_BACKEND); console.log(" Text API : starts on http://127.0.0.1:" + PREFERRED_LLM_PORT); console.log(" Speech : managed locally, API on frontend port"); console.log(" TTS : Kokoro ONNX managed locally, API on frontend port"); console.log(" ============================================================"); console.log(""); // Do not auto-start backend; wait for selection from the Web UI console.log(" [backend] Ready. Waiting for model load request from the webapp..."); }); // Graceful shutdown process.on("SIGINT", async () => { await killBackend(); await killOpenVinoWorker(); await killLlm(); await stopSpeech(); await stopTts(); process.exit(0); }); process.on("SIGTERM", async () => { await killBackend(); await killOpenVinoWorker(); await killLlm(); await stopSpeech(); await stopTts(); process.exit(0); });