feat: add static file server, backend process manager, and base frontend components for local AI image generation

This commit is contained in:
techjarves committed 2026-06-15 06:41:15 +05:30
1 parent f1e7c29a5d
commit 2e24396b35
3 files changed
+657 -72

No files matched your search

+110 -27
View File
@@ -35,6 +35,7 @@ const MODEL_LIBRARY = [
resolution: "1024x1024",
notes: "Photorealistic SDXL model tuned for 4-8 step Lightning generation.",
url: "https://huggingface.co/RunDiffusion/Juggernaut-XL-Lightning/resolve/main/Juggernaut_RunDiffusionPhoto2_Lightning_4Steps.safetensors",
recommendedTiers: ["Mid", "High"],
},
{
name: "DreamShaper XL Lightning",
@@ -44,6 +45,7 @@ const MODEL_LIBRARY = [
resolution: "1024x1024",
notes: "All-around SDXL model for fantasy art, renders, illustration, and stylized images.",
url: "https://huggingface.co/Lykon/dreamshaper-xl-lightning/resolve/main/DreamShaperXL_Lightning.safetensors",
recommendedTiers: ["Mid", "High"],
},
],
},
@@ -58,6 +60,7 @@ const MODEL_LIBRARY = [
resolution: "512x512",
notes: "General-purpose SD 1.5 model for illustration, anime, and semi-realistic portraits.",
url: "https://huggingface.co/Lykon/DreamShaper/resolve/main/DreamShaper_8_pruned.safetensors",
recommendedTiers: ["Low"],
},
{
name: "CyberRealistic V8",
@@ -67,6 +70,7 @@ const MODEL_LIBRARY = [
resolution: "512x512",
notes: "High-fidelity realism and photorealism in the SD 1.5 ecosystem.",
url: "https://huggingface.co/cyberdelia/CyberRealistic/resolve/main/CyberRealistic_V8_FP16.safetensors",
recommendedTiers: ["Low"],
},
{
name: "ReV Animated v1.2.2",
@@ -76,6 +80,7 @@ const MODEL_LIBRARY = [
resolution: "512x512",
notes: "Semi-realistic, 2.5D, anime, fantasy, and stylized digital art.",
url: "https://huggingface.co/danbrown/RevAnimated-v1-2-2/resolve/main/rev-animated-v1-2-2.safetensors",
recommendedTiers: ["Low"],
},
],
},
@@ -93,6 +98,7 @@ const TEXT_MODEL_LIBRARY = [
resolution: "N/A",
notes: "Extremely fast, lightweight assistant, perfect for low RAM/VRAM machines.",
url: "https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF/resolve/main/qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
recommendedTiers: ["Low"],
},
{
name: "SmolLM2 1.7B Instruct Q4_K_M",
@@ -102,6 +108,42 @@ const TEXT_MODEL_LIBRARY = [
resolution: "N/A",
notes: "Excellent lightweight assistant with strong logic, reasoning, and prompt expansion capabilities.",
url: "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/main/smollm2-1.7b-instruct-q4_k_m.gguf",
recommendedTiers: ["Low"],
},
{
name: "Qwen2.5-Coder-7B-Instruct",
filename: "qwen2.5-coder-7b-instruct-q4_k_m.gguf",
format: "GGUF",
size: "4.7 GB",
approxSize: "4.7 GB",
resolution: "N/A",
notes: "Highly intelligent coding and text assistant. Recommended for mid/high tier systems.",
url: "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/main/qwen2.5-coder-7b-instruct-q4_k_m.gguf",
recommendedTiers: ["Mid", "High"],
},
{
name: "Llama-3.1-8B-Instruct",
filename: "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf",
format: "GGUF",
size: "4.9 GB",
approxSize: "4.9 GB",
resolution: "N/A",
notes: "Excellent general-purpose text model. Recommended for high-tier systems.",
url: "https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf",
recommendedTiers: ["High"],
},
{
name: "LLaVA 1.5 7B",
filename: "ggml-model-q4_k.gguf",
format: "GGUF",
size: "4.5 GB",
approxSize: "4.5 GB",
resolution: "N/A",
notes: "Multimodal model capable of understanding images. Downloads companion vision projector automatically.",
url: "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q4_k.gguf",
projectorUrl: "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf",
projectorFilename: "mmproj-model-f16.gguf",
recommendedTiers: ["Mid", "High"],
},
],
},
@@ -143,6 +185,20 @@ const COREML_MODEL_LIBRARY = [
},
];
const getHardwareTier = (specs) => {
if (!specs) return "Low";
const ram = Number(specs.ram_total_gb) || 0;
const vram = Number(specs.gpu_vram_gb) || 0;
if (vram >= 12 || (vram >= 8 && ram >= 32)) {
return "High";
} else if (vram >= 6 || ram >= 16) {
return "Mid";
} else {
return "Low";
}
};
function ModelManager({
activeModel,
setActiveModel,
@@ -288,32 +344,6 @@ function ModelManager({
}
};
const startProgressPolling = (modelId) => {
setDownloadingModelId(modelId);
const interval = setInterval(async () => {
try {
const status = await getDownloadProgress();
if (status.active) {
setDownloadProgress(status.progress === -1 ? 0 : status.progress);
setDownloadSpeed(`${status.speed} • ETA ${status.eta}s`);
} else {
clearInterval(interval);
setDownloadingModelId(null);
if (status.error) {
if (!String(status.error).toLowerCase().includes("cancelled")) {
showAlert({ title: "Download Failed", message: status.error, danger: true });
}
} else {
fetchModels();
}
}
} catch (e) {
console.error("Polling error:", e);
}
}, 1000);
};
const downloadByUrl = async (url, expectedFilename) => {
if (downloadingModelId) return;
@@ -352,6 +382,40 @@ function ModelManager({
}
};
const startProgressPolling = (modelId) => {
setDownloadingModelId(modelId);
const interval = setInterval(async () => {
try {
const status = await getDownloadProgress();
if (status.active) {
setDownloadProgress(status.progress === -1 ? 0 : status.progress);
setDownloadSpeed(`${status.speed} • ETA ${status.eta}s`);
} else {
clearInterval(interval);
setDownloadingModelId(null);
if (status.error) {
if (!String(status.error).toLowerCase().includes("cancelled")) {
showAlert({ title: "Download Failed", message: status.error, danger: true });
}
} else {
fetchModels();
if (modelId === "ggml-model-q4_k.gguf") {
setTimeout(() => {
downloadByUrl(
"https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf",
"mmproj-model-f16.gguf"
);
}, 500);
}
}
}
} catch (e) {
console.error("Polling error:", e);
}
}, 1000);
};
const handleUrlDownloadSubmit = async (e) => {
e.preventDefault();
if (!downloadUrl.trim()) return;
@@ -809,6 +873,8 @@ function ModelManager({
</button>
</div>
{/* Active Model Status Tonal Box */}
{activeModelType === "image" && activeModel && (
<div className="m3-card" style={{ borderLeft: "4px solid var(--md-sys-color-primary)", background: "var(--md-sys-color-primary-container)", color: "var(--md-sys-color-on-primary-container)" }}>
@@ -1006,10 +1072,27 @@ function ModelManager({
{section.items.map((model) => {
const installed = modelNames.includes(model.filename);
const downloading = downloadingModelId === model.filename || downloadingModelId === `${model.filename}.zip`;
const systemTier = getHardwareTier(specs);
const isRecommended = model.recommendedTiers && model.recommendedTiers.includes(systemTier);
return (
<div key={model.filename} style={{ display: "flex", flexDirection: "column", gap: "10px", padding: "14px", background: "var(--md-sys-color-surface-variant)", border: "1px solid var(--md-sys-color-outline-variant)", borderRadius: "var(--md-shape-corner-medium)" }}>
<div style={{ display: "flex", flexDirection: "column", gap: "4px" }}>
<span style={{ fontWeight: 700 }}>{model.name}</span>
<span style={{ fontWeight: 700, display: "flex", alignItems: "center", gap: "6px", flexWrap: "wrap" }}>
{model.name}
{isRecommended && (
<span style={{
fontSize: "0.68rem",
color: "#2e7d32",
background: "#e8f5e9",
padding: "2px 6px",
borderRadius: "4px",
fontWeight: "bold",
border: "1px solid #c8e6c9"
}}>
[Recommended Fit]
</span>
)}
</span>
<span style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)" }}>
{model.format} • approx. {model.approxSize} {model.resolution !== "N/A" && `• ${model.resolution}`}
</span>
+217 -22
View File
@@ -1,5 +1,5 @@
import React, { useCallback, useEffect, useRef, useState } from "react";
import { Bot, LoaderCircle, Send, Trash2, Square, History, Plus } from "lucide-react";
import { Bot, LoaderCircle, Send, Trash2, Square, History, Plus, Paperclip, X } from "lucide-react";
import {
chatWithLlm,
getDownloadProgress,
@@ -30,6 +30,64 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
const completedDownloadRef = useRef("");
const loadingModelRef = useRef(null);
const [attachments, setAttachments] = useState([]);
const fileInputRef = useRef(null);
const isImage = (file) => {
return /\.(jpe?g|png|webp)$/i.test(file.name) || file.type.startsWith("image/");
};
const isTextFile = (file) => {
return /\.(txt|md|csv|js|jsx|ts|tsx|py|json|css|html|java|cpp|c|h|rs|go|sh|bat|xml|yaml|yml)$/i.test(file.name) || file.type.startsWith("text/");
};
const handleFileChange = (e) => {
const files = Array.from(e.target.files);
if (!files || files.length === 0) return;
files.forEach((file) => {
if (isImage(file)) {
const reader = new FileReader();
reader.onload = (event) => {
setAttachments((prev) => [
...prev,
{
id: Math.random().toString(36).substring(7),
file,
type: "image",
name: file.name,
dataUrl: event.target.result,
},
]);
};
reader.readAsDataURL(file);
} else if (isTextFile(file)) {
const reader = new FileReader();
reader.onload = (event) => {
setAttachments((prev) => [
...prev,
{
id: Math.random().toString(36).substring(7),
file,
type: "document",
name: file.name,
content: event.target.result,
},
]);
};
reader.readAsText(file);
} else {
showAlert({
title: "Unsupported File",
message: `File "${file.name}" is not supported. Please select an image (JPEG, PNG, WEBP) or a text/code file.`,
danger: true,
});
}
});
e.target.value = "";
};
useEffect(() => {
loadingModelRef.current = loadingModel;
}, [loadingModel]);
@@ -179,7 +237,12 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
setSelectedModel(conv.model);
}
// Try to compute approximate token usage from messages
const total = conv.messages.reduce((sum, m) => sum + m.content.split(/\s+/).length, 0);
const total = conv.messages.reduce((sum, m) => {
const text = Array.isArray(m.content)
? m.content.map(c => c.text || "").join(" ")
: (m.content || "");
return sum + text.split(/\s+/).length;
}, 0);
setTokenUsage({
prompt_tokens: Math.round(total * 0.7),
completion_tokens: Math.round(total * 0.3),
@@ -204,7 +267,8 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
const sendMessage = async () => {
const text = input.trim();
if (!text || isBusy || !status.ready) return;
const hasAttachments = attachments.length > 0;
if ((!text && !hasAttachments) || isBusy || !status.ready) return;
let convId = activeConversationId;
let isNew = false;
@@ -214,12 +278,40 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
isNew = true;
}
const nextMessages = [...messages, { role: "user", content: text }];
// Delimited context blocks for document attachments
let documentContext = "";
attachments.filter(att => att.type === "document").forEach((att) => {
documentContext += `\n[Attached File: ${att.name}]\n${att.content}\n`;
});
const combinedText = documentContext ? `${text}\n\n${documentContext}`.trim() : text;
const imageAttachments = attachments.filter(att => att.type === "image");
let userMessageContent;
if (imageAttachments.length > 0) {
userMessageContent = [
{
type: "text",
text: combinedText
},
...imageAttachments.map((img) => ({
type: "image_url",
image_url: {
url: img.dataUrl
}
}))
];
} else {
userMessageContent = combinedText;
}
const nextMessages = [...messages, { role: "user", content: userMessageContent }];
setMessages(nextMessages);
setInput("");
setIsBusy(true);
const firstTitle = isNew ? (text.slice(0, 26) + (text.length > 26 ? "..." : "")) : null;
const displayTitleText = text || (imageAttachments.length > 0 ? "Sent Image" : "Sent File");
const firstTitle = isNew ? (displayTitleText.slice(0, 26) + (displayTitleText.length > 26 ? "..." : "")) : null;
saveConversationState(convId, nextMessages, selectedModel, firstTitle);
try {
@@ -238,6 +330,9 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
setTokenUsage(response.usage);
}
saveConversationState(convId, finalMessages, selectedModel);
// Clean up attached files on success
setAttachments([]);
} catch (err) {
const finalMessages = [...nextMessages, { role: "assistant", content: `Error: ${err.message}`, error: true }];
setMessages(finalMessages);
@@ -510,7 +605,30 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
{messages.map((message, index) => (
<div key={`${message.role}-${index}`} className={`chat-message ${message.role} ${message.error ? "error" : ""}`}>
<strong>{message.role === "user" ? "You" : "Local AI"}</strong>
<div>{message.content}</div>
<div>
{Array.isArray(message.content) ? (
<div style={{ display: "flex", flexDirection: "column", gap: "8px" }}>
{message.content.map((item, idx) => {
if (item.type === "text") {
return <span key={idx} style={{ whiteSpace: "pre-wrap" }}>{item.text}</span>;
}
if (item.type === "image_url") {
return (
<img
key={idx}
src={item.image_url.url}
alt="Attached image"
style={{ maxWidth: "240px", maxHeight: "180px", objectFit: "contain", borderRadius: "6px", marginTop: "4px" }}
/>
);
}
return null;
})}
</div>
) : (
<span style={{ whiteSpace: "pre-wrap" }}>{message.content}</span>
)}
</div>
</div>
))}
{isBusy && status.ready && <div className="chat-thinking"><LoaderCircle className="progress-spinner" size={16} /> Generating...</div>}
@@ -519,22 +637,99 @@ function TextChat({ specs, showAlert, showConfirm, textSettings, setTextSettings
<div ref={bottomRef} />
</div>
<div className="chat-composer">
<textarea
value={input}
onChange={(event) => setInput(event.target.value)}
onKeyDown={(event) => {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
sendMessage();
}
}}
placeholder={status.ready ? "Message your local model..." : "Select and load a GGUF model above to begin"}
disabled={!status.ready || isBusy}
/>
<button className="m3-btn m3-btn-filled" onClick={sendMessage} disabled={!input.trim() || !status.ready || isBusy}>
<Send size={17} /> Send
</button>
<div className="chat-composer" style={{ display: "flex", flexDirection: "column", gap: "10px" }}>
{attachments.length > 0 && (
<div style={{ display: "flex", flexWrap: "wrap", gap: "8px", paddingBottom: "8px" }}>
{attachments.map((att) => (
<div key={att.id} style={{
position: "relative",
display: "flex",
alignItems: "center",
gap: "6px",
padding: "6px 28px 6px 8px",
background: "var(--md-sys-color-surface-variant)",
border: "1px solid var(--border-color)",
borderRadius: "6px",
fontSize: "0.8rem",
maxWidth: "200px"
}}>
{att.type === "image" ? (
<img src={att.dataUrl} alt={att.name} style={{ width: "24px", height: "24px", objectFit: "cover", borderRadius: "3px" }} />
) : (
<span style={{ fontWeight: 600 }}>📄</span>
)}
<span style={{
whiteSpace: "nowrap",
overflow: "hidden",
textOverflow: "ellipsis",
color: "var(--md-sys-color-on-surface-variant)"
}} title={att.name}>
{att.name}
</span>
<button
onClick={() => setAttachments(prev => prev.filter(item => item.id !== att.id))}
style={{
position: "absolute",
right: "4px",
top: "50%",
transform: "translateY(-50%)",
background: "none",
border: "none",
color: "var(--md-sys-color-error)",
cursor: "pointer",
padding: "2px",
display: "flex",
alignItems: "center",
justifyContent: "center"
}}
>
<X size={14} />
</button>
</div>
))}
</div>
)}
<div style={{ display: "flex", gap: "10px", width: "100%" }}>
<input
type="file"
ref={fileInputRef}
style={{ display: "none" }}
multiple
onChange={handleFileChange}
/>
<button
className="m3-btn m3-btn-tonal"
onClick={() => fileInputRef.current?.click()}
style={{ padding: "0 12px", height: "48px", display: "flex", alignItems: "center", justifyContent: "center" }}
disabled={!status.ready || isBusy}
title="Attach files or images"
>
<Paperclip size={17} />
</button>
<textarea
value={input}
onChange={(event) => setInput(event.target.value)}
onKeyDown={(event) => {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
sendMessage();
}
}}
placeholder={status.ready ? "Message your local model..." : "Select and load a GGUF model above to begin"}
disabled={!status.ready || isBusy}
style={{ flex: 1 }}
/>
<button
className="m3-btn m3-btn-filled"
onClick={sendMessage}
disabled={(!input.trim() && attachments.length === 0) || !status.ready || isBusy}
style={{ height: "48px" }}
>
<Send size={17} /> Send
</button>
</div>
</div>
</section>
</div>
+330 -23
View File
@@ -220,42 +220,217 @@ function getGpuInfo() {
if (cachedGpuInfo) return cachedGpuInfo;
if (osPlatform === "win32") {
try {
const stdout = execSync(
'powershell -NoProfile -Command "Get-CimInstance Win32_VideoController | ForEach-Object { [PSCustomObject]@{ Name = $_.Name; VRAM = [math]::Round($_.AdapterRAM / 1GB, 2) } } | ConvertTo-Json"',
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }
).trim();
if (stdout) {
let gpus = [];
try {
const parsed = JSON.parse(stdout);
if (Array.isArray(parsed)) {
gpus = parsed;
} else if (parsed && typeof parsed === "object") {
gpus = [parsed];
}
} catch (_) {}
if (gpus.length > 0) {
const discreteKeywords = ["nvidia", "amd", "arc", "geforce", "radeon", "rtx", "gtx"];
let selectedGpu = gpus.find(gpu => {
const name = String(gpu.Name || "").toLowerCase();
return discreteKeywords.some(kw => name.includes(kw));
});
if (!selectedGpu) {
selectedGpu = gpus[0];
}
cachedGpuInfo = {
name: selectedGpu.Name || "Unknown GPU",
vram_gb: typeof selectedGpu.VRAM === "number" ? selectedGpu.VRAM : 0
};
return cachedGpuInfo;
}
}
} catch (_) {}
// Fallback if Powershell fails or returns empty
try {
const output = execSync(
"powershell -NoProfile -Command \"Get-CimInstance Win32_VideoController | Select-Object -First 1 -ExpandProperty Name\"",
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] },
).trim();
if (output) {
cachedGpuInfo = { name: output };
cachedGpuInfo = { name: output, vram_gb: 0 };
return cachedGpuInfo;
}
} catch (_) {}
}
if (osPlatform === "linux") {
const linuxGpu = detectLinuxGpuFromSysfs();
if (linuxGpu) {
cachedGpuInfo = linuxGpu;
return cachedGpuInfo;
}
}
if (osPlatform === "darwin") {
try {
const output = execSync(
"system_profiler SPDisplaysDataType | awk -F': ' '/Chipset Model|Vendor/ {print $2; exit}'",
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] },
).trim();
if (output) {
cachedGpuInfo = { name: output };
return cachedGpuInfo;
const output = execSync("system_profiler SPDisplaysDataType", {
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
});
const lines = output.split("\n");
let currentGpu = null;
const gpus = [];
for (const line of lines) {
const trimmed = line.trim();
if (trimmed.startsWith("Chipset Model:")) {
if (currentGpu) {
gpus.push(currentGpu);
}
currentGpu = {
name: trimmed.substring("Chipset Model:".length).trim(),
vram_gb: 0,
};
} else if (currentGpu && (trimmed.startsWith("VRAM (Total):") || trimmed.startsWith("VRAM (Dynamic, Max):"))) {
const valStr = trimmed.split(":")[1].trim(); // e.g. "2 GB" or "1536 MB"
let bytesVal = 0;
const numMatch = valStr.match(/^([\d\.]+)\s*(GB|MB)/i);
if (numMatch) {
const val = parseFloat(numMatch[1]);
const unit = numMatch[2].toUpperCase();
if (unit === "GB") {
bytesVal = val;
} else if (unit === "MB") {
bytesVal = val / 1024;
}
}
currentGpu.vram_gb = Math.round(bytesVal * 100) / 100;
}
}
if (currentGpu) {
gpus.push(currentGpu);
}
if (gpus.length > 0) {
const isAppleSilicon = gpus.some(g => {
const name = g.name.toLowerCase();
return name.includes("apple") || name.includes("m1") || name.includes("m2") || name.includes("m3") || name.includes("m4");
}) || (os.arch() === "arm64");
if (isAppleSilicon) {
const totalRamGb = Math.round((os.totalmem() / (1024 ** 3)) * 100) / 100;
let selected = gpus.find(g => g.name.toLowerCase().includes("apple")) || gpus[0];
cachedGpuInfo = {
name: selected.name,
vram_gb: totalRamGb
};
return cachedGpuInfo;
} else {
const discreteKeywords = ["nvidia", "amd", "radeon", "geforce"];
let selected = gpus.find(gpu => {
const name = gpu.name.toLowerCase();
return discreteKeywords.some(kw => name.includes(kw));
});
if (!selected) {
selected = gpus[0];
}
cachedGpuInfo = {
name: selected.name,
vram_gb: selected.vram_gb
};
return cachedGpuInfo;
}
}
} catch (_) {}
cachedGpuInfo = { name: "Apple Metal GPU" };
// Fallback if system_profiler command fails
const totalRamGb = Math.round((os.totalmem() / (1024 ** 3)) * 100) / 100;
cachedGpuInfo = { name: "Apple Metal GPU", vram_gb: totalRamGb };
return cachedGpuInfo;
}
cachedGpuInfo = { name: "Unavailable" };
if (osPlatform === "linux") {
// AMD cards: Scan /sys/class/drm/card*/device/vendor and mem_info_vram_total
try {
if (fs.existsSync("/sys/class/drm")) {
const cards = fs.readdirSync("/sys/class/drm").filter(name => name.startsWith("card"));
for (const card of cards) {
const vendorPath = `/sys/class/drm/${card}/device/vendor`;
const vramPath = `/sys/class/drm/${card}/device/mem_info_vram_total`;
if (fs.existsSync(vendorPath) && fs.existsSync(vramPath)) {
const vendor = fs.readFileSync(vendorPath, "utf8").trim();
if (vendor === "0x1002") {
const vramBytesStr = fs.readFileSync(vramPath, "utf8").trim();
const vramBytes = parseFloat(vramBytesStr);
if (!isNaN(vramBytes)) {
cachedGpuInfo = {
name: "AMD Radeon GPU",
vram_gb: Math.round((vramBytes / (1024 ** 3)) * 100) / 100
};
return cachedGpuInfo;
}
}
}
}
}
} catch (_) {}
// NVIDIA cards: read /proc/driver/nvidia/gpus/*/information or run nvidia-smi
try {
if (fs.existsSync("/proc/driver/nvidia/gpus")) {
const gpuDirs = fs.readdirSync("/proc/driver/nvidia/gpus");
for (const gpuDir of gpuDirs) {
const infoPath = `/proc/driver/nvidia/gpus/${gpuDir}/information`;
if (fs.existsSync(infoPath)) {
const info = fs.readFileSync(infoPath, "utf8");
const modelMatch = info.match(/Model:\s+(.*)/);
if (modelMatch) {
const modelName = modelMatch[1].trim();
let vram_gb = 0;
try {
const smiOut = execSync("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits", {
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"]
}).trim();
const parsedVal = parseFloat(smiOut);
if (!isNaN(parsedVal)) {
vram_gb = Math.round((parsedVal / 1024) * 100) / 100;
}
} catch (_) {}
cachedGpuInfo = {
name: modelName,
vram_gb: vram_gb || 8.0
};
return cachedGpuInfo;
}
}
}
}
} catch (_) {}
try {
const smiOut = execSync("nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits", {
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"]
}).trim();
if (smiOut) {
const lines = smiOut.split("\n");
if (lines[0]) {
const [name, totalMb] = lines[0].split(",").map(p => p.trim());
const parsedVal = parseFloat(totalMb);
cachedGpuInfo = {
name: name || "NVIDIA GPU",
vram_gb: !isNaN(parsedVal) ? Math.round((parsedVal / 1024) * 100) / 100 : 8.0
};
return cachedGpuInfo;
}
}
} catch (_) {}
const linuxGpu = detectLinuxGpuFromSysfs();
if (linuxGpu) {
cachedGpuInfo = {
name: linuxGpu.name,
vram_gb: 0
};
return cachedGpuInfo;
}
}
cachedGpuInfo = { name: "Unavailable", vram_gb: 0 };
return cachedGpuInfo;
}
@@ -365,13 +540,109 @@ if (osPlatform === "darwin") {
function getHardwareSpecs() {
const cpus = os.cpus();
const gpu = getGpuInfo();
const ramTotalGb = roundGb(os.totalmem());
const gpuVramGb = gpu.vram_gb || 0;
// Tiering logic
let tier = "low";
let tierReason = "";
const isAppleSilicon = osPlatform === "darwin" && (os.arch() === "arm64" || String(cpus[0]?.model || "").toLowerCase().includes("apple"));
if (isAppleSilicon) {
if (ramTotalGb >= 16) {
tier = "high";
tierReason = `Apple Silicon Mac with ${ramTotalGb} GB unified memory.`;
} else if (ramTotalGb >= 8) {
tier = "mid";
tierReason = `Apple Silicon Mac with ${ramTotalGb} GB unified memory.`;
} else {
tier = "low";
tierReason = `Apple Silicon Mac with ${ramTotalGb} GB unified memory.`;
}
} else {
if (gpuVramGb >= 12) {
tier = "high";
tierReason = `Discrete GPU with ${gpuVramGb} GB VRAM.`;
} else if (gpuVramGb >= 6) {
tier = "mid";
tierReason = `GPU with ${gpuVramGb} GB VRAM.`;
} else {
tier = "low";
tierReason = `GPU with ${gpuVramGb} GB VRAM and ${ramTotalGb} GB RAM.`;
}
}
// Recommended models catalog matching contract exactly
let recommended_models = [];
if (tier === "high") {
recommended_models = [
{
name: "Llama 3 8B Instruct (Q8_0)",
filename: "Meta-Llama-3-8B-Instruct-Q8_0.gguf",
approxSize: "8.5 GB",
url: "https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q8_0.gguf",
notes: "Near-lossless precision for high quality responses.",
reason: tierReason
},
{
name: "Mistral 7B Instruct v0.3 (Q8_0)",
filename: "Mistral-7B-Instruct-v0.3.Q8_0.gguf",
approxSize: "7.7 GB",
url: "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3.Q8_0.gguf",
notes: "Superb instruction-following and context window size.",
reason: tierReason
}
];
} else if (tier === "mid") {
recommended_models = [
{
name: "Llama 3 8B Instruct (Q4_K_M)",
filename: "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
approxSize: "4.8 GB",
url: "https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
notes: "Great balance of speed and performance.",
reason: tierReason
},
{
name: "Phi-3 Mini Instruct (Q4_K_M)",
filename: "Phi-3-mini-4k-instruct-q4.gguf",
approxSize: "2.2 GB",
url: "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf",
notes: "Lightweight and fast, optimized by Microsoft.",
reason: tierReason
}
];
} else {
recommended_models = [
{
name: "Qwen 2.5 Coder 0.5B Instruct (Q4_K_M)",
filename: "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
approxSize: "491 MB",
url: "https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF/resolve/main/qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
notes: "Extremely fast, lightweight assistant, perfect for low RAM/VRAM machines.",
reason: tierReason
},
{
name: "SmolLM2 1.7B Instruct (Q4_K_M)",
filename: "smollm2-1.7b-instruct-q4_k_m.gguf",
approxSize: "1.1 GB",
url: "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/main/smollm2-1.7b-instruct-q4_k_m.gguf",
notes: "Excellent lightweight assistant with strong logic, reasoning, and prompt expansion capabilities.",
reason: tierReason
}
];
}
return {
os_name: `${os.type()} ${os.release()}`,
cpu_name: cpus[0]?.model || "Unavailable",
cpu_cores_physical: Math.max(1, Math.round(cpus.length / 2)),
cpu_cores_logical: cpus.length || 1,
ram_total_gb: roundGb(os.totalmem()),
ram_total_gb: ramTotalGb,
gpu_name: gpu.name,
gpu_vram_gb: gpuVramGb,
recommended_models
};
}
@@ -381,13 +652,14 @@ function getTelemetry() {
if (osPlatform === "darwin" && cachedMacRamUsedGb !== null) {
ram_used_gb = cachedMacRamUsedGb;
}
const gpu = getGpuInfo();
return {
cpu_usage: getCpuUsagePercent(),
ram_used_gb,
ram_total_gb: roundGb(os.totalmem()),
gpu_name: vram?.gpu_name || getGpuInfo().name,
gpu_name: vram?.gpu_name || gpu.name,
vram_used_gb: vram?.vram_used_gb || 0,
vram_total_gb: vram?.vram_total_gb || 0,
vram_total_gb: vram?.vram_total_gb || gpu.vram_gb || 0,
};
}
@@ -1518,6 +1790,34 @@ async function startLlm(settings = {}) {
backendBinary: path.basename(backend.path),
};
let mmprojPath = null;
const lowerFilename = filename.toLowerCase();
const isMultimodal = lowerFilename.includes("llava") ||
lowerFilename.includes("vision") ||
lowerFilename.includes("qwen2vl") ||
lowerFilename === "ggml-model-q4_k.gguf";
if (isMultimodal) {
if (settings.mmproj) {
const customProjPath = path.join(LLM_MODELS, path.basename(settings.mmproj));
if (fs.existsSync(customProjPath)) {
mmprojPath = customProjPath;
}
}
if (!mmprojPath) {
try {
const files = fs.readdirSync(LLM_MODELS);
const mmprojFile = files.find(file => {
const lower = file.toLowerCase();
return lower.endsWith(".gguf") && (lower === "mmproj-model-f16.gguf" || lower.includes("mmproj"));
});
if (mmprojFile) {
mmprojPath = path.join(LLM_MODELS, mmprojFile);
}
} catch (_) {}
}
}
const args = [
"--model", modelPath,
"--host", "127.0.0.1",
@@ -1526,6 +1826,9 @@ async function startLlm(settings = {}) {
"--threads", String(llmSettings.threads),
"--n-gpu-layers", String(llmSettings.gpuLayers),
];
if (mmprojPath) {
args.push("--mmproj", mmprojPath);
}
const spawnEnv = { ...process.env };
const backendDir = path.dirname(backend.path);
if (osPlatform === "linux") {
@@ -1959,7 +2262,7 @@ function describeDownloadHttpError(statusCode, url, headers = {}) {
return `HTTP ${statusCode}`;
}
function startModelDownload(url, overrideFilename = null, targetDir = MODELS, kind = "image") {
function startModelDownload(url, overrideFilename = null, targetDir = MODELS, kind = "image", redirectCount = 0) {
if (downloadState.active && !overrideFilename) {
console.log(" [download] Already downloading a model");
return;
@@ -2035,6 +2338,10 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki
failDownload("Redirect response did not include a Location header");
return;
}
if (redirectCount > 10) {
failDownload("Too many redirects");
return;
}
console.log(` [download] Redirected to ${redirectUrl}`);
// Clean up redirected request to avoid triggering error handlers later
@@ -2045,7 +2352,7 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki
try { fs.unlinkSync(tempPath); } catch (_) {}
activeDownload = null;
downloadState.active = false;
startModelDownload(redirectUrl, filename, targetDir, kind);
startModelDownload(redirectUrl, filename, targetDir, kind, redirectCount + 1);
return;
}