From cf1b7e548a36c537d82243e0604240985b21328c Mon Sep 17 00:00:00 2001 From: techjarves Date: Mon, 15 Jun 2026 07:22:54 +0530 Subject: [PATCH] feat: initialize project with static file server, backend management, and React frontend components --- app/frontend/src/App.css | 109 +++++++ app/frontend/src/App.jsx | 62 +--- app/frontend/src/components/ModelManager.jsx | 185 +++++++++++- app/frontend/src/components/Settings.jsx | 64 +++- app/frontend/src/components/TextChat.jsx | 32 +- app/frontend/src/services/api.js | 80 +++++ scripts/serve.cjs | 291 ++++++++++++++++++- 7 files changed, 744 insertions(+), 79 deletions(-) diff --git a/app/frontend/src/App.css b/app/frontend/src/App.css index 319ddf8..6912785 100644 --- a/app/frontend/src/App.css +++ b/app/frontend/src/App.css @@ -1268,6 +1268,115 @@ html[data-theme="dark"] .m3-segment-item:hover:not(.active):not(.disabled) { transition: width 0.25s ease; } +.model-discovery-controls { + display: flex; + flex-direction: column; + gap: 12px; + margin-bottom: 16px; + padding: 16px; + border: 1px solid var(--border-color-muted); + border-radius: var(--md-shape-corner-large); + background: var(--md-sys-color-surface); +} + +.model-search-box { + display: flex; + align-items: center; + gap: 10px; + min-height: 44px; + padding: 0 12px; + border: 1px solid var(--border-color); + border-radius: var(--md-shape-corner-medium); + background: var(--md-sys-color-surface-variant); + color: var(--md-sys-color-outline); +} + +.model-search-box:focus-within { + border-color: var(--md-sys-color-primary); +} + +.model-search-box input { + flex: 1; + min-width: 0; + border: 0; + outline: 0; + background: transparent; + color: var(--md-sys-color-on-surface); + font: inherit; +} + +.model-search-box button { + display: inline-flex; + padding: 4px; + border: 0; + background: transparent; + color: inherit; + cursor: pointer; +} + +.model-filter-row { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px; +} + +.model-filter-chip { + display: inline-flex; + align-items: center; + gap: 6px; + min-height: 34px; + padding: 0 12px; + border: 1px solid var(--border-color); + border-radius: var(--md-shape-corner-full); + background: transparent; + color: var(--md-sys-color-on-surface-variant); + font-weight: 600; + cursor: pointer; +} + +.model-filter-chip.selected { + border-color: var(--md-sys-color-primary); + background: var(--md-sys-color-primary-container); + color: var(--md-sys-color-on-primary-container); +} + +.model-filter-clear { + border: 0; + background: transparent; + color: var(--md-sys-color-primary); + font-weight: 600; + cursor: pointer; +} + +.model-discovery-loading { + display: flex; + align-items: center; + gap: 12px; + color: var(--md-sys-color-outline); + font-size: 0.82rem; + font-weight: 600; +} + +.model-search-warning { + padding: 10px 12px; + border-radius: var(--md-shape-corner-small); + background: var(--md-sys-color-error-container); + color: var(--md-sys-color-on-error-container); + font-size: 0.8rem; +} + +.model-search-empty { + color: var(--md-sys-color-on-surface-variant); + text-align: center; +} + +.model-load-more { + align-self: center; + min-width: 180px; + margin: 4px 0 8px; +} + /* Importer & Library */ .import-box { border: 2px dashed var(--border-color); diff --git a/app/frontend/src/App.jsx b/app/frontend/src/App.jsx index 8c2fa2b..855d107 100644 --- a/app/frontend/src/App.jsx +++ b/app/frontend/src/App.jsx @@ -447,13 +447,6 @@ function App() { } }, [serverRunning, isStoppingServer, showAlert]); - const readinessIssues = [ - ...(health?.stale ? ["Restart Local AI Image Generator so the local server loads the latest API."] : []), - ...(health?.issues || []), - ]; - const cleanupBytes = cleanupItems.reduce((sum, item) => sum + Number(item.sizeBytes || 0), 0); - const showReadinessPanel = Boolean(health && (readinessIssues.length > 0 || cleanupItems.length > 0)); - return (
{/* Sidebar with Navigation Rail & Specs */} @@ -472,54 +465,6 @@ function App() { setTheme={setTheme} /> - {showReadinessPanel && ( -
0 ? "readiness-card-warning" : ""}`}> -
-
-

- {health?.stale ? "Restart Required" : readinessIssues.length > 0 ? "System Readiness" : "Safe Cleanup Available"} -

-

- {health?.stale - ? `Running server build: ${health.build || "unknown"}` - : readinessIssues.length > 0 - ? "Local AI Image Generator found setup items that may need attention." - : `${cleanupItems.length} temporary item${cleanupItems.length === 1 ? "" : "s"} can be cleaned (${formatBytes(cleanupBytes)}).`} -

-
-
- - - {cleanupItems.length > 0 && ( - - )} -
-
- {readinessIssues.length > 0 && ( -
- {readinessIssues.slice(0, 4).map((issue) => ( - {issue} - ))} -
- )} - {cleanupItems.length > 0 && ( -
- {cleanupItems.slice(0, 3).map((item) => ( - - {item.name} · {item.size} · {item.reason} - - ))} -
- )} -
- )} - {/* Dynamic Workspace Container */}
diff --git a/app/frontend/src/components/ModelManager.jsx b/app/frontend/src/components/ModelManager.jsx index e62d14e..0719bac 100644 --- a/app/frontend/src/components/ModelManager.jsx +++ b/app/frontend/src/components/ModelManager.jsx @@ -1,5 +1,5 @@ import React, { memo, useState, useEffect, useCallback } from "react"; -import { FolderOpen, DownloadCloud, RefreshCw, Database, Trash2, Square, HardDrive, Library, AlertTriangle } from "lucide-react"; +import { FolderOpen, DownloadCloud, RefreshCw, Database, Trash2, Square, HardDrive, Library, AlertTriangle, Search, X } from "lucide-react"; import { listLocalModels, startServer, @@ -20,9 +20,16 @@ import { downloadLlmModel, deleteLlmModel, importLlmModel, - getLlmStatus + getLlmStatus, + searchHuggingFaceModels } from "../services/api"; +const MODEL_FILTERS = [ + { id: "potato", label: "Potato PC" }, + { id: "vision", label: "Vision" }, + { id: "uncensored", label: "Uncensored" }, +]; + const MODEL_LIBRARY = [ { group: "SDXL - Best Quality", @@ -233,6 +240,16 @@ function ModelManager({ const [backendInfo, setBackendInfo] = useState({ backendMode: "", backendBinary: "", backendDevice: "" }); const [activeLlmModel, setActiveLlmModel] = useState(null); const [llmRunning, setLlmRunning] = useState(false); + const [modelSearch, setModelSearch] = useState(""); + const [selectedFilters, setSelectedFilters] = useState([]); + const [huggingFaceModels, setHuggingFaceModels] = useState([]); + const [isSearchingModels, setIsSearchingModels] = useState(false); + const [modelSearchError, setModelSearchError] = useState(""); + const [hasHuggingFaceResults, setHasHuggingFaceResults] = useState(false); + const [modelSearchPage, setModelSearchPage] = useState(1); + const [hasMoreModels, setHasMoreModels] = useState(false); + const [isLoadingMoreModels, setIsLoadingMoreModels] = useState(false); + const searchRequestRef = React.useRef(0); const cancelLoadRef = React.useRef(false); @@ -253,6 +270,9 @@ function ModelManager({ backendOptions?.options?.some((opt) => opt.id === "apple-npu") || backendOptions?.unavailable?.some((opt) => opt.id === "apple-npu") ); + const displayedHuggingFaceModels = huggingFaceModels.filter((model) => ( + selectedFilters.every((filterId) => model.tags?.includes(filterId)) + )); let visibleModelLibrary = []; if (activeModelType === "image") { @@ -264,8 +284,74 @@ function ModelManager({ visibleModelLibrary = [...visibleModelLibrary, ...COREML_MODEL_LIBRARY]; } } else { - visibleModelLibrary = [...TEXT_MODEL_LIBRARY]; + visibleModelLibrary = [{ + group: "Recommended Text Models from Hugging Face", + items: hasHuggingFaceResults ? displayedHuggingFaceModels : TEXT_MODEL_LIBRARY[0].items, + }]; } + + useEffect(() => { + if (activeModelType !== "text") return; + const requestId = ++searchRequestRef.current; + const timer = setTimeout(async () => { + setIsSearchingModels(true); + setModelSearchError(""); + try { + const result = await searchHuggingFaceModels(modelSearch, selectedFilters, 1); + if (requestId !== searchRequestRef.current) return; + const matchingModels = result.models.filter((model) => ( + selectedFilters.every((filterId) => model.tags?.includes(filterId)) + )); + setHuggingFaceModels(matchingModels); + setHasHuggingFaceResults(true); + setModelSearchPage(1); + setHasMoreModels(result.hasMore); + } catch (err) { + if (requestId !== searchRequestRef.current) return; + setHuggingFaceModels([]); + setHasHuggingFaceResults(false); + setHasMoreModels(false); + setModelSearchError(err.message || "Could not search Hugging Face."); + } finally { + if (requestId === searchRequestRef.current) setIsSearchingModels(false); + } + }, modelSearch ? 400 : 0); + return () => clearTimeout(timer); + }, [activeModelType, modelSearch, selectedFilters]); + + const toggleModelFilter = (filterId) => { + setSelectedFilters((current) => ( + current.includes(filterId) + ? current.filter((id) => id !== filterId) + : [...current, filterId] + )); + }; + + const handleLoadMoreModels = async () => { + if (isLoadingMoreModels || !hasMoreModels) return; + const nextPage = modelSearchPage + 1; + setIsLoadingMoreModels(true); + setModelSearchError(""); + try { + const [result] = await Promise.all([ + searchHuggingFaceModels(modelSearch, selectedFilters, nextPage), + new Promise((resolve) => setTimeout(resolve, 350)), + ]); + const matchingModels = result.models.filter((model) => ( + selectedFilters.every((filterId) => model.tags?.includes(filterId)) + )); + setHuggingFaceModels((current) => { + const known = new Set(current.map((model) => model.id)); + return [...current, ...matchingModels.filter((model) => !known.has(model.id))]; + }); + setModelSearchPage(result.page); + setHasMoreModels(result.hasMore); + } catch (err) { + setModelSearchError(err.message || "Could not load more Hugging Face models."); + } finally { + setIsLoadingMoreModels(false); + } + }; const getLocalModelInfo = (modelId) => localModels.map(normalizeModel).find((model) => model.filename === modelId); @@ -1060,12 +1146,70 @@ function ModelManager({

- Model Library + {activeModelType === "text" ? "Hugging Face Model Library" : "Model Library"}

+ {activeModelType === "text" && ( +
+
+ + setModelSearch(event.target.value)} + placeholder="Search Hugging Face GGUF models..." + aria-label="Search Hugging Face models" + /> + {modelSearch && ( + + )} +
+
+ {MODEL_FILTERS.map((filter) => { + const selected = selectedFilters.includes(filter.id); + return ( + + ); + })} + {selectedFilters.length > 0 && ( + + )} +
+ {isSearchingModels && ( +
+ + Loading models from Hugging Face... +
+ )} + {modelSearchError && ( +
+ {modelSearchError} Showing the built-in fallback catalog. +
+ )} +
+ )} +
- {visibleModelLibrary.map((section) => ( + {activeModelType === "text" && hasHuggingFaceResults && !isSearchingModels && displayedHuggingFaceModels.length === 0 && ( +
+ No downloadable single-file GGUF models matched this search and filter combination. +
+ )} + {visibleModelLibrary.filter((section) => section.items.length > 0).map((section) => (

{section.group}

@@ -1073,14 +1217,17 @@ function ModelManager({ const installed = modelNames.includes(model.filename); const downloading = downloadingModelId === model.filename || downloadingModelId === `${model.filename}.zip`; const systemTier = getHardwareTier(specs); - const isRecommended = model.recommendedTiers && model.recommendedTiers.includes(systemTier); + const isRecommended = typeof model.recommendedFit === "boolean" + ? model.recommendedFit + : model.recommendedTiers && model.recommendedTiers.includes(systemTier); + const fitLabel = model.fitLabel || "Recommended Fit"; return ( -
+
{model.name} {isRecommended && ( - - [Recommended Fit] + [{fitLabel}] )} {model.format} • approx. {model.approxSize} {model.resolution !== "N/A" && `• ${model.resolution}`} + {model.size && model.size !== "Unknown" && ( + + File size: {model.size} + + )}

{model.notes} @@ -1129,7 +1281,7 @@ function ModelManager({ {installed ? "Downloaded" : downloading ? "Downloading" : "Download"} - Save to Downloads folder + {model.pageUrl ? "View on Hugging Face" : "Save to Downloads folder"}

); @@ -1149,6 +1301,17 @@ function ModelManager({
))} + {activeModelType === "text" && hasHuggingFaceResults && hasMoreModels && !isSearchingModels && ( + + )}
{/* Local Drag and Drop Importer */} diff --git a/app/frontend/src/components/Settings.jsx b/app/frontend/src/components/Settings.jsx index e65cf03..e42e839 100644 --- a/app/frontend/src/components/Settings.jsx +++ b/app/frontend/src/components/Settings.jsx @@ -1,6 +1,6 @@ import React, { memo, useEffect } from "react"; import { Crop, Sliders, Cpu, Info, MessageSquare, SlidersHorizontal } from "lucide-react"; -import { stopServer } from "../services/api"; +import { stopServer, formatBytes } from "../services/api"; const ASPECT_RATIOS = [ { id: "1:1", label: "1:1 Square", width: 512, height: 512, sdxl_width: 1024, sdxl_height: 1024, desc: "Social posts & avatars" }, @@ -31,6 +31,13 @@ function Settings({ setTextSettings, showAlert = async ({ message }) => window.alert(message), showConfirm = async ({ message }) => window.confirm(message), + health, + cleanupItems, + isReadinessBusy, + refreshReadiness, + copyDiagnostics, + cleanupSafeItems, + diagnosticsCopied, }) { const isSD15OrCustom = activeModel ? isSD15OrCustomModel(activeModel) : false; const isOpenVinoNpu = constraints.backendType === "openvino-npu"; @@ -41,6 +48,13 @@ function Settings({ const isMac = availableBackends.some(b => b.id === "metal" || b.id === "apple-npu") || (specs?.os_name && (specs.os_name.toLowerCase().includes("darwin") || specs.os_name.toLowerCase().includes("mac"))); + const readinessIssues = [ + ...(health?.stale ? ["Restart Local AI Image Generator so the local server loads the latest API."] : []), + ...(health?.issues || []), + ]; + const cleanupBytes = (cleanupItems || []).reduce((sum, item) => sum + Number(item.sizeBytes || 0), 0); + const showReadinessPanel = Boolean(health && (readinessIssues.length > 0 || (cleanupItems && cleanupItems.length > 0))); + useEffect(() => { if (isOpenVinoNpu && constraints.steps > 8) { setConstraints((prev) => ({ ...prev, steps: 8, npuSteps: 8 })); @@ -153,6 +167,54 @@ function Settings({

+ {showReadinessPanel && ( +
0 ? "readiness-card-warning" : ""}`} style={{ marginBottom: "20px" }}> +
+
+

+ {health?.stale ? "Restart Required" : readinessIssues.length > 0 ? "System Readiness" : "Safe Cleanup Available"} +

+

+ {health?.stale + ? `Running server build: ${health.build || "unknown"}` + : readinessIssues.length > 0 + ? "Local AI Image Generator found setup items that may need attention." + : `${(cleanupItems || []).length} temporary item${(cleanupItems || []).length === 1 ? "" : "s"} can be cleaned (${formatBytes(cleanupBytes)}).`} +

+
+
+ + + {(cleanupItems || []).length > 0 && ( + + )} +
+
+ {readinessIssues.length > 0 && ( +
+ {readinessIssues.slice(0, 4).map((issue) => ( + {issue} + ))} +
+ )} + {(cleanupItems || []).length > 0 && ( +
+ {cleanupItems.slice(0, 3).map((item) => ( + + {item.name} · {item.size} · {item.reason} + + ))} +
+ )} +
+ )} +
{/* Left Column: Image Settings */}
diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx index 1b3a940..fa66173 100644 --- a/app/frontend/src/components/TextChat.jsx +++ b/app/frontend/src/components/TextChat.jsx @@ -1,10 +1,10 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import { Bot, LoaderCircle, Send, Trash2, Square, History, Paperclip, X } from "lucide-react"; import { - chatWithLlm, getDownloadProgress, getLlmStatus, listLlmModels, + streamChatWithLlm, startLlm, stopLlm, } from "../services/api"; @@ -279,22 +279,38 @@ function TextChat({ const firstTitle = isNew ? (displayTitleText.slice(0, 26) + (displayTitleText.length > 26 ? "..." : "")) : null; saveConversationState(convId, nextMessages, selectedModel, firstTitle); + setMessages([...nextMessages, { role: "assistant", content: "" }]); + try { const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant."; const requestMessages = [ ...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []), ...nextMessages, ]; - const response = await chatWithLlm(requestMessages, { - temperature: textSettings?.temperature || 0.7, - maxTokens: 768 + + let assistantText = ""; + + const response = await streamChatWithLlm(requestMessages, { + temperature: textSettings?.temperature || 0.7, + maxTokens: 768, + }, (_token, fullText) => { + assistantText = fullText; + setMessages((prev) => { + const updated = [...prev]; + if (updated.length > 0 && updated[updated.length - 1].role === "assistant") { + updated[updated.length - 1] = { + ...updated[updated.length - 1], + content: fullText, + }; + } + return updated; + }); }); - const finalMessages = [...nextMessages, { role: "assistant", content: response.content }]; + + const finalMessages = [...nextMessages, { role: "assistant", content: assistantText }]; setMessages(finalMessages); - if (response.usage) { - setTokenUsage(response.usage); - } saveConversationState(convId, finalMessages, selectedModel); + if (response.usage) setTokenUsage(response.usage); // Clean up attached files on success setAttachments([]); diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js index 240f995..f96a918 100644 --- a/app/frontend/src/services/api.js +++ b/app/frontend/src/services/api.js @@ -321,6 +321,22 @@ export async function listLlmModels() { return (data.models || []).map(normalizeModel); } +export async function searchHuggingFaceModels(query = "", filters = [], page = 1) { + const params = new URLSearchParams(); + if (query.trim()) params.set("query", query.trim()); + if (filters.length > 0) params.set("filters", filters.join(",")); + params.set("page", String(page)); + const res = await fetch(`/api/huggingface/models?${params.toString()}`); + const data = await readJsonResponse(res, "Hugging Face model search returned invalid data."); + return { + models: Array.isArray(data.models) ? data.models : [], + source: data.source || "huggingface", + warning: data.warning || "", + page: Number(data.page || page), + hasMore: Boolean(data.hasMore), + }; +} + export async function startLlm(model, options = {}) { const res = await fetch("/api/llm/start", { method: "POST", @@ -359,6 +375,70 @@ export async function chatWithLlm(messages, options = {}) { }; } +export async function streamChatWithLlm(messages, options = {}, onToken = () => {}) { + const res = await fetch("/api/llm/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages, + temperature: options.temperature, + max_tokens: options.maxTokens, + stream: true, + }), + }); + + if (!res.ok) { + const data = await readJsonResponse(res, `Chat request failed (HTTP ${res.status}).`); + throw new Error(data.error || `Chat request failed (HTTP ${res.status}).`); + } + if (!res.body) throw new Error("Streaming is not supported by this browser."); + + const reader = res.body.getReader(); + const decoder = new TextDecoder("utf-8"); + let buffer = ""; + let content = ""; + let usage = null; + + const consumeEvent = (eventText) => { + const data = eventText + .split(/\r?\n/) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trimStart()) + .join("\n") + .trim(); + if (!data || data === "[DONE]") return data === "[DONE]"; + + const parsed = JSON.parse(data); + const token = parsed.choices?.[0]?.delta?.content || ""; + if (token) { + content += token; + onToken(token, content); + } + if (parsed.usage) usage = parsed.usage; + return false; + }; + + let finished = false; + while (!finished) { + const { done, value } = await reader.read(); + buffer += decoder.decode(value || new Uint8Array(), { stream: !done }); + + const events = buffer.split(/\r?\n\r?\n/); + buffer = events.pop() || ""; + for (const eventText of events) { + if (consumeEvent(eventText)) { + finished = true; + break; + } + } + if (done) break; + } + + if (!finished && buffer.trim()) consumeEvent(buffer); + if (!content) throw new Error("The text model returned an empty streamed response."); + return { content, usage }; +} + export async function downloadLlmModel(url) { const res = await fetch("/api/llm/download-model", { method: "POST", diff --git a/scripts/serve.cjs b/scripts/serve.cjs index e3b849d..ea2aeca 100644 --- a/scripts/serve.cjs +++ b/scripts/serve.cjs @@ -11,6 +11,9 @@ const os = require("os"); const path = require("path"); const { spawn, spawnSync, execSync, exec } = require("child_process"); +const HF_MODEL_CACHE_TTL_MS = 5 * 60 * 1000; +const hfModelCache = new Map(); + function readPort(value, fallback) { const port = parseInt(value, 10); return Number.isInteger(port) && port > 0 && port < 65536 ? port : fallback; @@ -1551,6 +1554,203 @@ function requestJson(url, payload = null, timeoutMs = 120000) { }); } +function requestHttpsJson(url, timeoutMs = 30000) { + return new Promise((resolve, reject) => { + const req = https.get(url, { + headers: { + "Accept": "application/json", + "User-Agent": "Local-AI-Image-Generator/1.0", + }, + timeout: timeoutMs, + }, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => { + try { + const parsed = JSON.parse(body || "null"); + if (res.statusCode < 200 || res.statusCode >= 300) { + reject(new Error(parsed?.error || `Hugging Face returned HTTP ${res.statusCode}`)); + return; + } + resolve(parsed); + } catch (err) { + reject(new Error(`Hugging Face returned invalid JSON: ${err.message}`)); + } + }); + }); + req.on("error", reject); + req.on("timeout", () => req.destroy(new Error("Hugging Face request timed out"))); + }); +} + +function getModelParameterBillions(value) { + const text = String(value || ""); + const matches = [...text.matchAll(/(?:^|[-_/ ])(\d+(?:\.\d+)?)\s*b(?:[-_/ ]|$)/gi)]; + if (matches.length === 0) return null; + return Math.min(...matches.map((match) => Number(match[1])).filter(Number.isFinite)); +} + +function selectGgufFile(siblings = [], tier = "low") { + const files = siblings + .map((item) => typeof item === "string" ? item : item?.rfilename) + .filter((name) => name && /\.gguf$/i.test(name)) + .filter((name) => !/(?:^|\/)(?:mmproj|mtp)(?:[-_/]|$)|-\d{5}-of-\d{5}\.gguf$/i.test(name)); + const preferences = tier === "high" + ? [/q6_k\.gguf$/i, /q5_k_m\.gguf$/i, /q4_k_m\.gguf$/i, /q8_0\.gguf$/i] + : [/q4_k_m\.gguf$/i, /q4_k_s\.gguf$/i, /q4_0\.gguf$/i, /q3_k_m\.gguf$/i, /iq4_xs\.gguf$/i]; + for (const pattern of preferences) { + const match = files.find((name) => pattern.test(name)); + if (match) return match; + } + return files[0] || ""; +} + +function classifyHuggingFaceModel(model, filename) { + const searchable = `${model.id || ""} ${filename || ""} ${(model.tags || []).join(" ")}`.toLowerCase(); + const parameters = getModelParameterBillions(searchable); + return { + potato: parameters !== null && parameters <= 3, + vision: /(?:vision|llava|multimodal|(?:^|[-_/ ])vl(?:[-_/ ]|$)|moondream)/i.test(searchable), + uncensored: /uncensored|abliterated|heretic/i.test(searchable), + parameters, + }; +} + +function getTextModelFit(sizeBytes, specs = getHardwareSpecs()) { + if (!sizeBytes) { + return { recommended: false, mode: "unknown", label: "", reason: "File size is unavailable, so compatibility cannot be estimated." }; + } + + const ramBytes = Math.max(0, Number(specs.ram_total_gb) || 0) * (1024 ** 3); + const vramBytes = Math.max(0, Number(specs.gpu_vram_gb) || 0) * (1024 ** 3); + const isAppleSilicon = osPlatform === "darwin" && /apple/i.test(String(specs.cpu_name || "")); + const systemReserve = Math.max(4 * (1024 ** 3), ramBytes * 0.3); + const usableRam = Math.max(0, ramBytes - systemReserve); + const runtimeNeed = (sizeBytes * 1.25) + (1.5 * (1024 ** 3)); + const fastGpuFit = vramBytes >= runtimeNeed; + const cpuOrOffloadFit = usableRam >= runtimeNeed; + const unifiedMemoryFit = isAppleSilicon && usableRam >= runtimeNeed; + const recommended = fastGpuFit || cpuOrOffloadFit || unifiedMemoryFit; + + if (recommended) { + const mode = fastGpuFit ? "GPU memory" : isAppleSilicon ? "unified memory" : "system RAM with CPU/GPU offload"; + return { + recommended: true, + mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : "ram", + label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : "Fits in RAM", + reason: `${formatBytes(sizeBytes)} weights fit the estimated ${mode} budget with runtime headroom.`, + }; + } + + return { + recommended: false, + mode: "too-large", + label: "", + reason: `${formatBytes(sizeBytes)} weights need about ${formatBytes(runtimeNeed)} including runtime headroom; this computer has about ${formatBytes(usableRam)} usable RAM and ${formatBytes(vramBytes)} VRAM.`, + }; +} + +async function addHuggingFaceFileSize(model) { + try { + const detail = await requestHttpsJson(`https://huggingface.co/api/models/${model.id}?blobs=true`); + const file = (detail.siblings || []).find((item) => item?.rfilename === model.repositoryFilename); + const sizeBytes = Number(file?.size || file?.lfs?.size || 0); + const fit = getTextModelFit(sizeBytes); + return { + ...model, + sizeBytes, + size: sizeBytes > 0 ? formatBytes(sizeBytes) : "Unknown", + approxSize: sizeBytes > 0 ? formatBytes(sizeBytes) : model.approxSize, + recommendedFit: fit.recommended, + fitMode: fit.mode, + fitLabel: fit.label, + fitReason: fit.reason, + }; + } catch (_) { + return { + ...model, + sizeBytes: 0, + size: "Unknown", + recommendedFit: false, + fitMode: "unknown", + fitLabel: "", + fitReason: "File size is unavailable, so compatibility cannot be estimated.", + }; + } +} + +async function searchHuggingFaceModels(query, filters) { + const specs = getHardwareSpecs(); + const tier = specs.gpu_vram_gb >= 10 || specs.ram_total_gb >= 32 + ? "high" + : specs.gpu_vram_gb >= 5 || specs.ram_total_gb >= 16 ? "mid" : "low"; + const defaultSearch = tier === "low" ? "1B instruct" : tier === "mid" ? "7B instruct" : "8B instruct"; + const searchTerms = [ + query.trim() || defaultSearch, + filters.includes("vision") ? "vision" : "", + filters.includes("uncensored") ? "uncensored" : "", + ].filter(Boolean).join(" "); + const cacheKey = `${tier}|${searchTerms}|${filters.sort().join(",")}`; + const cached = hfModelCache.get(cacheKey); + if (cached && Date.now() - cached.createdAt < HF_MODEL_CACHE_TTL_MS) return cached.models; + + const params = new URLSearchParams({ + filter: "gguf", + search: searchTerms, + sort: "downloads", + direction: "-1", + limit: "50", + full: "true", + }); + const results = await requestHttpsJson(`https://huggingface.co/api/models?${params.toString()}`); + const models = []; + + for (const model of Array.isArray(results) ? results : []) { + if (model.private || model.gated || model.disabled) continue; + const filename = selectGgufFile(model.siblings, tier); + if (!filename) continue; + const traits = classifyHuggingFaceModel(model, filename); + if (filters.includes("potato") && !traits.potato) continue; + if (filters.includes("vision") && !traits.vision) continue; + if (filters.includes("uncensored") && !traits.uncensored) continue; + + const isLikelyChatModel = /instruct|chat|assistant|coder|uncensored|abliterated|vision|llava|\bvl\b/i.test(`${model.id} ${filename}`); + if (!query.trim() && !isLikelyChatModel) continue; + const normalizedQuery = query.trim().toLowerCase().replace(/[-_/]+/g, " ").replace(/\s+/g, " "); + const normalizedModelName = `${model.id} ${filename}`.toLowerCase().replace(/[-_/]+/g, " ").replace(/\s+/g, " "); + const queryWords = normalizedQuery.split(" ").filter(Boolean); + const exactPhraseMatch = normalizedQuery && normalizedModelName.includes(normalizedQuery); + const matchedQueryWords = queryWords.filter((word) => normalizedModelName.includes(word)).length; + const score = Number(model.downloads || 0) + + (traits.potato && tier === "low" ? 10000000 : 0) + + (traits.parameters && tier === "mid" && traits.parameters >= 4 && traits.parameters <= 9 ? 10000000 : 0) + + (traits.parameters && tier === "high" && traits.parameters >= 7 && traits.parameters <= 14 ? 10000000 : 0) + + (exactPhraseMatch ? 100000000 : 0) + + (matchedQueryWords * 5000000); + models.push({ + id: model.id, + name: String(model.id || "").split("/").pop().replace(/[-_]+/g, " "), + filename: path.basename(filename), + repositoryFilename: filename, + format: "GGUF", + approxSize: traits.parameters ? `~${traits.parameters}B parameters` : "Size shown before download", + resolution: "N/A", + notes: `Community GGUF from ${String(model.id || "").split("/")[0]}. ${Number(model.downloads || 0).toLocaleString()} Hugging Face downloads.`, + url: `https://huggingface.co/${model.id}/resolve/main/${filename.split("/").map(encodeURIComponent).join("/")}`, + pageUrl: `https://huggingface.co/${model.id}`, + downloads: Number(model.downloads || 0), + likes: Number(model.likes || 0), + tags: Object.entries(traits).filter(([key, value]) => key !== "parameters" && value).map(([key]) => key), + score, + }); + } + + const ranked = models.sort((a, b) => b.score - a.score); + hfModelCache.set(cacheKey, { createdAt: Date.now(), models: ranked }); + return ranked; +} + async function killOpenVinoWorker() { if (!openvinoProc) { openvinoReady = false; @@ -2932,6 +3132,32 @@ const server = http.createServer(async (req, res) => { return json(res, 200, { models: getLlmModels() }); } + if (req.url.startsWith("/api/huggingface/models") && req.method === "GET") { + const parsed = new URL(req.url, `http://${req.headers.host || "localhost"}`); + const query = String(parsed.searchParams.get("query") || "").slice(0, 120); + const filters = String(parsed.searchParams.get("filters") || "") + .split(",") + .map((value) => value.trim().toLowerCase()) + .filter((value) => ["potato", "vision", "uncensored"].includes(value)); + const page = Math.max(1, Math.min(20, Number(parsed.searchParams.get("page")) || 1)); + const pageSize = 9; + try { + const rankedModels = await searchHuggingFaceModels(query, [...new Set(filters)]); + const start = (page - 1) * pageSize; + const pageModels = rankedModels.slice(start, start + pageSize); + const models = await Promise.all(pageModels.map(addHuggingFaceFileSize)); + return json(res, 200, { + ok: true, + source: "huggingface", + models, + page, + hasMore: start + models.length < rankedModels.length, + }); + } catch (err) { + return json(res, 502, { ok: false, error: err.message || "Hugging Face search failed." }); + } + } + if (req.url === "/api/llm/start" && req.method === "POST") { const body = await readJsonBody(req, res); if (!body) return; @@ -2955,14 +3181,71 @@ const server = http.createServer(async (req, res) => { if (!body) return; if (!llmReady) return json(res, 409, { ok: false, error: "Load a text model before sending a message." }); try { - const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, { + const isStream = body.stream === true; + const requestData = JSON.stringify({ model: llmSettings.model || "local-model", messages: Array.isArray(body.messages) ? body.messages : [], temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : 0.7, max_tokens: Math.max(1, Math.min(4096, Number(body.max_tokens) || 512)), - stream: false, - }, 300000); - return json(res, 200, result); + stream: isStream, + }); + + if (isStream) { + const clientReq = http.request({ + hostname: "127.0.0.1", + port: PORT_LLM, + path: "/v1/chat/completions", + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(requestData), + } + }, (clientRes) => { + if (clientRes.statusCode < 200 || clientRes.statusCode >= 300) { + let errorBody = ""; + clientRes.setEncoding("utf8"); + clientRes.on("data", (chunk) => { errorBody += chunk; }); + clientRes.on("end", () => { + let message = `Text backend returned HTTP ${clientRes.statusCode}`; + try { + message = JSON.parse(errorBody || "{}").error?.message || message; + } catch (_) {} + json(res, clientRes.statusCode || 500, { ok: false, error: message }); + }); + return; + } + res.writeHead(200, { + "Content-Type": "text/event-stream; charset=utf-8", + "Cache-Control": "no-cache, no-transform", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }); + res.flushHeaders?.(); + clientRes.on("data", (chunk) => res.write(chunk)); + clientRes.on("end", () => res.end()); + clientRes.on("error", (err) => res.destroy(err)); + }); + + clientReq.on("error", (err) => { + console.error("LLM stream request error:", err); + if (!res.headersSent) { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: err.message })); + } else { + res.end(); + } + }); + + clientReq.write(requestData); + clientReq.end(); + res.on("close", () => { + if (!res.writableEnded && !clientReq.destroyed) clientReq.destroy(); + }); + return; + } else { + const result = await requestJson(`http://127.0.0.1:${PORT_LLM}/v1/chat/completions`, JSON.parse(requestData), 300000); + return json(res, 200, result); + } } catch (err) { return json(res, 500, { ok: false, error: err.message || String(err) }); }