diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx
index 9b08da8..01b9a63 100644
--- a/app/frontend/src/components/TextChat.jsx
+++ b/app/frontend/src/components/TextChat.jsx
@@ -150,6 +150,17 @@ function TextChat({
const wordCount = value.split(/\s+/).filter(Boolean).length;
return Math.max(wordCount, Math.ceil(value.length / 4));
};
+
+ const clamp = (value, min, max) => Math.max(min, Math.min(max, value));
+
+ const getAutoMaxResponseTokens = (promptTokens, thinkingEnabled) => {
+ const contextLimit = Number(status.settings?.contextSize || textSettings?.contextSize || 4096);
+ const safetyBuffer = thinkingEnabled ? 768 : 512;
+ const preferredMinimum = thinkingEnabled ? 512 : 256;
+ const available = contextLimit - promptTokens - safetyBuffer;
+ if (available <= 0) return 64;
+ return clamp(available, Math.min(preferredMinimum, available), 4096);
+ };
const bottomRef = useRef(null);
const textareaRef = useRef(null);
@@ -192,14 +203,45 @@ function TextChat({
return /\.(txt|md|csv|js|jsx|ts|tsx|py|json|css|html|java|cpp|c|h|rs|go|sh|bat|xml|yaml|yml)$/i.test(file.name) || file.type.startsWith("text/");
};
+ const optimizeImageForVision = (file, maxSide = 1024, quality = 0.92) => new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onerror = () => reject(new Error(`Could not read ${file.name}`));
+ reader.onload = () => {
+ const originalDataUrl = reader.result;
+ const img = new Image();
+ img.onerror = () => reject(new Error(`Could not decode ${file.name}`));
+ img.onload = () => {
+ const scale = Math.min(1, maxSide / Math.max(img.width, img.height));
+ const width = Math.max(1, Math.round(img.width * scale));
+ const height = Math.max(1, Math.round(img.height * scale));
+ const canvas = document.createElement("canvas");
+ canvas.width = width;
+ canvas.height = height;
+ const ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0, width, height);
+ const sendDataUrl = canvas.toDataURL("image/jpeg", quality);
+ resolve({
+ previewDataUrl: originalDataUrl,
+ sendDataUrl,
+ originalWidth: img.width,
+ originalHeight: img.height,
+ width,
+ height,
+ });
+ };
+ img.src = originalDataUrl;
+ };
+ reader.readAsDataURL(file);
+ });
+
const handleFileChange = (e) => {
const files = Array.from(e.target.files);
if (!files || files.length === 0) return;
files.forEach((file) => {
if (isImage(file)) {
- const reader = new FileReader();
- reader.onload = (event) => {
+ optimizeImageForVision(file)
+ .then((image) => {
setAttachments((prev) => [
...prev,
{
@@ -207,11 +249,18 @@ function TextChat({
file,
type: "image",
name: file.name,
- dataUrl: event.target.result,
+ dataUrl: image.previewDataUrl,
+ sendDataUrl: image.sendDataUrl,
+ width: image.width,
+ height: image.height,
+ originalWidth: image.originalWidth,
+ originalHeight: image.originalHeight,
},
]);
- };
- reader.readAsDataURL(file);
+ })
+ .catch((err) => {
+ showAlert({ title: "Image Error", message: err.message || String(err), danger: true });
+ });
} else if (isTextFile(file)) {
const reader = new FileReader();
reader.onload = (event) => {
@@ -279,7 +328,7 @@ function TextChat({
const buildTextStartOptions = (settings) => ({
threads: settings?.threads || specs?.cpu_cores_physical || 4,
- contextSize: settings?.contextSize || 4096,
+ contextSize: settings?.contextSize ?? 0,
gpuLayers: settings?.gpuLayers ?? -1,
enableThinking: settings?.enableThinking !== false,
flashAttn: settings?.flashAttn,
@@ -360,13 +409,14 @@ function TextChat({
const refresh = useCallback(async () => {
const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]);
- setModels(nextModels);
+ const selectableModels = nextModels.filter((model) => !model.isProjector);
+ setModels(selectableModels);
setStatus(nextStatus);
const active = nextStatus.settings?.model;
setSelectedModel((current) => {
const saved = localStorage.getItem("selectedLlmModel");
- const savedExists = nextModels.some((m) => m.filename === saved);
- return active || current || (savedExists ? saved : "") || nextModels[0]?.filename || "";
+ const savedExists = selectableModels.some((m) => m.filename === saved);
+ return active || current || (savedExists ? saved : "") || selectableModels[0]?.filename || "";
});
}, []);
@@ -453,7 +503,7 @@ function TextChat({
const result = await startLlm(filename, {
threads: textSettings?.threads || specs?.cpu_cores_physical || 4,
- contextSize: textSettings?.contextSize || 4096,
+ contextSize: textSettings?.contextSize ?? 0,
gpuLayers: textSettings?.gpuLayers ?? -1,
enableThinking: textSettings?.enableThinking !== false,
flashAttn: textSettings?.flashAttn,
@@ -512,15 +562,29 @@ function TextChat({
documentContext += `\n[Attached File: ${att.name}]\n${att.content}\n`;
});
- const combinedText = documentContext ? `${text}\n\n${documentContext}`.trim() : text;
const imageAttachments = attachments.filter(att => att.type === "image");
+ const visionInstruction = imageAttachments.length > 0
+ ? "For attached images: answer in natural language, describe the actual image content, and read visible text carefully when relevant. Do not output raw JSON, bounding boxes, OCR layout objects, or detection labels unless the user explicitly asks for that format."
+ : "";
+ const requestText = imageAttachments.length > 0
+ ? (text && text !== "?" ? text : "Describe what is visible in the image.")
+ : text;
+ const displayText = text || (imageAttachments.length > 0 ? "Describe what is visible in the image." : "");
+ const requestCombinedText = [
+ requestText,
+ documentContext,
+ ].filter(Boolean).join("\n\n").trim();
+ const displayCombinedText = documentContext
+ ? [displayText, documentContext].filter(Boolean).join("\n\n").trim()
+ : displayText;
let userMessageContent;
+ let requestUserMessageContent;
if (imageAttachments.length > 0) {
userMessageContent = [
{
type: "text",
- text: combinedText
+ text: displayCombinedText
},
...imageAttachments.map((img) => ({
type: "image_url",
@@ -529,11 +593,25 @@ function TextChat({
}
}))
];
+ requestUserMessageContent = [
+ {
+ type: "text",
+ text: requestCombinedText
+ },
+ ...imageAttachments.map((img) => ({
+ type: "image_url",
+ image_url: {
+ url: img.sendDataUrl || img.dataUrl
+ }
+ }))
+ ];
} else {
- userMessageContent = combinedText;
+ userMessageContent = displayCombinedText;
+ requestUserMessageContent = requestCombinedText;
}
const nextMessages = [...messages, { role: "user", content: userMessageContent }];
+ const requestConversationMessages = [...messages, { role: "user", content: requestUserMessageContent }];
followGenerationRef.current = true;
setMessages(nextMessages);
setInput("");
@@ -547,7 +625,7 @@ function TextChat({
setMessages([...nextMessages, {
role: "assistant",
content: "",
- generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 },
+ generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0, vision: imageAttachments.length > 0 },
}]);
const controller = new AbortController();
@@ -555,9 +633,13 @@ function TextChat({
try {
const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant.";
+ const combinedSystemPrompt = [
+ systemPrompt.trim(),
+ visionInstruction,
+ ].filter(Boolean).join("\n\n");
const requestMessages = [
- ...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []),
- ...nextMessages,
+ ...(combinedSystemPrompt ? [{ role: "system", content: combinedSystemPrompt }] : []),
+ ...requestConversationMessages,
];
const promptTokenEstimate = requestMessages.reduce((sum, message) => {
const messageText = Array.isArray(message.content)
@@ -579,10 +661,11 @@ function TextChat({
let thinkingEndedAt = null;
let thinkingDuration = 0;
- const baseMaxTokens = textSettings?.maxTokens || 384;
- const effectiveMaxTokens = textSettings?.enableThinking !== false
- ? Math.max(baseMaxTokens, 1024)
- : baseMaxTokens;
+ const thinkingEnabled = textSettings?.enableThinking !== false;
+ const manualMaxTokens = textSettings?.maxTokens || 1024;
+ const effectiveMaxTokens = textSettings?.responseTokenMode === "manual"
+ ? (thinkingEnabled ? Math.max(manualMaxTokens, 1024) : manualMaxTokens)
+ : getAutoMaxResponseTokens(promptTokenEstimate, thinkingEnabled);
const response = await streamChatWithLlm(requestMessages, {
temperature: textSettings?.temperature || 0.7,
@@ -592,7 +675,7 @@ function TextChat({
minP: textSettings?.minP,
repeatPenalty: textSettings?.repeatPenalty,
seed: textSettings?.seed,
- enableThinking: textSettings?.enableThinking !== false,
+ enableThinking: thinkingEnabled,
signal: controller.signal,
}, (_token, fullText, _reasoningToken, fullReasoning) => {
const now = performance.now();
@@ -929,6 +1012,9 @@ function TextChat({
);
const displayContent = Array.isArray(message.content) ? message.content : processed.content;
const displayReasoning = processed.reasoning;
+ const hasDisplayContent = Array.isArray(displayContent)
+ ? displayContent.length > 0
+ : Boolean(String(displayContent || "").trim());
return (
)}
-
- {Array.isArray(displayContent) ? (
-
- {displayContent.map((item, idx) => {
- if (item.type === "text") return
;
- if (item.type === "image_url") return (
-
- );
- return null;
- })}
-
- ) : (
-
- )}
-
+ {(hasDisplayContent || message.error) && (
+
+ {Array.isArray(displayContent) ? (
+
+ {displayContent.map((item, idx) => {
+ if (item.type === "text") return
;
+ if (item.type === "image_url") return (
+
+ );
+ return null;
+ })}
+
+ ) : (
+
+ )}
+
+ )}
{/* Generation stats pill */}
{message.role === "assistant" && message.generationStats && !message.error && (
<>
{message.generationStats.truncated && (
- Response reached the token limit. Increase Max Response Tokens or ask "continue".
+ Response reached the token limit. Ask "continue" or switch Max Response Tokens to Manual for a larger cap.
)}
{message.generationStats.status === "starting" ? (
- <> Waiting for first token...
+ <> {message.generationStats.vision ? "Processing image..." : "Waiting for first token..."}
) : message.generationStats.status === "streaming" ? (
<>⚡ {message.generationStats.tokensPerSecond.toFixed(1)} tok/s
) : (
diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js
index 6f56907..9c529e3 100644
--- a/app/frontend/src/services/api.js
+++ b/app/frontend/src/services/api.js
@@ -54,6 +54,7 @@ export function normalizeModel(model) {
format: model?.format || "Local Weights File",
backendType: model?.backendType || "",
resolution: model?.resolution || "",
+ isProjector: Boolean(model?.isProjector),
};
}
diff --git a/linux.sh b/linux.sh
index acce2af..828d285 100755
--- a/linux.sh
+++ b/linux.sh
@@ -27,6 +27,7 @@ SETUP_SCRIPT="$SCRIPT_DIR/scripts/setup.sh"
SERVE_SCRIPT="$SCRIPT_DIR/scripts/serve.cjs"
FRONTEND_PORT="${FRONTEND_PORT:-1420}"
+LLM_PORT="${LLM_PORT:-10086}"
SETUP_REASON=""
SETUP_MODE="Repair"
MAX_PERF_FLAG=""
@@ -155,10 +156,11 @@ if [[ -n "$SETUP_REASON" ]]; then
# Clear any existing frontend and backend server processes
if command -v lsof >/dev/null 2>&1; then
- lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
+ lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
+ fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
fi
if ! bash "$SETUP_SCRIPT" $MAX_PERF_FLAG; then
@@ -179,10 +181,11 @@ echo ""
# Clear frontend and backend ports
if command -v lsof >/dev/null 2>&1; then
- lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
+ lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
+ fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
fi
# Start the server
@@ -210,7 +213,7 @@ echo " ============================================================"
echo " Running!"
echo " Web UI: http://localhost:${FRONTEND_PORT}"
echo " GPU API: Auto-selected by the app (starts at 8080)"
-echo " Text API: Starts when a GGUF model is loaded (port 10086)"
+echo " Text API: Starts when a GGUF model is loaded (port ${LLM_PORT})"
echo ""
echo " Press Ctrl+C in this window to stop all services."
echo " ============================================================"
diff --git a/mac.sh b/mac.sh
index 14bbdc5..7175b6c 100755
--- a/mac.sh
+++ b/mac.sh
@@ -30,6 +30,7 @@ SETUP_SCRIPT="$SCRIPT_DIR/scripts/setup.sh"
SERVE_SCRIPT="$SCRIPT_DIR/scripts/serve.cjs"
FRONTEND_PORT="${FRONTEND_PORT:-1420}"
+LLM_PORT="${LLM_PORT:-10086}"
SETUP_REASON=""
SETUP_MODE="Repair"
@@ -129,10 +130,11 @@ if [[ -n "$SETUP_REASON" ]]; then
# Clear any existing frontend and backend server processes
if command -v lsof >/dev/null 2>&1; then
- lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
+ lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
+ fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
fi
if ! bash "$SETUP_SCRIPT"; then
@@ -153,10 +155,11 @@ echo ""
# Clear frontend and backend ports
if command -v lsof >/dev/null 2>&1; then
- lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
+ lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
+ fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
fi
# Start the server
@@ -184,7 +187,7 @@ echo " ============================================================"
echo " Running!"
echo " Web UI: http://localhost:${FRONTEND_PORT}"
echo " GPU API: Auto-selected by the app (starts at 8080)"
-echo " Text API: Starts when a GGUF model is loaded (port 10086)"
+echo " Text API: Starts when a GGUF model is loaded (port ${LLM_PORT})"
echo ""
echo " Press Ctrl+C in this window to stop all services."
echo " ============================================================"
diff --git a/scripts/serve.cjs b/scripts/serve.cjs
index 769d17a..6361529 100644
--- a/scripts/serve.cjs
+++ b/scripts/serve.cjs
@@ -129,8 +129,10 @@ let openvinoModel = null;
let openvinoWidth = null;
let openvinoHeight = null;
let llmProc = null;
+let llmProcSeq = 0;
let llmReady = false;
let llmError = null;
+let llmOperationQueue = Promise.resolve();
let llmSettings = {
model: null,
threads: Math.max(1, Math.min(16, os.cpus().length || 4)),
@@ -152,6 +154,12 @@ let llmSettings = {
ubatchSize: 512,
performanceProfile: "balanced",
};
+
+function runExclusiveLlmOperation(operation) {
+ const run = llmOperationQueue.catch(() => {}).then(operation);
+ llmOperationQueue = run.catch(() => {});
+ return run;
+}
let backendLoadState = {
active: false,
phase: "",
@@ -1254,6 +1262,16 @@ function checkPort(port) {
});
}
+async function waitForPortAvailable(port, timeoutMs = 5000) {
+ const startedAt = Date.now();
+ while (Date.now() - startedAt < timeoutMs) {
+ const status = await checkPort(port);
+ if (status.available) return true;
+ await new Promise((resolve) => setTimeout(resolve, 150));
+ }
+ return false;
+}
+
async function findAvailableBackendPort() {
const preferred = await checkPort(PREFERRED_BACKEND_PORT);
if (preferred.available) return PREFERRED_BACKEND_PORT;
@@ -1285,69 +1303,62 @@ async function findAvailableLlmPort() {
}
function getLlmBackend() {
+ return getLlmBackendCandidates()[0] || null;
+}
+
+function getLlmBackendCandidates() {
+ const candidates = [];
+ const includeUndetectedGpu = process.env.LOCALAI_LLM_TRY_UNDETECTED_GPU === "1";
+ const add = (key, backendPath, mode, detected = true) => {
+ if (!backendPath || !fs.existsSync(backendPath)) return;
+ if (key !== "cpu" && !detected && !includeUndetectedGpu) return;
+ if (candidates.some((item) => item.path === backendPath)) return;
+ candidates.push({ key, path: backendPath, mode, detected });
+ };
+
if (osPlatform === "win32") {
- if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) {
- return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) {
- return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) {
- return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) {
- return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) {
- return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) {
- return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
- }
- return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" };
+ add("cuda", LLM_BACKEND_PATHS.winCuda, "Auto (CUDA/CPU)", Boolean(detectLlamaCudaGpu()));
+ add("vulkan", LLM_BACKEND_PATHS.winVulkan, "Auto (Vulkan/CPU)", Boolean(detectLlamaVulkanGpu()));
+ add("sycl", LLM_BACKEND_PATHS.winSycl, "Auto (SYCL/CPU)", Boolean(detectLlamaSyclGpu()));
+ add("cpu", LLM_BACKEND_PATHS.winCpu, "CPU", true);
} else if (osPlatform === "darwin") {
- return {
- path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64,
- mode: process.arch === "arm64" ? "Metal GPU" : "CPU",
- };
+ add("metal", process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64, process.arch === "arm64" ? "Metal GPU" : "CPU", true);
} else {
- if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda) && detectLlamaCudaGpu()) {
- return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl) && detectLlamaSyclGpu()) {
- return { path: LLM_BACKEND_PATHS.linuxSycl, mode: "Auto (SYCL/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan) && detectLlamaVulkanGpu()) {
- return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) {
- return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl)) {
- return { path: LLM_BACKEND_PATHS.linuxSycl, mode: "Auto (SYCL/CPU)" };
- }
- if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) {
- return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
- }
- return { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" };
+ add("cuda", LLM_BACKEND_PATHS.linuxCuda, "Auto (CUDA/CPU)", Boolean(detectLlamaCudaGpu()));
+ add("sycl", LLM_BACKEND_PATHS.linuxSycl, "Auto (SYCL/CPU)", Boolean(detectLlamaSyclGpu()));
+ add("vulkan", LLM_BACKEND_PATHS.linuxVulkan, "Auto (Vulkan/CPU)", Boolean(detectLlamaVulkanGpu()));
+ add("cpu", LLM_BACKEND_PATHS.linuxCpu, "CPU", true);
}
+
+ const priority = osPlatform === "win32"
+ ? { cuda: 0, vulkan: 1, sycl: 2, metal: 3, cpu: 9 }
+ : osPlatform === "darwin"
+ ? { metal: 0, cpu: 9 }
+ : { cuda: 0, sycl: 1, vulkan: 2, cpu: 9 };
+
+ return candidates.sort((a, b) => {
+ if (a.key === "cpu" && b.key !== "cpu") return 1;
+ if (b.key === "cpu" && a.key !== "cpu") return -1;
+ const detectedDelta = Number(b.detected) - Number(a.detected);
+ if (detectedDelta) return detectedDelta;
+ return (priority[a.key] ?? 8) - (priority[b.key] ?? 8);
+ });
}
function getLlmModels() {
try {
return fs.readdirSync(LLM_MODELS)
- .filter((filename) => {
- const lower = filename.toLowerCase();
- return lower.endsWith(".gguf") && !lower.includes("mmproj");
- })
+ .filter((filename) => filename.toLowerCase().endsWith(".gguf"))
.map((filename) => {
const stats = fs.statSync(path.join(LLM_MODELS, filename));
+ const lower = filename.toLowerCase();
return {
filename,
name: filename,
sizeBytes: stats.size,
size: formatBytes(stats.size),
format: "GGUF",
+ isProjector: lower.includes("mmproj"),
};
});
} catch (_) {
@@ -1355,11 +1366,35 @@ function getLlmModels() {
}
}
-function pingLlmReady() {
+function pingLlmReady(expectedModel = "") {
return new Promise((resolve) => {
const req = http.get(`http://127.0.0.1:${PORT_LLM}/v1/models`, (res) => {
- res.resume();
- resolve(res.statusCode >= 200 && res.statusCode < 500);
+ let body = "";
+ res.setEncoding("utf8");
+ res.on("data", (chunk) => { body += chunk; });
+ res.on("end", () => {
+ if (res.statusCode < 200 || res.statusCode >= 500) {
+ resolve(false);
+ return;
+ }
+ if (!expectedModel) {
+ resolve(true);
+ return;
+ }
+ try {
+ const parsed = JSON.parse(body || "{}");
+ const servedIds = Array.isArray(parsed.data)
+ ? parsed.data.map((item) => String(item.id || item.model || ""))
+ : [];
+ const expectedBase = path.basename(expectedModel).toLowerCase();
+ resolve(servedIds.some((id) => {
+ const lower = id.toLowerCase();
+ return lower === expectedBase || lower.includes(expectedBase);
+ }));
+ } catch (_) {
+ resolve(false);
+ }
+ });
});
req.setTimeout(1000, () => {
req.destroy();
@@ -1369,39 +1404,66 @@ function pingLlmReady() {
});
}
-async function waitForLlmReady(maxAttempts = 240) {
+async function waitForLlmReady(maxAttempts = 240, expectedProc = llmProc, expectedModel = "") {
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
- if (!llmProc) throw new Error(llmError || "llama.cpp exited during startup.");
- if (await pingLlmReady()) {
+ if (!expectedProc || llmProc !== expectedProc) throw new Error(llmError || "llama.cpp exited during startup.");
+ if (await pingLlmReady(expectedModel)) {
llmReady = true;
return;
}
await new Promise((resolve) => setTimeout(resolve, 500));
}
- throw new Error("llama.cpp did not become ready within 2 minutes.");
+ throw new Error(`llama.cpp did not become ready within ${Math.round(maxAttempts / 120)} minutes.`);
}
-function killLlm() {
+function waitForChildExit(proc, timeoutMs) {
return new Promise((resolve) => {
- llmReady = false;
- llmSettings.supportsVision = false;
- llmSettings.visionMode = "none";
- llmSettings.visionStatus = "Projector not loaded";
- llmSettings.mmproj = null;
- if (!llmProc) {
- resolve();
+ if (!proc || proc.exitCode !== null || proc.signalCode !== null) {
+ resolve(true);
return;
}
- const proc = llmProc;
- llmProc = null;
- try { proc.kill("SIGTERM"); } catch (_) {}
- setTimeout(() => {
- try { proc.kill("SIGKILL"); } catch (_) {}
- resolve();
- }, 1200);
+ let done = false;
+ const finish = (exited) => {
+ if (done) return;
+ done = true;
+ clearTimeout(timer);
+ proc.off("exit", onExit);
+ resolve(exited);
+ };
+ const onExit = () => finish(true);
+ const timer = setTimeout(() => finish(false), timeoutMs);
+ proc.once("exit", onExit);
});
}
+async function killLlm() {
+ llmReady = false;
+ llmSettings.supportsVision = false;
+ llmSettings.visionMode = "none";
+ llmSettings.visionStatus = "Projector not loaded";
+ llmSettings.mmproj = null;
+ if (!llmProc) {
+ await waitForPortAvailable(PORT_LLM, 1500);
+ return;
+ }
+
+ const proc = llmProc;
+ const procPort = PORT_LLM;
+ llmProc = null;
+
+ try { proc.kill("SIGTERM"); } catch (_) {}
+ const exitedAfterTerm = await waitForChildExit(proc, 2500);
+ if (!exitedAfterTerm) {
+ try { proc.kill("SIGKILL"); } catch (_) {}
+ await waitForChildExit(proc, 2500);
+ }
+
+ const released = await waitForPortAvailable(procPort, 5000);
+ if (!released) {
+ console.warn(` [llm] Port ${procPort} is still busy after stopping llama.cpp.`);
+ }
+}
+
function getSetupPaths() {
const appDir = path.join(ROOT, "app");
if (osPlatform === "win32") {
@@ -2055,6 +2117,28 @@ function selectGgufFile(siblings = [], tier = "low") {
return files[0] || "";
}
+function selectMmprojFile(siblings = [], modelFilename = "") {
+ const files = siblings
+ .map((item) => typeof item === "string" ? item : item?.rfilename)
+ .filter((name) => name && /\.gguf$/i.test(name))
+ .filter((name) => /(?:^|[-_/])mmproj(?:[-_.\/]|$)|mmproj/i.test(name));
+ if (files.length === 0) return "";
+
+ const modelBase = path.basename(modelFilename).toLowerCase();
+ const modelStem = modelBase.replace(/\.gguf$/i, "").replace(/\.(q\d(?:_k_[ms])?|iq\d_[a-z0-9]+|bf16|f16|f32)$/i, "");
+ const scored = files.map((name) => {
+ const lower = name.toLowerCase();
+ let score = 0;
+ if (/bf16/i.test(lower)) score += 6;
+ if (/f16/i.test(lower)) score += 5;
+ if (/mmproj-model-f16/i.test(lower)) score += 4;
+ if (modelStem && lower.includes(modelStem)) score += 10;
+ return { name, score };
+ });
+ scored.sort((a, b) => b.score - a.score || a.name.length - b.name.length);
+ return scored[0]?.name || "";
+}
+
function classifyHuggingFaceModel(model, filename) {
const searchable = `${model.id || ""} ${filename || ""} ${(model.tags || []).join(" ")}`.toLowerCase();
const parameters = getModelParameterBillions(searchable);
@@ -2179,6 +2263,7 @@ async function searchHuggingFaceModels(query, filters) {
if (model.private || model.gated || model.disabled) continue;
const filename = selectGgufFile(model.siblings, tier);
if (!filename) continue;
+ const projectorFilename = selectMmprojFile(model.siblings, filename);
const traits = classifyHuggingFaceModel(model, filename);
if (filters.includes("vision") && !traits.vision) continue;
if (filters.includes("uncensored") && !traits.uncensored) continue;
@@ -2206,6 +2291,8 @@ async function searchHuggingFaceModels(query, filters) {
resolution: "N/A",
notes: `Community GGUF from ${String(model.id || "").split("/")[0]}. ${Number(model.downloads || 0).toLocaleString()} Hugging Face downloads.`,
url: `https://huggingface.co/${model.id}/resolve/main/${filename.split("/").map(encodeURIComponent).join("/")}`,
+ projectorUrl: projectorFilename ? `https://huggingface.co/${model.id}/resolve/main/${projectorFilename.split("/").map(encodeURIComponent).join("/")}` : "",
+ projectorFilename: projectorFilename ? `${model.id.replace(/\//g, "--")}--${path.basename(projectorFilename)}` : "",
pageUrl: `https://huggingface.co/${model.id}`,
downloads: Number(model.downloads || 0),
likes: Number(model.likes || 0),
@@ -2478,6 +2565,29 @@ function chooseAutoContext(modelFilename, isGpu) {
return 2048;
}
+function buildLlmLoadProfiles(settings = {}, backend = {}) {
+ const requestedContext = Number(settings.contextSize);
+ const safeContext = requestedContext && requestedContext > 0
+ ? Math.max(512, Math.min(4096, requestedContext))
+ : 4096;
+ const isCpu = backend.key === "cpu" || backend.mode === "CPU";
+ const profiles = [
+ { name: "requested", gpuLayers: isCpu ? 0 : undefined },
+ {
+ name: "safe",
+ contextSize: safeContext,
+ gpuLayers: isCpu ? 0 : 0,
+ flashAttn: false,
+ cacheTypeK: "q4_0",
+ cacheTypeV: "q4_0",
+ batchSize: 256,
+ ubatchSize: 256,
+ enableThinking: false,
+ },
+ ];
+ return profiles;
+}
+
async function startLlm(settings = {}) {
const filename = path.basename(String(settings.model || ""));
const modelPath = path.join(LLM_MODELS, filename);
@@ -2488,7 +2598,47 @@ async function startLlm(settings = {}) {
throw new Error("Text generation requires a .gguf model.");
}
- const backend = getLlmBackend();
+ const candidates = getLlmBackendCandidates();
+ if (candidates.length === 0) {
+ throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend.");
+ }
+
+ const failures = [];
+ for (const backend of candidates) {
+ for (const profile of buildLlmLoadProfiles(settings, backend)) {
+ try {
+ await startLlmWithBackend({ ...settings, __loadProfile: profile }, backend);
+ llmSettings.backendFallbacks = failures;
+ return;
+ } catch (err) {
+ const message = err.message || String(err);
+ failures.push({
+ backend: backend.mode,
+ binary: path.basename(backend.path),
+ profile: profile.name,
+ error: message.slice(-500),
+ });
+ console.warn(` [llm] Load failed on ${backend.mode} (${profile.name}): ${message}`);
+ await killLlm();
+ }
+ }
+ }
+
+ const last = failures[failures.length - 1];
+ llmSettings.backendFallbacks = failures;
+ throw new Error(`Text model failed on all available llama.cpp backends. Last failure: ${last?.error || "unknown error"}`);
+}
+
+async function startLlmWithBackend(settings = {}, backend) {
+ const filename = path.basename(String(settings.model || ""));
+ const modelPath = path.join(LLM_MODELS, filename);
+ if (!filename || !pathInside(modelPath, LLM_MODELS) || !fs.existsSync(modelPath)) {
+ throw new Error("Select a downloaded GGUF text model first.");
+ }
+ if (!filename.toLowerCase().endsWith(".gguf")) {
+ throw new Error("Text generation requires a .gguf model.");
+ }
+
if (!backend) {
throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend.");
}
@@ -2499,7 +2649,8 @@ async function startLlm(settings = {}) {
PORT_LLM = await findAvailableLlmPort();
llmError = null;
- let contextSize = Number(settings.contextSize);
+ const loadProfile = settings.__loadProfile || {};
+ let contextSize = Number(loadProfile.contextSize ?? settings.contextSize);
if (!contextSize || contextSize <= 0) {
const isGpu = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.startsWith("Auto");
contextSize = chooseAutoContext(filename, isGpu);
@@ -2509,14 +2660,14 @@ async function startLlm(settings = {}) {
}
const isSyclBackend = backend.mode.includes("SYCL") || path.basename(path.dirname(backend.path)).toLowerCase() === "sycl";
- const requestedGpuLayers = Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1;
+ const requestedGpuLayers = Number.isFinite(Number(loadProfile.gpuLayers ?? settings.gpuLayers)) ? Number(loadProfile.gpuLayers ?? settings.gpuLayers) : -1;
const effectiveGpuLayers = isSyclBackend ? getStableSyclGpuLayers(filename, requestedGpuLayers) : requestedGpuLayers;
- const effectiveFlashAttn = isSyclBackend ? false : settings.flashAttn !== false;
- const effectiveCacheTypeK = isSyclBackend ? "q4_0" : settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0";
- const effectiveCacheTypeV = isSyclBackend ? "q4_0" : settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0";
- const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(settings.batchSize) || llmSettings.batchSize || 512) : Number(settings.batchSize) || llmSettings.batchSize || 512;
- const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(settings.ubatchSize) || llmSettings.ubatchSize || 512;
- const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : settings.enableThinking !== false;
+ const effectiveFlashAttn = isSyclBackend ? false : (loadProfile.flashAttn ?? (settings.flashAttn !== false));
+ const effectiveCacheTypeK = isSyclBackend ? "q4_0" : loadProfile.cacheTypeK || settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0";
+ const effectiveCacheTypeV = isSyclBackend ? "q4_0" : loadProfile.cacheTypeV || settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0";
+ const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512) : Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512;
+ const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512;
+ const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : (loadProfile.enableThinking ?? (settings.enableThinking !== false));
llmSettings = {
...llmSettings,
@@ -2527,6 +2678,7 @@ async function startLlm(settings = {}) {
requestedGpuLayers,
backendMode: backend.mode,
backendBinary: path.basename(backend.path),
+ loadProfile: loadProfile.name || "requested",
enableThinking: effectiveEnableThinking,
supportsThinking: /deepseek|qwen3|gemma-4|gemma4|think|r1|e2b|reasoning/i.test(filename),
// New performance settings
@@ -2692,9 +2844,15 @@ async function startLlm(settings = {}) {
}
console.log(" [llm] Starting:", backend.path, args.join(" "));
- llmProc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv });
- llmProc.stdout.on("data", (data) => process.stdout.write(" [llm] " + data.toString()));
- llmProc.stderr.on("data", (data) => {
+ const proc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv });
+ const procSeq = ++llmProcSeq;
+ llmProc = proc;
+ proc.stdout.on("data", (data) => {
+ if (llmProc !== proc) return;
+ process.stdout.write(" [llm] " + data.toString());
+ });
+ proc.stderr.on("data", (data) => {
+ if (llmProc !== proc) return;
const output = data.toString();
process.stderr.write(" [llm-err] " + output);
if (/Vulkan\d+\s*:/i.test(output)) llmSettings.backendMode = "Vulkan GPU";
@@ -2706,14 +2864,18 @@ async function startLlm(settings = {}) {
llmError = output.trim().slice(-1200);
}
});
- llmProc.on("exit", (code) => {
+ proc.on("exit", (code) => {
+ if (llmProc !== proc) {
+ console.log(" [llm] stale process exited with code", code);
+ return;
+ }
llmReady = false;
llmProc = null;
if (code !== 0 && code !== null && !llmError) llmError = `llama.cpp exited with code ${code}`;
- console.log(" [llm] exited with code", code);
+ console.log(" [llm] exited with code", code, `(process ${procSeq})`);
});
- await waitForLlmReady();
+ await waitForLlmReady(backend.key === "cpu" ? 720 : 360, proc, filename);
}
async function startBackend(settings = {}) {
@@ -3155,7 +3317,11 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki
kind,
};
- console.log(` [download] Starting download of ${filename} from ${url}`);
+ if (redirectCount > 0) {
+ console.log(` [download] Continuing redirected download of ${filename}`);
+ } else {
+ console.log(` [download] Starting download of ${filename} from ${url}`);
+ }
let downloadFinalized = false;
const failDownload = (message, err = null) => {
@@ -3198,16 +3364,17 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki
failDownload("Too many redirects");
return;
}
- console.log(` [download] Redirected to ${redirectUrl}`);
+ console.log(` [download] Following redirect for ${filename}`);
+ downloadState.speed = "Following redirect";
// Clean up redirected request to avoid triggering error handlers later
request.removeAllListeners("error");
request.destroy();
+ response.resume();
fileStream.close();
try { fs.unlinkSync(tempPath); } catch (_) {}
activeDownload = null;
- downloadState.active = false;
startModelDownload(redirectUrl, filename, targetDir, kind, redirectCount + 1);
return;
}
@@ -3818,17 +3985,17 @@ const server = http.createServer(async (req, res) => {
const body = await readJsonBody(req, res);
if (!body) return;
try {
- await startLlm(body);
+ await runExclusiveLlmOperation(() => startLlm(body));
return json(res, 200, { ok: true, ready: llmReady, port: PORT_LLM, settings: llmSettings });
} catch (err) {
llmError = err.message || String(err);
- await killLlm();
+ await runExclusiveLlmOperation(() => killLlm());
return json(res, 500, { ok: false, error: llmError });
}
}
if (req.url === "/api/llm/stop" && req.method === "POST") {
- await killLlm();
+ await runExclusiveLlmOperation(() => killLlm());
return json(res, 200, { ok: true });
}
@@ -3862,7 +4029,7 @@ async function retryLowerContext() {
enableThinking: llmSettings.enableThinking,
};
- await startLlm(newSettings);
+ await runExclusiveLlmOperation(() => startLlm(newSettings));
}
async function doLlmChat(req, res, body, retryCount = 0) {
diff --git a/windows.bat b/windows.bat
index 94e435f..04a105d 100644
--- a/windows.bat
+++ b/windows.bat
@@ -14,6 +14,7 @@ set LLM_VULKAN_BACKEND=%APP%\llm-backend\win\vulkan\llama-server.exe
set LLM_CPU_BACKEND=%APP%\llm-backend\win\cpu\llama-server.exe
set SERVE=%~dp0scripts\serve.cjs
if "%FRONTEND_PORT%"=="" set FRONTEND_PORT=1420
+if "%LLM_PORT%"=="" set LLM_PORT=10086
set SETUP_REASON=
set SETUP_MODE=Repair
@@ -60,6 +61,7 @@ pause >nul
:: Clear old frontend and backend server processes before setup so app/tools/node-win can be replaced
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul
+for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%LLM_PORT% "') do taskkill /f /pid %%a >nul 2>nul
powershell -ExecutionPolicy Bypass -File "%SETUP%"
if errorlevel 1 (
@@ -81,9 +83,10 @@ echo ============================================================
echo.
:: Clear frontend and backend ports to prevent address conflicts.
-echo Clearing frontend port %FRONTEND_PORT% and backend port 8080...
+echo Clearing frontend port %FRONTEND_PORT%, backend port 8080, and text port %LLM_PORT%...
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul
+for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%LLM_PORT% "') do taskkill /f /pid %%a >nul 2>nul
:: Start frontend server + backend manager (serve.cjs manages sd-vulkan.exe)
echo Starting Local AI Image Generator...
@@ -95,7 +98,7 @@ echo ============================================================
echo Running!
echo Web UI: http://localhost:%FRONTEND_PORT%
echo GPU API: Auto-selected by the app (starts at 8080)
-echo Text API: Starts when a GGUF model is loaded (port 10086)
+echo Text API: Starts when a GGUF model is loaded (port %LLM_PORT%)
echo.
echo Press Ctrl+C in this window to stop all services.
echo ============================================================