add text chat UI with LLM configuration and infrastructure support for the new backend port
This commit is contained in:
techjarves committed
1 parent
55e01f137e
commit
8fa0a670c1
10 files changed
+631
-195
No files matched your search
+114
-33
@@ -677,6 +677,10 @@ body {
|
||||
animation: msgSlideIn 0.22s cubic-bezier(0.22, 1, 0.36, 1) both;
|
||||
}
|
||||
|
||||
.chat-message-row.ai {
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
@keyframes msgSlideIn {
|
||||
from { opacity: 0; transform: translateY(10px); }
|
||||
to { opacity: 1; transform: translateY(0); }
|
||||
@@ -714,7 +718,9 @@ body {
|
||||
.chat-avatar.ai {
|
||||
background: linear-gradient(135deg, var(--md-sys-color-primary), color-mix(in srgb, var(--md-sys-color-primary) 70%, #7c3aed));
|
||||
color: var(--md-sys-color-on-primary);
|
||||
box-shadow: 0 2px 8px color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
|
||||
box-shadow:
|
||||
0 0 0 3px color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent),
|
||||
0 8px 20px color-mix(in srgb, var(--md-sys-color-primary) 28%, transparent);
|
||||
}
|
||||
|
||||
.chat-avatar.user {
|
||||
@@ -728,7 +734,7 @@ body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
max-width: min(1180px, 92%);
|
||||
gap: 4px;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.chat-message-row.user .chat-bubble-wrap {
|
||||
@@ -777,12 +783,13 @@ body {
|
||||
|
||||
/* Sender label */
|
||||
.chat-sender-label {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.06em;
|
||||
font-size: 0.66rem;
|
||||
font-weight: 800;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
color: var(--md-sys-color-outline);
|
||||
padding: 0 2px;
|
||||
line-height: 1.1;
|
||||
}
|
||||
|
||||
/* ─── Generation Stats ────────────────────────────────────────── */
|
||||
@@ -802,23 +809,23 @@ body {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
margin-top: 4px;
|
||||
padding: 3px 10px;
|
||||
margin-top: 2px;
|
||||
padding: 4px 10px;
|
||||
border-radius: 99px;
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface-variant) 70%, transparent);
|
||||
border: 1px solid var(--border-color);
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface-variant) 82%, transparent);
|
||||
border: 1px solid color-mix(in srgb, var(--border-color) 88%, var(--md-sys-color-primary) 12%);
|
||||
color: var(--md-sys-color-on-surface-variant);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
backdrop-filter: blur(4px);
|
||||
font-size: 0.68rem;
|
||||
font-weight: 800;
|
||||
backdrop-filter: blur(8px);
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.chat-generation-stats.streaming {
|
||||
color: var(--md-sys-color-primary);
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 8%, var(--md-sys-color-surface-variant));
|
||||
animation: statsGlow 1.5s ease-in-out infinite;
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 38%, transparent);
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 10%, var(--md-sys-color-surface-variant));
|
||||
box-shadow: 0 6px 18px color-mix(in srgb, var(--md-sys-color-primary) 10%, transparent);
|
||||
}
|
||||
|
||||
@keyframes statsGlow {
|
||||
@@ -2582,54 +2589,128 @@ html[data-theme^="dark"] .m3-segment-item:hover:not(.active):not(.disabled) {
|
||||
|
||||
/* Chat Thinking Block (Reasoning Process) */
|
||||
.chat-thinking-container {
|
||||
margin-bottom: 6px;
|
||||
border-radius: 8px;
|
||||
border: 1px dashed rgba(255, 255, 255, 0.15);
|
||||
background: rgba(255, 255, 255, 0.03);
|
||||
margin-bottom: 4px;
|
||||
width: fit-content;
|
||||
max-width: min(420px, 100%);
|
||||
border-radius: 999px;
|
||||
border: 1px solid color-mix(in srgb, var(--md-sys-color-primary) 24%, var(--border-color));
|
||||
background:
|
||||
linear-gradient(135deg,
|
||||
color-mix(in srgb, var(--md-sys-color-primary) 10%, var(--md-sys-color-surface-variant)),
|
||||
color-mix(in srgb, var(--md-sys-color-surface-variant) 88%, transparent));
|
||||
box-shadow: 0 8px 24px color-mix(in srgb, var(--md-sys-color-primary) 9%, transparent);
|
||||
overflow: hidden;
|
||||
max-width: 100%;
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
|
||||
.chat-thinking-container:has(.chat-thinking-content) {
|
||||
border-radius: 13px;
|
||||
max-width: min(720px, 92vw);
|
||||
}
|
||||
|
||||
.chat-thinking-header {
|
||||
width: 100%;
|
||||
width: auto;
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 6px 10px;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
gap: 10px;
|
||||
padding: 7px 11px;
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: var(--md-sys-color-on-surface-variant);
|
||||
font-size: 0.76rem;
|
||||
font-weight: 500;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 800;
|
||||
cursor: pointer;
|
||||
text-align: left;
|
||||
transition: background 0.2s;
|
||||
transition: background 0.18s ease, color 0.18s ease;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.chat-thinking-header:hover {
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 7%, transparent);
|
||||
color: var(--md-sys-color-on-surface);
|
||||
}
|
||||
|
||||
.chat-thinking-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
opacity: 0.8;
|
||||
gap: 8px;
|
||||
opacity: 0.92;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.chat-thinking-title::before {
|
||||
content: "";
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
background: var(--md-sys-color-primary);
|
||||
box-shadow: 0 0 0 4px color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent);
|
||||
animation: thinkingPulse 1.35s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes thinkingPulse {
|
||||
0%, 100% {
|
||||
transform: scale(0.9);
|
||||
opacity: 0.72;
|
||||
}
|
||||
50% {
|
||||
transform: scale(1.12);
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.chat-thinking-content {
|
||||
padding: 8px 10px;
|
||||
padding: 9px 12px 10px;
|
||||
font-size: 0.76rem;
|
||||
line-height: 1.45;
|
||||
color: var(--md-sys-color-on-surface-variant);
|
||||
opacity: 0.75;
|
||||
border-top: 1px dashed rgba(255, 255, 255, 0.1);
|
||||
opacity: 0.78;
|
||||
border-top: 1px solid color-mix(in srgb, var(--border-color) 78%, transparent);
|
||||
white-space: pre-wrap;
|
||||
font-family: var(--font-family-monospace, monospace);
|
||||
background: rgba(0, 0, 0, 0.1);
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface) 28%, transparent);
|
||||
max-height: 150px;
|
||||
overflow-y: auto;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color:
|
||||
color-mix(in srgb, var(--md-sys-color-primary) 38%, var(--md-sys-color-outline))
|
||||
color-mix(in srgb, var(--md-sys-color-surface) 65%, transparent);
|
||||
}
|
||||
|
||||
.chat-thinking-content::-webkit-scrollbar {
|
||||
width: 10px;
|
||||
}
|
||||
|
||||
.chat-thinking-content::-webkit-scrollbar-track {
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface) 60%, transparent);
|
||||
border-left: 1px solid color-mix(in srgb, var(--border-color) 70%, transparent);
|
||||
}
|
||||
|
||||
.chat-thinking-content::-webkit-scrollbar-thumb {
|
||||
min-height: 34px;
|
||||
border-radius: 999px;
|
||||
border: 2px solid transparent;
|
||||
background:
|
||||
linear-gradient(
|
||||
180deg,
|
||||
color-mix(in srgb, var(--md-sys-color-primary) 62%, #ffffff 8%),
|
||||
color-mix(in srgb, var(--md-sys-color-primary) 38%, var(--md-sys-color-outline))
|
||||
)
|
||||
border-box;
|
||||
background-clip: padding-box;
|
||||
}
|
||||
|
||||
.chat-thinking-content::-webkit-scrollbar-thumb:hover {
|
||||
background:
|
||||
linear-gradient(
|
||||
180deg,
|
||||
color-mix(in srgb, var(--md-sys-color-primary) 78%, #ffffff 10%),
|
||||
color-mix(in srgb, var(--md-sys-color-primary) 52%, var(--md-sys-color-outline))
|
||||
)
|
||||
border-box;
|
||||
background-clip: padding-box;
|
||||
}
|
||||
|
||||
/* ============================================================
|
||||
|
||||
@@ -122,7 +122,8 @@ function App() {
|
||||
return {
|
||||
...parsed,
|
||||
enableThinking: parsed.enableThinking === true,
|
||||
maxTokens: parsed.maxTokens || 384,
|
||||
maxTokens: !parsed.maxTokens || parsed.maxTokens === 384 ? 1024 : parsed.maxTokens,
|
||||
responseTokenMode: parsed.responseTokenMode || "auto",
|
||||
};
|
||||
} catch (_) {}
|
||||
}
|
||||
@@ -134,7 +135,8 @@ function App() {
|
||||
enableThinking: false,
|
||||
// New performance settings
|
||||
gpuLayers: -1,
|
||||
maxTokens: 384,
|
||||
maxTokens: 1024,
|
||||
responseTokenMode: "auto",
|
||||
topP: 0.95,
|
||||
topK: 40,
|
||||
minP: 0.05,
|
||||
@@ -167,10 +169,42 @@ function App() {
|
||||
if (saved) {
|
||||
try {
|
||||
setConversations(JSON.parse(saved));
|
||||
} catch (_) {}
|
||||
} catch (_) {
|
||||
localStorage.removeItem("chat_conversations");
|
||||
}
|
||||
}
|
||||
}, []);
|
||||
|
||||
const sanitizeMessageForStorage = (message) => {
|
||||
if (!Array.isArray(message?.content)) return message;
|
||||
return {
|
||||
...message,
|
||||
content: message.content.map((item) => {
|
||||
if (item?.type !== "image_url") return item;
|
||||
return {
|
||||
type: "text",
|
||||
text: "[Attached image omitted from saved chat history]",
|
||||
};
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
const sanitizeConversationForStorage = (conversation) => ({
|
||||
...conversation,
|
||||
messages: Array.isArray(conversation.messages)
|
||||
? conversation.messages.map(sanitizeMessageForStorage)
|
||||
: [],
|
||||
});
|
||||
|
||||
const persistConversations = (list) => {
|
||||
const compactList = list.map(sanitizeConversationForStorage);
|
||||
try {
|
||||
localStorage.setItem("chat_conversations", JSON.stringify(compactList));
|
||||
} catch (err) {
|
||||
console.warn("Could not save chat history:", err);
|
||||
}
|
||||
};
|
||||
|
||||
const saveConversationState = useCallback((id, msgs, modelName, newTitle = null) => {
|
||||
setConversations((prev) => {
|
||||
const list = [...prev];
|
||||
@@ -192,7 +226,7 @@ function App() {
|
||||
timestamp: Date.now()
|
||||
});
|
||||
}
|
||||
localStorage.setItem("chat_conversations", JSON.stringify(list));
|
||||
persistConversations(list);
|
||||
return list;
|
||||
});
|
||||
}, []);
|
||||
@@ -201,7 +235,7 @@ function App() {
|
||||
if (e) e.stopPropagation();
|
||||
setConversations((prev) => {
|
||||
const filtered = prev.filter((c) => c.id !== id);
|
||||
localStorage.setItem("chat_conversations", JSON.stringify(filtered));
|
||||
persistConversations(filtered);
|
||||
return filtered;
|
||||
});
|
||||
if (activeConversationId === id) {
|
||||
|
||||
@@ -249,6 +249,7 @@ function ModelManager({
|
||||
const [hasMoreModels, setHasMoreModels] = useState(false);
|
||||
const [isLoadingMoreModels, setIsLoadingMoreModels] = useState(false);
|
||||
const searchRequestRef = React.useRef(0);
|
||||
const pendingCompanionDownloadRef = React.useRef(null);
|
||||
|
||||
const cancelLoadRef = React.useRef(false);
|
||||
|
||||
@@ -261,7 +262,10 @@ function ModelManager({
|
||||
}
|
||||
};
|
||||
|
||||
const modelNames = localModels.map((model) => normalizeModel(model).filename);
|
||||
const normalizedLocalModels = localModels.map(normalizeModel).filter((model) => model.filename);
|
||||
const displayedLocalModels = normalizedLocalModels.filter((model) => !model.isProjector);
|
||||
const allModelNames = normalizedLocalModels.map((model) => model.filename);
|
||||
const modelNames = displayedLocalModels.map((model) => model.filename);
|
||||
const isBusy = loadingModelId !== null || isUnloading;
|
||||
|
||||
const openvinoSupported = Boolean(backendOptions?.openvinoNpu?.supported);
|
||||
@@ -430,10 +434,10 @@ function ModelManager({
|
||||
}
|
||||
};
|
||||
|
||||
const downloadByUrl = async (url, expectedFilename) => {
|
||||
const downloadByUrl = async (url, expectedFilename, companion = null, options = {}) => {
|
||||
if (downloadingModelId) return;
|
||||
|
||||
if (modelNames.includes(expectedFilename)) {
|
||||
if (!options.skipExistingConfirm && allModelNames.includes(expectedFilename)) {
|
||||
if (!(await showConfirm({
|
||||
title: "Overwrite Model?",
|
||||
message: `"${expectedFilename}" is already downloaded. Do you want to download and overwrite it?`,
|
||||
@@ -452,6 +456,7 @@ function ModelManager({
|
||||
}
|
||||
|
||||
try {
|
||||
pendingCompanionDownloadRef.current = companion;
|
||||
let res;
|
||||
if (activeModelType === "image") {
|
||||
res = await downloadModel(url);
|
||||
@@ -461,9 +466,11 @@ function ModelManager({
|
||||
if (res && res.ok) {
|
||||
startProgressPolling(expectedFilename);
|
||||
} else {
|
||||
pendingCompanionDownloadRef.current = null;
|
||||
showAlert({ title: "Download Failed", message: res.error || "Unknown error", danger: true });
|
||||
}
|
||||
} catch (err) {
|
||||
pendingCompanionDownloadRef.current = null;
|
||||
showAlert({ title: "Download Error", message: err.message, danger: true });
|
||||
}
|
||||
};
|
||||
@@ -486,11 +493,15 @@ function ModelManager({
|
||||
}
|
||||
} else {
|
||||
fetchModels();
|
||||
if (modelId === "ggml-model-q4_k.gguf") {
|
||||
const companion = pendingCompanionDownloadRef.current;
|
||||
pendingCompanionDownloadRef.current = null;
|
||||
if (companion?.url && companion?.filename && !allModelNames.includes(companion.filename)) {
|
||||
setTimeout(() => {
|
||||
downloadByUrl(
|
||||
"https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf",
|
||||
"mmproj-model-f16.gguf"
|
||||
companion.url,
|
||||
companion.filename,
|
||||
null,
|
||||
{ skipExistingConfirm: true }
|
||||
);
|
||||
}, 500);
|
||||
}
|
||||
@@ -538,13 +549,35 @@ function ModelManager({
|
||||
return;
|
||||
}
|
||||
|
||||
const hasMainModel = modelNames.includes(model.filename);
|
||||
const hasProjector = !model.projectorFilename || allModelNames.includes(model.projectorFilename);
|
||||
if (hasMainModel && model.projectorUrl && model.projectorFilename && !hasProjector) {
|
||||
const confirmed = await showConfirm({
|
||||
title: "Download Vision Projector?",
|
||||
message: `"${model.name}" is downloaded, but image input needs the matching projector file (${model.projectorFilename}). Download it now?`,
|
||||
confirmLabel: "Download Projector",
|
||||
});
|
||||
if (confirmed) {
|
||||
await downloadByUrl(model.projectorUrl, model.projectorFilename, null, { skipExistingConfirm: true });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const confirmed = await showConfirm({
|
||||
title: "Download Model?",
|
||||
message: `Download "${model.name}" (~${model.approxSize})?`,
|
||||
message: model.projectorUrl
|
||||
? `Download "${model.name}" (~${model.approxSize}) and its required vision projector?`
|
||||
: `Download "${model.name}" (~${model.approxSize})?`,
|
||||
confirmLabel: "Download",
|
||||
});
|
||||
if (confirmed) {
|
||||
await downloadByUrl(model.url, model.filename);
|
||||
await downloadByUrl(
|
||||
model.url,
|
||||
model.filename,
|
||||
model.projectorUrl && model.projectorFilename
|
||||
? { url: model.projectorUrl, filename: model.projectorFilename }
|
||||
: null
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -627,7 +660,7 @@ function ModelManager({
|
||||
device: "",
|
||||
});
|
||||
const cores = textSettings?.threads || specs?.cpu_cores_physical || 4;
|
||||
const context = textSettings?.contextSize || 4096;
|
||||
const context = textSettings?.contextSize ?? 0;
|
||||
const res = await startLlm(modelId, {
|
||||
threads: cores,
|
||||
contextSize: context,
|
||||
@@ -1066,7 +1099,7 @@ function ModelManager({
|
||||
<div className="m3-card" style={{ marginTop: "24px" }}>
|
||||
<h3 className="m3-card-title">
|
||||
<Database size={18} style={{ color: "var(--md-sys-color-primary)" }} />
|
||||
Local {activeModelType === "image" ? "Image" : "Text"} Models ({localModels.length})
|
||||
Local {activeModelType === "image" ? "Image" : "Text"} Models ({displayedLocalModels.length})
|
||||
</h3>
|
||||
|
||||
{isLoadingModels ? (
|
||||
@@ -1074,7 +1107,7 @@ function ModelManager({
|
||||
<RefreshCw className="progress-spinner" size={16} />
|
||||
<span style={{ fontSize: "0.9rem" }}>Scanning models folder...</span>
|
||||
</div>
|
||||
) : localModels.length === 0 ? (
|
||||
) : displayedLocalModels.length === 0 ? (
|
||||
<p style={{ fontSize: "0.9rem", color: "var(--md-sys-color-outline)", textAlign: "center", padding: "16px 0" }}>
|
||||
{activeModelType === "image"
|
||||
? "No image models detected in app/models/. Download from the library below or import a file."
|
||||
@@ -1082,7 +1115,7 @@ function ModelManager({
|
||||
</p>
|
||||
) : (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: "10px" }}>
|
||||
{localModels.map((model) => {
|
||||
{displayedLocalModels.map((model) => {
|
||||
const filename = model.filename;
|
||||
const isActive = activeModelType === "image" ? activeModel === filename : activeLlmModel === filename;
|
||||
|
||||
@@ -1221,8 +1254,11 @@ function ModelManager({
|
||||
<h4 style={{ fontWeight: 700, marginBottom: "12px" }}>{section.group}</h4>
|
||||
<div style={{ display: "grid", gridTemplateColumns: "repeat(auto-fit, minmax(280px, 1fr))", gap: "12px" }}>
|
||||
{section.items.map((model) => {
|
||||
const installed = modelNames.includes(model.filename);
|
||||
const downloading = downloadingModelId === model.filename || downloadingModelId === `${model.filename}.zip`;
|
||||
const hasMainModel = modelNames.includes(model.filename);
|
||||
const hasProjector = !model.projectorFilename || allModelNames.includes(model.projectorFilename);
|
||||
const needsProjector = Boolean(hasMainModel && model.projectorFilename && !hasProjector);
|
||||
const installed = hasMainModel && hasProjector;
|
||||
const downloading = downloadingModelId === model.filename || downloadingModelId === model.projectorFilename || downloadingModelId === `${model.filename}.zip`;
|
||||
const systemTier = getHardwareTier(specs);
|
||||
const isRecommended = typeof model.recommendedFit === "boolean"
|
||||
? model.recommendedFit
|
||||
@@ -1289,7 +1325,7 @@ function ModelManager({
|
||||
disabled={installed || downloadingModelId !== null}
|
||||
>
|
||||
{installed ? <HardDrive size={14} /> : downloading ? <RefreshCw className="progress-spinner" size={14} /> : <DownloadCloud size={14} />}
|
||||
<span>{installed ? "Downloaded" : downloading ? "Downloading" : "Download"}</span>
|
||||
<span>{installed ? "Downloaded" : downloading ? "Downloading" : needsProjector ? "Download Vision File" : "Download"}</span>
|
||||
</button>
|
||||
<a
|
||||
href={model.pageUrl || model.url}
|
||||
|
||||
@@ -232,7 +232,7 @@ function Settings({
|
||||
|
||||
const buildTextStartOptions = (settings) => ({
|
||||
threads: settings?.threads || specs?.cpu_cores_physical || 4,
|
||||
contextSize: settings?.contextSize || 4096,
|
||||
contextSize: settings?.contextSize ?? 0,
|
||||
gpuLayers: settings?.gpuLayers ?? -1,
|
||||
enableThinking: settings?.enableThinking !== false,
|
||||
flashAttn: settings?.flashAttn,
|
||||
@@ -685,17 +685,37 @@ function Settings({
|
||||
<div className="m3-slider-group">
|
||||
<div className="m3-slider-header">
|
||||
<span className="m3-slider-label">Max Response Tokens</span>
|
||||
<span className="settings-value-badge">{textSettings.maxTokens || 384}</span>
|
||||
<span className="settings-value-badge">
|
||||
{(textSettings.responseTokenMode || "auto") === "auto" ? "Auto" : (textSettings.maxTokens || 1024)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="m3-segmented-button" style={{ marginBottom: "10px" }}>
|
||||
{[
|
||||
{ id: "auto", label: "Auto" },
|
||||
{ id: "manual", label: "Manual" },
|
||||
].map((mode) => (
|
||||
<button
|
||||
key={mode.id}
|
||||
className={`m3-segment-item ${(textSettings.responseTokenMode || "auto") === mode.id ? "active" : ""}`}
|
||||
onClick={() => updateTextSetting("responseTokenMode", mode.id)}
|
||||
>
|
||||
{mode.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<input
|
||||
type="range"
|
||||
className="m3-slider"
|
||||
value={textSettings.maxTokens || 384}
|
||||
value={textSettings.maxTokens || 1024}
|
||||
onChange={(e) => updateTextSetting("maxTokens", parseInt(e.target.value))}
|
||||
min="64"
|
||||
max="2048"
|
||||
max="4096"
|
||||
step="64"
|
||||
disabled={(textSettings.responseTokenMode || "auto") === "auto"}
|
||||
/>
|
||||
<span style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)" }}>
|
||||
Auto uses remaining context with a safety buffer. Manual uses the slider value.
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="m3-slider-group">
|
||||
|
||||
@@ -150,6 +150,17 @@ function TextChat({
|
||||
const wordCount = value.split(/\s+/).filter(Boolean).length;
|
||||
return Math.max(wordCount, Math.ceil(value.length / 4));
|
||||
};
|
||||
|
||||
const clamp = (value, min, max) => Math.max(min, Math.min(max, value));
|
||||
|
||||
const getAutoMaxResponseTokens = (promptTokens, thinkingEnabled) => {
|
||||
const contextLimit = Number(status.settings?.contextSize || textSettings?.contextSize || 4096);
|
||||
const safetyBuffer = thinkingEnabled ? 768 : 512;
|
||||
const preferredMinimum = thinkingEnabled ? 512 : 256;
|
||||
const available = contextLimit - promptTokens - safetyBuffer;
|
||||
if (available <= 0) return 64;
|
||||
return clamp(available, Math.min(preferredMinimum, available), 4096);
|
||||
};
|
||||
|
||||
const bottomRef = useRef(null);
|
||||
const textareaRef = useRef(null);
|
||||
@@ -192,14 +203,45 @@ function TextChat({
|
||||
return /\.(txt|md|csv|js|jsx|ts|tsx|py|json|css|html|java|cpp|c|h|rs|go|sh|bat|xml|yaml|yml)$/i.test(file.name) || file.type.startsWith("text/");
|
||||
};
|
||||
|
||||
const optimizeImageForVision = (file, maxSide = 1024, quality = 0.92) => new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onerror = () => reject(new Error(`Could not read ${file.name}`));
|
||||
reader.onload = () => {
|
||||
const originalDataUrl = reader.result;
|
||||
const img = new Image();
|
||||
img.onerror = () => reject(new Error(`Could not decode ${file.name}`));
|
||||
img.onload = () => {
|
||||
const scale = Math.min(1, maxSide / Math.max(img.width, img.height));
|
||||
const width = Math.max(1, Math.round(img.width * scale));
|
||||
const height = Math.max(1, Math.round(img.height * scale));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const ctx = canvas.getContext("2d");
|
||||
ctx.drawImage(img, 0, 0, width, height);
|
||||
const sendDataUrl = canvas.toDataURL("image/jpeg", quality);
|
||||
resolve({
|
||||
previewDataUrl: originalDataUrl,
|
||||
sendDataUrl,
|
||||
originalWidth: img.width,
|
||||
originalHeight: img.height,
|
||||
width,
|
||||
height,
|
||||
});
|
||||
};
|
||||
img.src = originalDataUrl;
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
|
||||
const handleFileChange = (e) => {
|
||||
const files = Array.from(e.target.files);
|
||||
if (!files || files.length === 0) return;
|
||||
|
||||
files.forEach((file) => {
|
||||
if (isImage(file)) {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (event) => {
|
||||
optimizeImageForVision(file)
|
||||
.then((image) => {
|
||||
setAttachments((prev) => [
|
||||
...prev,
|
||||
{
|
||||
@@ -207,11 +249,18 @@ function TextChat({
|
||||
file,
|
||||
type: "image",
|
||||
name: file.name,
|
||||
dataUrl: event.target.result,
|
||||
dataUrl: image.previewDataUrl,
|
||||
sendDataUrl: image.sendDataUrl,
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
originalWidth: image.originalWidth,
|
||||
originalHeight: image.originalHeight,
|
||||
},
|
||||
]);
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
})
|
||||
.catch((err) => {
|
||||
showAlert({ title: "Image Error", message: err.message || String(err), danger: true });
|
||||
});
|
||||
} else if (isTextFile(file)) {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (event) => {
|
||||
@@ -279,7 +328,7 @@ function TextChat({
|
||||
|
||||
const buildTextStartOptions = (settings) => ({
|
||||
threads: settings?.threads || specs?.cpu_cores_physical || 4,
|
||||
contextSize: settings?.contextSize || 4096,
|
||||
contextSize: settings?.contextSize ?? 0,
|
||||
gpuLayers: settings?.gpuLayers ?? -1,
|
||||
enableThinking: settings?.enableThinking !== false,
|
||||
flashAttn: settings?.flashAttn,
|
||||
@@ -360,13 +409,14 @@ function TextChat({
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]);
|
||||
setModels(nextModels);
|
||||
const selectableModels = nextModels.filter((model) => !model.isProjector);
|
||||
setModels(selectableModels);
|
||||
setStatus(nextStatus);
|
||||
const active = nextStatus.settings?.model;
|
||||
setSelectedModel((current) => {
|
||||
const saved = localStorage.getItem("selectedLlmModel");
|
||||
const savedExists = nextModels.some((m) => m.filename === saved);
|
||||
return active || current || (savedExists ? saved : "") || nextModels[0]?.filename || "";
|
||||
const savedExists = selectableModels.some((m) => m.filename === saved);
|
||||
return active || current || (savedExists ? saved : "") || selectableModels[0]?.filename || "";
|
||||
});
|
||||
}, []);
|
||||
|
||||
@@ -453,7 +503,7 @@ function TextChat({
|
||||
|
||||
const result = await startLlm(filename, {
|
||||
threads: textSettings?.threads || specs?.cpu_cores_physical || 4,
|
||||
contextSize: textSettings?.contextSize || 4096,
|
||||
contextSize: textSettings?.contextSize ?? 0,
|
||||
gpuLayers: textSettings?.gpuLayers ?? -1,
|
||||
enableThinking: textSettings?.enableThinking !== false,
|
||||
flashAttn: textSettings?.flashAttn,
|
||||
@@ -512,15 +562,29 @@ function TextChat({
|
||||
documentContext += `\n[Attached File: ${att.name}]\n${att.content}\n`;
|
||||
});
|
||||
|
||||
const combinedText = documentContext ? `${text}\n\n${documentContext}`.trim() : text;
|
||||
const imageAttachments = attachments.filter(att => att.type === "image");
|
||||
const visionInstruction = imageAttachments.length > 0
|
||||
? "For attached images: answer in natural language, describe the actual image content, and read visible text carefully when relevant. Do not output raw JSON, bounding boxes, OCR layout objects, or detection labels unless the user explicitly asks for that format."
|
||||
: "";
|
||||
const requestText = imageAttachments.length > 0
|
||||
? (text && text !== "?" ? text : "Describe what is visible in the image.")
|
||||
: text;
|
||||
const displayText = text || (imageAttachments.length > 0 ? "Describe what is visible in the image." : "");
|
||||
const requestCombinedText = [
|
||||
requestText,
|
||||
documentContext,
|
||||
].filter(Boolean).join("\n\n").trim();
|
||||
const displayCombinedText = documentContext
|
||||
? [displayText, documentContext].filter(Boolean).join("\n\n").trim()
|
||||
: displayText;
|
||||
|
||||
let userMessageContent;
|
||||
let requestUserMessageContent;
|
||||
if (imageAttachments.length > 0) {
|
||||
userMessageContent = [
|
||||
{
|
||||
type: "text",
|
||||
text: combinedText
|
||||
text: displayCombinedText
|
||||
},
|
||||
...imageAttachments.map((img) => ({
|
||||
type: "image_url",
|
||||
@@ -529,11 +593,25 @@ function TextChat({
|
||||
}
|
||||
}))
|
||||
];
|
||||
requestUserMessageContent = [
|
||||
{
|
||||
type: "text",
|
||||
text: requestCombinedText
|
||||
},
|
||||
...imageAttachments.map((img) => ({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: img.sendDataUrl || img.dataUrl
|
||||
}
|
||||
}))
|
||||
];
|
||||
} else {
|
||||
userMessageContent = combinedText;
|
||||
userMessageContent = displayCombinedText;
|
||||
requestUserMessageContent = requestCombinedText;
|
||||
}
|
||||
|
||||
const nextMessages = [...messages, { role: "user", content: userMessageContent }];
|
||||
const requestConversationMessages = [...messages, { role: "user", content: requestUserMessageContent }];
|
||||
followGenerationRef.current = true;
|
||||
setMessages(nextMessages);
|
||||
setInput("");
|
||||
@@ -547,7 +625,7 @@ function TextChat({
|
||||
setMessages([...nextMessages, {
|
||||
role: "assistant",
|
||||
content: "",
|
||||
generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 },
|
||||
generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0, vision: imageAttachments.length > 0 },
|
||||
}]);
|
||||
|
||||
const controller = new AbortController();
|
||||
@@ -555,9 +633,13 @@ function TextChat({
|
||||
|
||||
try {
|
||||
const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant.";
|
||||
const combinedSystemPrompt = [
|
||||
systemPrompt.trim(),
|
||||
visionInstruction,
|
||||
].filter(Boolean).join("\n\n");
|
||||
const requestMessages = [
|
||||
...(systemPrompt.trim() ? [{ role: "system", content: systemPrompt.trim() }] : []),
|
||||
...nextMessages,
|
||||
...(combinedSystemPrompt ? [{ role: "system", content: combinedSystemPrompt }] : []),
|
||||
...requestConversationMessages,
|
||||
];
|
||||
const promptTokenEstimate = requestMessages.reduce((sum, message) => {
|
||||
const messageText = Array.isArray(message.content)
|
||||
@@ -579,10 +661,11 @@ function TextChat({
|
||||
let thinkingEndedAt = null;
|
||||
let thinkingDuration = 0;
|
||||
|
||||
const baseMaxTokens = textSettings?.maxTokens || 384;
|
||||
const effectiveMaxTokens = textSettings?.enableThinking !== false
|
||||
? Math.max(baseMaxTokens, 1024)
|
||||
: baseMaxTokens;
|
||||
const thinkingEnabled = textSettings?.enableThinking !== false;
|
||||
const manualMaxTokens = textSettings?.maxTokens || 1024;
|
||||
const effectiveMaxTokens = textSettings?.responseTokenMode === "manual"
|
||||
? (thinkingEnabled ? Math.max(manualMaxTokens, 1024) : manualMaxTokens)
|
||||
: getAutoMaxResponseTokens(promptTokenEstimate, thinkingEnabled);
|
||||
|
||||
const response = await streamChatWithLlm(requestMessages, {
|
||||
temperature: textSettings?.temperature || 0.7,
|
||||
@@ -592,7 +675,7 @@ function TextChat({
|
||||
minP: textSettings?.minP,
|
||||
repeatPenalty: textSettings?.repeatPenalty,
|
||||
seed: textSettings?.seed,
|
||||
enableThinking: textSettings?.enableThinking !== false,
|
||||
enableThinking: thinkingEnabled,
|
||||
signal: controller.signal,
|
||||
}, (_token, fullText, _reasoningToken, fullReasoning) => {
|
||||
const now = performance.now();
|
||||
@@ -929,6 +1012,9 @@ function TextChat({
|
||||
);
|
||||
const displayContent = Array.isArray(message.content) ? message.content : processed.content;
|
||||
const displayReasoning = processed.reasoning;
|
||||
const hasDisplayContent = Array.isArray(displayContent)
|
||||
? displayContent.length > 0
|
||||
: Boolean(String(displayContent || "").trim());
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -952,35 +1038,37 @@ function TextChat({
|
||||
isComplete={!message.generationStats || message.generationStats.status === "complete"}
|
||||
/>
|
||||
)}
|
||||
<div className={`chat-bubble ${message.error ? "error" : ""}`}>
|
||||
{Array.isArray(displayContent) ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: "8px" }}>
|
||||
{displayContent.map((item, idx) => {
|
||||
if (item.type === "text") return <MarkdownRenderer key={idx} content={item.text} />;
|
||||
if (item.type === "image_url") return (
|
||||
<img key={idx} src={item.image_url.url} alt="Attached image"
|
||||
style={{ maxWidth: "240px", maxHeight: "180px", objectFit: "contain", borderRadius: "8px", marginTop: "4px" }}
|
||||
/>
|
||||
);
|
||||
return null;
|
||||
})}
|
||||
</div>
|
||||
) : (
|
||||
<MarkdownRenderer content={displayContent} />
|
||||
)}
|
||||
</div>
|
||||
{(hasDisplayContent || message.error) && (
|
||||
<div className={`chat-bubble ${message.error ? "error" : ""}`}>
|
||||
{Array.isArray(displayContent) ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: "8px" }}>
|
||||
{displayContent.map((item, idx) => {
|
||||
if (item.type === "text") return <MarkdownRenderer key={idx} content={item.text} />;
|
||||
if (item.type === "image_url") return (
|
||||
<img key={idx} src={item.image_url.url} alt="Attached image"
|
||||
style={{ maxWidth: "240px", maxHeight: "180px", objectFit: "contain", borderRadius: "8px", marginTop: "4px" }}
|
||||
/>
|
||||
);
|
||||
return null;
|
||||
})}
|
||||
</div>
|
||||
) : (
|
||||
<MarkdownRenderer content={displayContent} />
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Generation stats pill */}
|
||||
{message.role === "assistant" && message.generationStats && !message.error && (
|
||||
<>
|
||||
{message.generationStats.truncated && (
|
||||
<div className="chat-generation-warning">
|
||||
Response reached the token limit. Increase Max Response Tokens or ask "continue".
|
||||
Response reached the token limit. Ask "continue" or switch Max Response Tokens to Manual for a larger cap.
|
||||
</div>
|
||||
)}
|
||||
<div className={`chat-generation-stats ${message.generationStats.status}`}>
|
||||
{message.generationStats.status === "starting" ? (
|
||||
<><LoaderCircle size={11} className="progress-spinner" /> Waiting for first token...</>
|
||||
<><LoaderCircle size={11} className="progress-spinner" /> {message.generationStats.vision ? "Processing image..." : "Waiting for first token..."}</>
|
||||
) : message.generationStats.status === "streaming" ? (
|
||||
<><span style={{ opacity: 0.7 }}>⚡</span> {message.generationStats.tokensPerSecond.toFixed(1)} tok/s</>
|
||||
) : (
|
||||
|
||||
@@ -54,6 +54,7 @@ export function normalizeModel(model) {
|
||||
format: model?.format || "Local Weights File",
|
||||
backendType: model?.backendType || "",
|
||||
resolution: model?.resolution || "",
|
||||
isProjector: Boolean(model?.isProjector),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ SETUP_SCRIPT="$SCRIPT_DIR/scripts/setup.sh"
|
||||
SERVE_SCRIPT="$SCRIPT_DIR/scripts/serve.cjs"
|
||||
|
||||
FRONTEND_PORT="${FRONTEND_PORT:-1420}"
|
||||
LLM_PORT="${LLM_PORT:-10086}"
|
||||
SETUP_REASON=""
|
||||
SETUP_MODE="Repair"
|
||||
MAX_PERF_FLAG=""
|
||||
@@ -155,10 +156,11 @@ if [[ -n "$SETUP_REASON" ]]; then
|
||||
|
||||
# Clear any existing frontend and backend server processes
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
|
||||
elif command -v fuser >/dev/null 2>&1; then
|
||||
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "8080/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
if ! bash "$SETUP_SCRIPT" $MAX_PERF_FLAG; then
|
||||
@@ -179,10 +181,11 @@ echo ""
|
||||
|
||||
# Clear frontend and backend ports
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
|
||||
elif command -v fuser >/dev/null 2>&1; then
|
||||
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "8080/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Start the server
|
||||
@@ -210,7 +213,7 @@ echo " ============================================================"
|
||||
echo " Running!"
|
||||
echo " Web UI: http://localhost:${FRONTEND_PORT}"
|
||||
echo " GPU API: Auto-selected by the app (starts at 8080)"
|
||||
echo " Text API: Starts when a GGUF model is loaded (port 10086)"
|
||||
echo " Text API: Starts when a GGUF model is loaded (port ${LLM_PORT})"
|
||||
echo ""
|
||||
echo " Press Ctrl+C in this window to stop all services."
|
||||
echo " ============================================================"
|
||||
|
||||
@@ -30,6 +30,7 @@ SETUP_SCRIPT="$SCRIPT_DIR/scripts/setup.sh"
|
||||
SERVE_SCRIPT="$SCRIPT_DIR/scripts/serve.cjs"
|
||||
|
||||
FRONTEND_PORT="${FRONTEND_PORT:-1420}"
|
||||
LLM_PORT="${LLM_PORT:-10086}"
|
||||
SETUP_REASON=""
|
||||
SETUP_MODE="Repair"
|
||||
|
||||
@@ -129,10 +130,11 @@ if [[ -n "$SETUP_REASON" ]]; then
|
||||
|
||||
# Clear any existing frontend and backend server processes
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
|
||||
elif command -v fuser >/dev/null 2>&1; then
|
||||
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "8080/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
if ! bash "$SETUP_SCRIPT"; then
|
||||
@@ -153,10 +155,11 @@ echo ""
|
||||
|
||||
# Clear frontend and backend ports
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
|
||||
lsof -t -i:"${FRONTEND_PORT}" -i:8080 -i:"${LLM_PORT}" | xargs kill -9 >/dev/null 2>&1 || true
|
||||
elif command -v fuser >/dev/null 2>&1; then
|
||||
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "8080/tcp" >/dev/null 2>&1 || true
|
||||
fuser -k "${LLM_PORT}/tcp" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Start the server
|
||||
@@ -184,7 +187,7 @@ echo " ============================================================"
|
||||
echo " Running!"
|
||||
echo " Web UI: http://localhost:${FRONTEND_PORT}"
|
||||
echo " GPU API: Auto-selected by the app (starts at 8080)"
|
||||
echo " Text API: Starts when a GGUF model is loaded (port 10086)"
|
||||
echo " Text API: Starts when a GGUF model is loaded (port ${LLM_PORT})"
|
||||
echo ""
|
||||
echo " Press Ctrl+C in this window to stop all services."
|
||||
echo " ============================================================"
|
||||
|
||||
+257
-90
@@ -129,8 +129,10 @@ let openvinoModel = null;
|
||||
let openvinoWidth = null;
|
||||
let openvinoHeight = null;
|
||||
let llmProc = null;
|
||||
let llmProcSeq = 0;
|
||||
let llmReady = false;
|
||||
let llmError = null;
|
||||
let llmOperationQueue = Promise.resolve();
|
||||
let llmSettings = {
|
||||
model: null,
|
||||
threads: Math.max(1, Math.min(16, os.cpus().length || 4)),
|
||||
@@ -152,6 +154,12 @@ let llmSettings = {
|
||||
ubatchSize: 512,
|
||||
performanceProfile: "balanced",
|
||||
};
|
||||
|
||||
function runExclusiveLlmOperation(operation) {
|
||||
const run = llmOperationQueue.catch(() => {}).then(operation);
|
||||
llmOperationQueue = run.catch(() => {});
|
||||
return run;
|
||||
}
|
||||
let backendLoadState = {
|
||||
active: false,
|
||||
phase: "",
|
||||
@@ -1254,6 +1262,16 @@ function checkPort(port) {
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForPortAvailable(port, timeoutMs = 5000) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
const status = await checkPort(port);
|
||||
if (status.available) return true;
|
||||
await new Promise((resolve) => setTimeout(resolve, 150));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function findAvailableBackendPort() {
|
||||
const preferred = await checkPort(PREFERRED_BACKEND_PORT);
|
||||
if (preferred.available) return PREFERRED_BACKEND_PORT;
|
||||
@@ -1285,69 +1303,62 @@ async function findAvailableLlmPort() {
|
||||
}
|
||||
|
||||
function getLlmBackend() {
|
||||
return getLlmBackendCandidates()[0] || null;
|
||||
}
|
||||
|
||||
function getLlmBackendCandidates() {
|
||||
const candidates = [];
|
||||
const includeUndetectedGpu = process.env.LOCALAI_LLM_TRY_UNDETECTED_GPU === "1";
|
||||
const add = (key, backendPath, mode, detected = true) => {
|
||||
if (!backendPath || !fs.existsSync(backendPath)) return;
|
||||
if (key !== "cpu" && !detected && !includeUndetectedGpu) return;
|
||||
if (candidates.some((item) => item.path === backendPath)) return;
|
||||
candidates.push({ key, path: backendPath, mode, detected });
|
||||
};
|
||||
|
||||
if (osPlatform === "win32") {
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.winCuda) && detectLlamaCudaGpu()) {
|
||||
return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan) && detectLlamaVulkanGpu()) {
|
||||
return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl) && detectLlamaSyclGpu()) {
|
||||
return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.winCuda)) {
|
||||
return { path: LLM_BACKEND_PATHS.winCuda, mode: "Auto (CUDA/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.winVulkan)) {
|
||||
return { path: LLM_BACKEND_PATHS.winVulkan, mode: "Auto (Vulkan/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.winSycl)) {
|
||||
return { path: LLM_BACKEND_PATHS.winSycl, mode: "Auto (SYCL/CPU)" };
|
||||
}
|
||||
return { path: LLM_BACKEND_PATHS.winCpu, mode: "CPU" };
|
||||
add("cuda", LLM_BACKEND_PATHS.winCuda, "Auto (CUDA/CPU)", Boolean(detectLlamaCudaGpu()));
|
||||
add("vulkan", LLM_BACKEND_PATHS.winVulkan, "Auto (Vulkan/CPU)", Boolean(detectLlamaVulkanGpu()));
|
||||
add("sycl", LLM_BACKEND_PATHS.winSycl, "Auto (SYCL/CPU)", Boolean(detectLlamaSyclGpu()));
|
||||
add("cpu", LLM_BACKEND_PATHS.winCpu, "CPU", true);
|
||||
} else if (osPlatform === "darwin") {
|
||||
return {
|
||||
path: process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64,
|
||||
mode: process.arch === "arm64" ? "Metal GPU" : "CPU",
|
||||
};
|
||||
add("metal", process.arch === "arm64" ? LLM_BACKEND_PATHS.macArm64 : LLM_BACKEND_PATHS.macX64, process.arch === "arm64" ? "Metal GPU" : "CPU", true);
|
||||
} else {
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda) && detectLlamaCudaGpu()) {
|
||||
return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl) && detectLlamaSyclGpu()) {
|
||||
return { path: LLM_BACKEND_PATHS.linuxSycl, mode: "Auto (SYCL/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan) && detectLlamaVulkanGpu()) {
|
||||
return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.linuxCuda)) {
|
||||
return { path: LLM_BACKEND_PATHS.linuxCuda, mode: "Auto (CUDA/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.linuxSycl)) {
|
||||
return { path: LLM_BACKEND_PATHS.linuxSycl, mode: "Auto (SYCL/CPU)" };
|
||||
}
|
||||
if (fs.existsSync(LLM_BACKEND_PATHS.linuxVulkan)) {
|
||||
return { path: LLM_BACKEND_PATHS.linuxVulkan, mode: "Auto (Vulkan/CPU)" };
|
||||
}
|
||||
return { path: LLM_BACKEND_PATHS.linuxCpu, mode: "CPU" };
|
||||
add("cuda", LLM_BACKEND_PATHS.linuxCuda, "Auto (CUDA/CPU)", Boolean(detectLlamaCudaGpu()));
|
||||
add("sycl", LLM_BACKEND_PATHS.linuxSycl, "Auto (SYCL/CPU)", Boolean(detectLlamaSyclGpu()));
|
||||
add("vulkan", LLM_BACKEND_PATHS.linuxVulkan, "Auto (Vulkan/CPU)", Boolean(detectLlamaVulkanGpu()));
|
||||
add("cpu", LLM_BACKEND_PATHS.linuxCpu, "CPU", true);
|
||||
}
|
||||
|
||||
const priority = osPlatform === "win32"
|
||||
? { cuda: 0, vulkan: 1, sycl: 2, metal: 3, cpu: 9 }
|
||||
: osPlatform === "darwin"
|
||||
? { metal: 0, cpu: 9 }
|
||||
: { cuda: 0, sycl: 1, vulkan: 2, cpu: 9 };
|
||||
|
||||
return candidates.sort((a, b) => {
|
||||
if (a.key === "cpu" && b.key !== "cpu") return 1;
|
||||
if (b.key === "cpu" && a.key !== "cpu") return -1;
|
||||
const detectedDelta = Number(b.detected) - Number(a.detected);
|
||||
if (detectedDelta) return detectedDelta;
|
||||
return (priority[a.key] ?? 8) - (priority[b.key] ?? 8);
|
||||
});
|
||||
}
|
||||
|
||||
function getLlmModels() {
|
||||
try {
|
||||
return fs.readdirSync(LLM_MODELS)
|
||||
.filter((filename) => {
|
||||
const lower = filename.toLowerCase();
|
||||
return lower.endsWith(".gguf") && !lower.includes("mmproj");
|
||||
})
|
||||
.filter((filename) => filename.toLowerCase().endsWith(".gguf"))
|
||||
.map((filename) => {
|
||||
const stats = fs.statSync(path.join(LLM_MODELS, filename));
|
||||
const lower = filename.toLowerCase();
|
||||
return {
|
||||
filename,
|
||||
name: filename,
|
||||
sizeBytes: stats.size,
|
||||
size: formatBytes(stats.size),
|
||||
format: "GGUF",
|
||||
isProjector: lower.includes("mmproj"),
|
||||
};
|
||||
});
|
||||
} catch (_) {
|
||||
@@ -1355,11 +1366,35 @@ function getLlmModels() {
|
||||
}
|
||||
}
|
||||
|
||||
function pingLlmReady() {
|
||||
function pingLlmReady(expectedModel = "") {
|
||||
return new Promise((resolve) => {
|
||||
const req = http.get(`http://127.0.0.1:${PORT_LLM}/v1/models`, (res) => {
|
||||
res.resume();
|
||||
resolve(res.statusCode >= 200 && res.statusCode < 500);
|
||||
let body = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { body += chunk; });
|
||||
res.on("end", () => {
|
||||
if (res.statusCode < 200 || res.statusCode >= 500) {
|
||||
resolve(false);
|
||||
return;
|
||||
}
|
||||
if (!expectedModel) {
|
||||
resolve(true);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(body || "{}");
|
||||
const servedIds = Array.isArray(parsed.data)
|
||||
? parsed.data.map((item) => String(item.id || item.model || ""))
|
||||
: [];
|
||||
const expectedBase = path.basename(expectedModel).toLowerCase();
|
||||
resolve(servedIds.some((id) => {
|
||||
const lower = id.toLowerCase();
|
||||
return lower === expectedBase || lower.includes(expectedBase);
|
||||
}));
|
||||
} catch (_) {
|
||||
resolve(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
req.setTimeout(1000, () => {
|
||||
req.destroy();
|
||||
@@ -1369,39 +1404,66 @@ function pingLlmReady() {
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForLlmReady(maxAttempts = 240) {
|
||||
async function waitForLlmReady(maxAttempts = 240, expectedProc = llmProc, expectedModel = "") {
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
||||
if (!llmProc) throw new Error(llmError || "llama.cpp exited during startup.");
|
||||
if (await pingLlmReady()) {
|
||||
if (!expectedProc || llmProc !== expectedProc) throw new Error(llmError || "llama.cpp exited during startup.");
|
||||
if (await pingLlmReady(expectedModel)) {
|
||||
llmReady = true;
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
throw new Error("llama.cpp did not become ready within 2 minutes.");
|
||||
throw new Error(`llama.cpp did not become ready within ${Math.round(maxAttempts / 120)} minutes.`);
|
||||
}
|
||||
|
||||
function killLlm() {
|
||||
function waitForChildExit(proc, timeoutMs) {
|
||||
return new Promise((resolve) => {
|
||||
llmReady = false;
|
||||
llmSettings.supportsVision = false;
|
||||
llmSettings.visionMode = "none";
|
||||
llmSettings.visionStatus = "Projector not loaded";
|
||||
llmSettings.mmproj = null;
|
||||
if (!llmProc) {
|
||||
resolve();
|
||||
if (!proc || proc.exitCode !== null || proc.signalCode !== null) {
|
||||
resolve(true);
|
||||
return;
|
||||
}
|
||||
const proc = llmProc;
|
||||
llmProc = null;
|
||||
try { proc.kill("SIGTERM"); } catch (_) {}
|
||||
setTimeout(() => {
|
||||
try { proc.kill("SIGKILL"); } catch (_) {}
|
||||
resolve();
|
||||
}, 1200);
|
||||
let done = false;
|
||||
const finish = (exited) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
proc.off("exit", onExit);
|
||||
resolve(exited);
|
||||
};
|
||||
const onExit = () => finish(true);
|
||||
const timer = setTimeout(() => finish(false), timeoutMs);
|
||||
proc.once("exit", onExit);
|
||||
});
|
||||
}
|
||||
|
||||
async function killLlm() {
|
||||
llmReady = false;
|
||||
llmSettings.supportsVision = false;
|
||||
llmSettings.visionMode = "none";
|
||||
llmSettings.visionStatus = "Projector not loaded";
|
||||
llmSettings.mmproj = null;
|
||||
if (!llmProc) {
|
||||
await waitForPortAvailable(PORT_LLM, 1500);
|
||||
return;
|
||||
}
|
||||
|
||||
const proc = llmProc;
|
||||
const procPort = PORT_LLM;
|
||||
llmProc = null;
|
||||
|
||||
try { proc.kill("SIGTERM"); } catch (_) {}
|
||||
const exitedAfterTerm = await waitForChildExit(proc, 2500);
|
||||
if (!exitedAfterTerm) {
|
||||
try { proc.kill("SIGKILL"); } catch (_) {}
|
||||
await waitForChildExit(proc, 2500);
|
||||
}
|
||||
|
||||
const released = await waitForPortAvailable(procPort, 5000);
|
||||
if (!released) {
|
||||
console.warn(` [llm] Port ${procPort} is still busy after stopping llama.cpp.`);
|
||||
}
|
||||
}
|
||||
|
||||
function getSetupPaths() {
|
||||
const appDir = path.join(ROOT, "app");
|
||||
if (osPlatform === "win32") {
|
||||
@@ -2055,6 +2117,28 @@ function selectGgufFile(siblings = [], tier = "low") {
|
||||
return files[0] || "";
|
||||
}
|
||||
|
||||
function selectMmprojFile(siblings = [], modelFilename = "") {
|
||||
const files = siblings
|
||||
.map((item) => typeof item === "string" ? item : item?.rfilename)
|
||||
.filter((name) => name && /\.gguf$/i.test(name))
|
||||
.filter((name) => /(?:^|[-_/])mmproj(?:[-_.\/]|$)|mmproj/i.test(name));
|
||||
if (files.length === 0) return "";
|
||||
|
||||
const modelBase = path.basename(modelFilename).toLowerCase();
|
||||
const modelStem = modelBase.replace(/\.gguf$/i, "").replace(/\.(q\d(?:_k_[ms])?|iq\d_[a-z0-9]+|bf16|f16|f32)$/i, "");
|
||||
const scored = files.map((name) => {
|
||||
const lower = name.toLowerCase();
|
||||
let score = 0;
|
||||
if (/bf16/i.test(lower)) score += 6;
|
||||
if (/f16/i.test(lower)) score += 5;
|
||||
if (/mmproj-model-f16/i.test(lower)) score += 4;
|
||||
if (modelStem && lower.includes(modelStem)) score += 10;
|
||||
return { name, score };
|
||||
});
|
||||
scored.sort((a, b) => b.score - a.score || a.name.length - b.name.length);
|
||||
return scored[0]?.name || "";
|
||||
}
|
||||
|
||||
function classifyHuggingFaceModel(model, filename) {
|
||||
const searchable = `${model.id || ""} ${filename || ""} ${(model.tags || []).join(" ")}`.toLowerCase();
|
||||
const parameters = getModelParameterBillions(searchable);
|
||||
@@ -2179,6 +2263,7 @@ async function searchHuggingFaceModels(query, filters) {
|
||||
if (model.private || model.gated || model.disabled) continue;
|
||||
const filename = selectGgufFile(model.siblings, tier);
|
||||
if (!filename) continue;
|
||||
const projectorFilename = selectMmprojFile(model.siblings, filename);
|
||||
const traits = classifyHuggingFaceModel(model, filename);
|
||||
if (filters.includes("vision") && !traits.vision) continue;
|
||||
if (filters.includes("uncensored") && !traits.uncensored) continue;
|
||||
@@ -2206,6 +2291,8 @@ async function searchHuggingFaceModels(query, filters) {
|
||||
resolution: "N/A",
|
||||
notes: `Community GGUF from ${String(model.id || "").split("/")[0]}. ${Number(model.downloads || 0).toLocaleString()} Hugging Face downloads.`,
|
||||
url: `https://huggingface.co/${model.id}/resolve/main/${filename.split("/").map(encodeURIComponent).join("/")}`,
|
||||
projectorUrl: projectorFilename ? `https://huggingface.co/${model.id}/resolve/main/${projectorFilename.split("/").map(encodeURIComponent).join("/")}` : "",
|
||||
projectorFilename: projectorFilename ? `${model.id.replace(/\//g, "--")}--${path.basename(projectorFilename)}` : "",
|
||||
pageUrl: `https://huggingface.co/${model.id}`,
|
||||
downloads: Number(model.downloads || 0),
|
||||
likes: Number(model.likes || 0),
|
||||
@@ -2478,6 +2565,29 @@ function chooseAutoContext(modelFilename, isGpu) {
|
||||
return 2048;
|
||||
}
|
||||
|
||||
function buildLlmLoadProfiles(settings = {}, backend = {}) {
|
||||
const requestedContext = Number(settings.contextSize);
|
||||
const safeContext = requestedContext && requestedContext > 0
|
||||
? Math.max(512, Math.min(4096, requestedContext))
|
||||
: 4096;
|
||||
const isCpu = backend.key === "cpu" || backend.mode === "CPU";
|
||||
const profiles = [
|
||||
{ name: "requested", gpuLayers: isCpu ? 0 : undefined },
|
||||
{
|
||||
name: "safe",
|
||||
contextSize: safeContext,
|
||||
gpuLayers: isCpu ? 0 : 0,
|
||||
flashAttn: false,
|
||||
cacheTypeK: "q4_0",
|
||||
cacheTypeV: "q4_0",
|
||||
batchSize: 256,
|
||||
ubatchSize: 256,
|
||||
enableThinking: false,
|
||||
},
|
||||
];
|
||||
return profiles;
|
||||
}
|
||||
|
||||
async function startLlm(settings = {}) {
|
||||
const filename = path.basename(String(settings.model || ""));
|
||||
const modelPath = path.join(LLM_MODELS, filename);
|
||||
@@ -2488,7 +2598,47 @@ async function startLlm(settings = {}) {
|
||||
throw new Error("Text generation requires a .gguf model.");
|
||||
}
|
||||
|
||||
const backend = getLlmBackend();
|
||||
const candidates = getLlmBackendCandidates();
|
||||
if (candidates.length === 0) {
|
||||
throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend.");
|
||||
}
|
||||
|
||||
const failures = [];
|
||||
for (const backend of candidates) {
|
||||
for (const profile of buildLlmLoadProfiles(settings, backend)) {
|
||||
try {
|
||||
await startLlmWithBackend({ ...settings, __loadProfile: profile }, backend);
|
||||
llmSettings.backendFallbacks = failures;
|
||||
return;
|
||||
} catch (err) {
|
||||
const message = err.message || String(err);
|
||||
failures.push({
|
||||
backend: backend.mode,
|
||||
binary: path.basename(backend.path),
|
||||
profile: profile.name,
|
||||
error: message.slice(-500),
|
||||
});
|
||||
console.warn(` [llm] Load failed on ${backend.mode} (${profile.name}): ${message}`);
|
||||
await killLlm();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const last = failures[failures.length - 1];
|
||||
llmSettings.backendFallbacks = failures;
|
||||
throw new Error(`Text model failed on all available llama.cpp backends. Last failure: ${last?.error || "unknown error"}`);
|
||||
}
|
||||
|
||||
async function startLlmWithBackend(settings = {}, backend) {
|
||||
const filename = path.basename(String(settings.model || ""));
|
||||
const modelPath = path.join(LLM_MODELS, filename);
|
||||
if (!filename || !pathInside(modelPath, LLM_MODELS) || !fs.existsSync(modelPath)) {
|
||||
throw new Error("Select a downloaded GGUF text model first.");
|
||||
}
|
||||
if (!filename.toLowerCase().endsWith(".gguf")) {
|
||||
throw new Error("Text generation requires a .gguf model.");
|
||||
}
|
||||
|
||||
if (!backend) {
|
||||
throw new Error("llama.cpp is not installed. Run the platform setup script to install the text backend.");
|
||||
}
|
||||
@@ -2499,7 +2649,8 @@ async function startLlm(settings = {}) {
|
||||
PORT_LLM = await findAvailableLlmPort();
|
||||
llmError = null;
|
||||
|
||||
let contextSize = Number(settings.contextSize);
|
||||
const loadProfile = settings.__loadProfile || {};
|
||||
let contextSize = Number(loadProfile.contextSize ?? settings.contextSize);
|
||||
if (!contextSize || contextSize <= 0) {
|
||||
const isGpu = backend.mode.includes("GPU") || backend.mode.includes("CUDA") || backend.mode.includes("Vulkan") || backend.mode.includes("Metal") || backend.mode.startsWith("Auto");
|
||||
contextSize = chooseAutoContext(filename, isGpu);
|
||||
@@ -2509,14 +2660,14 @@ async function startLlm(settings = {}) {
|
||||
}
|
||||
|
||||
const isSyclBackend = backend.mode.includes("SYCL") || path.basename(path.dirname(backend.path)).toLowerCase() === "sycl";
|
||||
const requestedGpuLayers = Number.isFinite(Number(settings.gpuLayers)) ? Number(settings.gpuLayers) : -1;
|
||||
const requestedGpuLayers = Number.isFinite(Number(loadProfile.gpuLayers ?? settings.gpuLayers)) ? Number(loadProfile.gpuLayers ?? settings.gpuLayers) : -1;
|
||||
const effectiveGpuLayers = isSyclBackend ? getStableSyclGpuLayers(filename, requestedGpuLayers) : requestedGpuLayers;
|
||||
const effectiveFlashAttn = isSyclBackend ? false : settings.flashAttn !== false;
|
||||
const effectiveCacheTypeK = isSyclBackend ? "q4_0" : settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0";
|
||||
const effectiveCacheTypeV = isSyclBackend ? "q4_0" : settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0";
|
||||
const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(settings.batchSize) || llmSettings.batchSize || 512) : Number(settings.batchSize) || llmSettings.batchSize || 512;
|
||||
const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(settings.ubatchSize) || llmSettings.ubatchSize || 512;
|
||||
const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : settings.enableThinking !== false;
|
||||
const effectiveFlashAttn = isSyclBackend ? false : (loadProfile.flashAttn ?? (settings.flashAttn !== false));
|
||||
const effectiveCacheTypeK = isSyclBackend ? "q4_0" : loadProfile.cacheTypeK || settings.cacheTypeK || llmSettings.cacheTypeK || "q8_0";
|
||||
const effectiveCacheTypeV = isSyclBackend ? "q4_0" : loadProfile.cacheTypeV || settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0";
|
||||
const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512) : Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512;
|
||||
const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512;
|
||||
const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : (loadProfile.enableThinking ?? (settings.enableThinking !== false));
|
||||
|
||||
llmSettings = {
|
||||
...llmSettings,
|
||||
@@ -2527,6 +2678,7 @@ async function startLlm(settings = {}) {
|
||||
requestedGpuLayers,
|
||||
backendMode: backend.mode,
|
||||
backendBinary: path.basename(backend.path),
|
||||
loadProfile: loadProfile.name || "requested",
|
||||
enableThinking: effectiveEnableThinking,
|
||||
supportsThinking: /deepseek|qwen3|gemma-4|gemma4|think|r1|e2b|reasoning/i.test(filename),
|
||||
// New performance settings
|
||||
@@ -2692,9 +2844,15 @@ async function startLlm(settings = {}) {
|
||||
}
|
||||
|
||||
console.log(" [llm] Starting:", backend.path, args.join(" "));
|
||||
llmProc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv });
|
||||
llmProc.stdout.on("data", (data) => process.stdout.write(" [llm] " + data.toString()));
|
||||
llmProc.stderr.on("data", (data) => {
|
||||
const proc = spawn(backend.path, args, { stdio: "pipe", env: spawnEnv });
|
||||
const procSeq = ++llmProcSeq;
|
||||
llmProc = proc;
|
||||
proc.stdout.on("data", (data) => {
|
||||
if (llmProc !== proc) return;
|
||||
process.stdout.write(" [llm] " + data.toString());
|
||||
});
|
||||
proc.stderr.on("data", (data) => {
|
||||
if (llmProc !== proc) return;
|
||||
const output = data.toString();
|
||||
process.stderr.write(" [llm-err] " + output);
|
||||
if (/Vulkan\d+\s*:/i.test(output)) llmSettings.backendMode = "Vulkan GPU";
|
||||
@@ -2706,14 +2864,18 @@ async function startLlm(settings = {}) {
|
||||
llmError = output.trim().slice(-1200);
|
||||
}
|
||||
});
|
||||
llmProc.on("exit", (code) => {
|
||||
proc.on("exit", (code) => {
|
||||
if (llmProc !== proc) {
|
||||
console.log(" [llm] stale process exited with code", code);
|
||||
return;
|
||||
}
|
||||
llmReady = false;
|
||||
llmProc = null;
|
||||
if (code !== 0 && code !== null && !llmError) llmError = `llama.cpp exited with code ${code}`;
|
||||
console.log(" [llm] exited with code", code);
|
||||
console.log(" [llm] exited with code", code, `(process ${procSeq})`);
|
||||
});
|
||||
|
||||
await waitForLlmReady();
|
||||
await waitForLlmReady(backend.key === "cpu" ? 720 : 360, proc, filename);
|
||||
}
|
||||
|
||||
async function startBackend(settings = {}) {
|
||||
@@ -3155,7 +3317,11 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki
|
||||
kind,
|
||||
};
|
||||
|
||||
console.log(` [download] Starting download of ${filename} from ${url}`);
|
||||
if (redirectCount > 0) {
|
||||
console.log(` [download] Continuing redirected download of ${filename}`);
|
||||
} else {
|
||||
console.log(` [download] Starting download of ${filename} from ${url}`);
|
||||
}
|
||||
|
||||
let downloadFinalized = false;
|
||||
const failDownload = (message, err = null) => {
|
||||
@@ -3198,16 +3364,17 @@ function startModelDownload(url, overrideFilename = null, targetDir = MODELS, ki
|
||||
failDownload("Too many redirects");
|
||||
return;
|
||||
}
|
||||
console.log(` [download] Redirected to ${redirectUrl}`);
|
||||
console.log(` [download] Following redirect for ${filename}`);
|
||||
downloadState.speed = "Following redirect";
|
||||
|
||||
// Clean up redirected request to avoid triggering error handlers later
|
||||
request.removeAllListeners("error");
|
||||
request.destroy();
|
||||
response.resume();
|
||||
|
||||
fileStream.close();
|
||||
try { fs.unlinkSync(tempPath); } catch (_) {}
|
||||
activeDownload = null;
|
||||
downloadState.active = false;
|
||||
startModelDownload(redirectUrl, filename, targetDir, kind, redirectCount + 1);
|
||||
return;
|
||||
}
|
||||
@@ -3818,17 +3985,17 @@ const server = http.createServer(async (req, res) => {
|
||||
const body = await readJsonBody(req, res);
|
||||
if (!body) return;
|
||||
try {
|
||||
await startLlm(body);
|
||||
await runExclusiveLlmOperation(() => startLlm(body));
|
||||
return json(res, 200, { ok: true, ready: llmReady, port: PORT_LLM, settings: llmSettings });
|
||||
} catch (err) {
|
||||
llmError = err.message || String(err);
|
||||
await killLlm();
|
||||
await runExclusiveLlmOperation(() => killLlm());
|
||||
return json(res, 500, { ok: false, error: llmError });
|
||||
}
|
||||
}
|
||||
|
||||
if (req.url === "/api/llm/stop" && req.method === "POST") {
|
||||
await killLlm();
|
||||
await runExclusiveLlmOperation(() => killLlm());
|
||||
return json(res, 200, { ok: true });
|
||||
}
|
||||
|
||||
@@ -3862,7 +4029,7 @@ async function retryLowerContext() {
|
||||
enableThinking: llmSettings.enableThinking,
|
||||
};
|
||||
|
||||
await startLlm(newSettings);
|
||||
await runExclusiveLlmOperation(() => startLlm(newSettings));
|
||||
}
|
||||
|
||||
async function doLlmChat(req, res, body, retryCount = 0) {
|
||||
|
||||
+5
-2
@@ -14,6 +14,7 @@ set LLM_VULKAN_BACKEND=%APP%\llm-backend\win\vulkan\llama-server.exe
|
||||
set LLM_CPU_BACKEND=%APP%\llm-backend\win\cpu\llama-server.exe
|
||||
set SERVE=%~dp0scripts\serve.cjs
|
||||
if "%FRONTEND_PORT%"=="" set FRONTEND_PORT=1420
|
||||
if "%LLM_PORT%"=="" set LLM_PORT=10086
|
||||
set SETUP_REASON=
|
||||
set SETUP_MODE=Repair
|
||||
|
||||
@@ -60,6 +61,7 @@ pause >nul
|
||||
:: Clear old frontend and backend server processes before setup so app/tools/node-win can be replaced
|
||||
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul
|
||||
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul
|
||||
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%LLM_PORT% "') do taskkill /f /pid %%a >nul 2>nul
|
||||
|
||||
powershell -ExecutionPolicy Bypass -File "%SETUP%"
|
||||
if errorlevel 1 (
|
||||
@@ -81,9 +83,10 @@ echo ============================================================
|
||||
echo.
|
||||
|
||||
:: Clear frontend and backend ports to prevent address conflicts.
|
||||
echo Clearing frontend port %FRONTEND_PORT% and backend port 8080...
|
||||
echo Clearing frontend port %FRONTEND_PORT%, backend port 8080, and text port %LLM_PORT%...
|
||||
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul
|
||||
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul
|
||||
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%LLM_PORT% "') do taskkill /f /pid %%a >nul 2>nul
|
||||
|
||||
:: Start frontend server + backend manager (serve.cjs manages sd-vulkan.exe)
|
||||
echo Starting Local AI Image Generator...
|
||||
@@ -95,7 +98,7 @@ echo ============================================================
|
||||
echo Running!
|
||||
echo Web UI: http://localhost:%FRONTEND_PORT%
|
||||
echo GPU API: Auto-selected by the app (starts at 8080)
|
||||
echo Text API: Starts when a GGUF model is loaded (port 10086)
|
||||
echo Text API: Starts when a GGUF model is loaded (port %LLM_PORT%)
|
||||
echo.
|
||||
echo Press Ctrl+C in this window to stop all services.
|
||||
echo ============================================================
|
||||
|
||||
Reference in new issue
Block a user