feat: add Settings and TextChat components with local server orchestration and configuration management
This commit is contained in:
techjarves committed
1 parent
4a840679d6
commit
e81cce9b32
5 files changed
+2229
-97
No files matched your search
Generated
+2028
File diff suppressed because it is too large.
Load diff
+121
-35
@@ -866,26 +866,34 @@ body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0;
|
||||
padding: 8px 14px 10px;
|
||||
padding: 10px 14px 12px;
|
||||
border-top: 1px solid var(--border-color);
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface) 80%, transparent);
|
||||
backdrop-filter: blur(12px);
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface) 85%, transparent);
|
||||
backdrop-filter: blur(16px);
|
||||
}
|
||||
|
||||
.chat-composer-inner {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
align-items: flex-end;
|
||||
gap: 8px;
|
||||
background: var(--md-sys-color-surface-variant);
|
||||
border: 1.5px solid var(--border-color);
|
||||
border-radius: 16px;
|
||||
padding: 4px 5px 4px 10px;
|
||||
transition: border-color 0.2s ease, box-shadow 0.2s ease;
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: 24px;
|
||||
padding: 6px 6px 6px 14px;
|
||||
transition: border-color 0.25s cubic-bezier(0.4, 0, 0.2, 1),
|
||||
box-shadow 0.25s cubic-bezier(0.4, 0, 0.2, 1),
|
||||
background-color 0.25s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
}
|
||||
|
||||
.chat-composer-inner:hover {
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 30%, var(--border-color));
|
||||
}
|
||||
|
||||
.chat-composer-inner:focus-within {
|
||||
border-color: var(--md-sys-color-primary);
|
||||
box-shadow: 0 0 0 3px color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent);
|
||||
box-shadow: 0 0 0 3px color-mix(in srgb, var(--md-sys-color-primary) 15%, transparent),
|
||||
0 4px 16px rgba(0, 0, 0, 0.08);
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface-variant) 97%, var(--md-sys-color-primary));
|
||||
}
|
||||
|
||||
.chat-composer textarea,
|
||||
@@ -907,31 +915,33 @@ body {
|
||||
border: none !important;
|
||||
background: transparent !important;
|
||||
color: var(--md-sys-color-on-surface);
|
||||
padding: 4px 0;
|
||||
padding: 6px 0;
|
||||
font: inherit;
|
||||
font-size: 0.92rem;
|
||||
outline: none;
|
||||
resize: none;
|
||||
min-height: 28px;
|
||||
max-height: 104px;
|
||||
max-height: 120px;
|
||||
line-height: 1.45;
|
||||
overflow-y: auto;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.chat-composer-textarea::placeholder {
|
||||
color: var(--md-sys-color-outline);
|
||||
font-weight: 400;
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
.chat-composer textarea {
|
||||
min-height: 34px;
|
||||
max-height: 104px;
|
||||
max-height: 120px;
|
||||
resize: vertical;
|
||||
}
|
||||
|
||||
.chat-composer-send-btn {
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
flex-shrink: 0;
|
||||
@@ -939,31 +949,33 @@ body {
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.18s ease;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
background: linear-gradient(135deg, var(--md-sys-color-primary), color-mix(in srgb, var(--md-sys-color-primary) 75%, #7c3aed));
|
||||
color: var(--md-sys-color-on-primary);
|
||||
box-shadow: 0 2px 8px color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
|
||||
box-shadow: 0 2px 8px color-mix(in srgb, var(--md-sys-color-primary) 30%, transparent);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.chat-composer-send-btn:hover:not(:disabled) {
|
||||
transform: scale(1.08);
|
||||
box-shadow: 0 4px 16px color-mix(in srgb, var(--md-sys-color-primary) 50%, transparent);
|
||||
transform: scale(1.08) translateY(-1px);
|
||||
box-shadow: 0 4px 14px color-mix(in srgb, var(--md-sys-color-primary) 45%, transparent);
|
||||
}
|
||||
|
||||
.chat-composer-send-btn:active:not(:disabled) {
|
||||
transform: scale(0.95);
|
||||
transform: scale(0.96) translateY(0);
|
||||
}
|
||||
|
||||
.chat-composer-send-btn:disabled {
|
||||
background: var(--md-sys-color-surface-variant);
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface-variant) 80%, transparent);
|
||||
color: var(--md-sys-color-outline);
|
||||
opacity: 0.5;
|
||||
box-shadow: none;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.chat-composer-stop-btn {
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
flex-shrink: 0;
|
||||
@@ -971,11 +983,12 @@ body {
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.18s ease;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
background: color-mix(in srgb, var(--md-sys-color-error) 15%, var(--md-sys-color-surface-variant));
|
||||
color: var(--md-sys-color-error);
|
||||
border: 1.5px solid color-mix(in srgb, var(--md-sys-color-error) 40%, transparent);
|
||||
animation: stopPulse 1.5s ease-in-out infinite;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
@keyframes stopPulse {
|
||||
@@ -985,12 +998,12 @@ body {
|
||||
|
||||
.chat-composer-stop-btn:hover {
|
||||
background: color-mix(in srgb, var(--md-sys-color-error) 22%, var(--md-sys-color-surface-variant));
|
||||
transform: scale(1.08);
|
||||
transform: scale(1.08) translateY(-1px);
|
||||
}
|
||||
|
||||
.chat-composer-attach-btn {
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
flex-shrink: 0;
|
||||
@@ -998,14 +1011,16 @@ body {
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.18s ease;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
background: transparent;
|
||||
color: var(--md-sys-color-outline);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.chat-composer-attach-btn:hover:not(:disabled) {
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 10%, transparent);
|
||||
color: var(--md-sys-color-primary);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.chat-composer-attach-btn:disabled {
|
||||
@@ -1022,27 +1037,98 @@ body {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 0 10px;
|
||||
padding: 0 12px;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--border-color);
|
||||
background: transparent;
|
||||
background: color-mix(in srgb, var(--md-sys-color-surface-variant) 40%, transparent);
|
||||
color: var(--md-sys-color-outline);
|
||||
font-family: Outfit, sans-serif;
|
||||
font-size: 0.78rem;
|
||||
font-weight: 700;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.chat-composer-deepthink-btn:hover:not(:disabled) {
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 40%, transparent);
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 8%, transparent);
|
||||
color: var(--md-sys-color-on-surface);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.chat-composer-deepthink-btn.active {
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent);
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 12%, transparent);
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 40%, transparent);
|
||||
color: var(--md-sys-color-primary);
|
||||
box-shadow: 0 2px 8px color-mix(in srgb, var(--md-sys-color-primary) 10%, transparent);
|
||||
}
|
||||
|
||||
.chat-composer-deepthink-btn:hover {
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
|
||||
.chat-composer-deepthink-btn.deepthink-btn.active {
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 14%, transparent);
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 45%, transparent);
|
||||
color: var(--md-sys-color-primary);
|
||||
box-shadow: 0 0 12px color-mix(in srgb, var(--md-sys-color-primary) 15%, transparent);
|
||||
}
|
||||
|
||||
.chat-composer-deepthink-btn.active:hover:not(:disabled) {
|
||||
background: color-mix(in srgb, var(--md-sys-color-primary) 18%, transparent);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
/* Custom Dropdown select */
|
||||
.chat-composer-select {
|
||||
height: 32px;
|
||||
width: 105px;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--border-color);
|
||||
background-color: color-mix(in srgb, var(--md-sys-color-surface-variant) 40%, transparent);
|
||||
color: var(--md-sys-color-outline);
|
||||
font-family: Outfit, sans-serif;
|
||||
font-size: 0.78rem;
|
||||
font-weight: 600;
|
||||
padding: 0 26px 0 12px;
|
||||
cursor: pointer;
|
||||
outline: none;
|
||||
margin-bottom: 2px;
|
||||
flex-shrink: 0;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
appearance: none;
|
||||
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23888888' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m6 9 6 6 6-6'/%3E%3C/svg%3E");
|
||||
background-repeat: no-repeat;
|
||||
background-position: right 10px center;
|
||||
}
|
||||
|
||||
.chat-composer-select:hover:not(:disabled) {
|
||||
border-color: color-mix(in srgb, var(--md-sys-color-primary) 40%, transparent);
|
||||
background-color: color-mix(in srgb, var(--md-sys-color-primary) 8%, transparent);
|
||||
color: var(--md-sys-color-on-surface);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.chat-composer-select:focus {
|
||||
border-color: var(--md-sys-color-primary);
|
||||
box-shadow: 0 0 0 2px color-mix(in srgb, var(--md-sys-color-primary) 15%, transparent);
|
||||
}
|
||||
|
||||
.chat-composer-select option {
|
||||
background-color: var(--md-sys-color-surface);
|
||||
color: var(--md-sys-color-on-surface);
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
@keyframes rotateThinking {
|
||||
0% {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.rotate-anim {
|
||||
animation: rotateThinking 7s linear infinite;
|
||||
}
|
||||
|
||||
.text-chat-settings {
|
||||
|
||||
@@ -394,7 +394,7 @@ function Settings({
|
||||
threads: settings?.threads || specs?.cpu_cores_physical || 4,
|
||||
contextSize: settings?.contextSize ?? 0,
|
||||
gpuLayers: settings?.gpuLayers ?? -1,
|
||||
enableThinking: settings?.enableThinking !== false,
|
||||
enableThinking: settings?.enableThinking === true,
|
||||
flashAttn: settings?.flashAttn,
|
||||
cacheTypeK: settings?.cacheTypeK,
|
||||
cacheTypeV: settings?.cacheTypeV,
|
||||
|
||||
@@ -194,6 +194,9 @@ function TextChat({
|
||||
const fileInputRef = useRef(null);
|
||||
const supportsVision = Boolean(status.ready && status.settings?.supportsVision);
|
||||
const supportsThinking = Boolean(status.ready && status.settings?.supportsThinking);
|
||||
const deepThinkEnabled = status.ready
|
||||
? status.settings?.enableThinking === true
|
||||
: textSettings?.enableThinking === true;
|
||||
const visionStatus = status.settings?.visionStatus || "Image input requires a matching mmproj projector file.";
|
||||
|
||||
const resizeComposerInput = useCallback(() => {
|
||||
@@ -344,7 +347,7 @@ function TextChat({
|
||||
threads: settings?.threads || specs?.cpu_cores_physical || 4,
|
||||
contextSize: settings?.contextSize ?? 0,
|
||||
gpuLayers: settings?.gpuLayers ?? -1,
|
||||
enableThinking: settings?.enableThinking !== false,
|
||||
enableThinking: settings?.enableThinking === true,
|
||||
flashAttn: settings?.flashAttn,
|
||||
cacheTypeK: settings?.cacheTypeK,
|
||||
cacheTypeV: settings?.cacheTypeV,
|
||||
@@ -358,7 +361,7 @@ function TextChat({
|
||||
});
|
||||
|
||||
const handleThinkingToggle = async () => {
|
||||
const enabled = textSettings.enableThinking === false;
|
||||
const enabled = !deepThinkEnabled;
|
||||
const nextSettings = { ...textSettings, enableThinking: enabled };
|
||||
|
||||
if (!status.ready || !status.settings?.model) {
|
||||
@@ -519,7 +522,7 @@ function TextChat({
|
||||
threads: textSettings?.threads || specs?.cpu_cores_physical || 4,
|
||||
contextSize: textSettings?.contextSize ?? 0,
|
||||
gpuLayers: textSettings?.gpuLayers ?? -1,
|
||||
enableThinking: textSettings?.enableThinking !== false,
|
||||
enableThinking: textSettings?.enableThinking === true,
|
||||
flashAttn: textSettings?.flashAttn,
|
||||
cacheTypeK: textSettings?.cacheTypeK,
|
||||
cacheTypeV: textSettings?.cacheTypeV,
|
||||
@@ -678,7 +681,7 @@ function TextChat({
|
||||
let thinkingEndedAt = null;
|
||||
let thinkingDuration = 0;
|
||||
|
||||
const thinkingEnabled = textSettings?.enableThinking !== false;
|
||||
const thinkingEnabled = deepThinkEnabled;
|
||||
const manualMaxTokens = textSettings?.maxTokens || 1024;
|
||||
const effectiveMaxTokens = textSettings?.responseTokenMode === "manual"
|
||||
? (thinkingEnabled ? Math.max(manualMaxTokens, 1024) : manualMaxTokens)
|
||||
@@ -707,7 +710,7 @@ function TextChat({
|
||||
: Math.max(0.05, (now - requestStartedAt) / 1000);
|
||||
|
||||
rawAssistantText = fullText;
|
||||
const processed = processMessageContent(fullText, fullReasoning, textSettings?.enableThinking !== false);
|
||||
const processed = processMessageContent(fullText, fullReasoning, deepThinkEnabled);
|
||||
assistantText = processed.content;
|
||||
assistantReasoning = processed.reasoning;
|
||||
|
||||
@@ -787,7 +790,7 @@ function TextChat({
|
||||
truncated: response.finishReason === "length",
|
||||
};
|
||||
|
||||
const processed = processMessageContent(response.content || rawAssistantText || assistantText, response.reasoningContent || assistantReasoning, textSettings?.enableThinking !== false);
|
||||
const processed = processMessageContent(response.content || rawAssistantText || assistantText, response.reasoningContent || assistantReasoning, deepThinkEnabled);
|
||||
const finalMessages = [...nextMessages, {
|
||||
role: "assistant",
|
||||
content: processed.content,
|
||||
@@ -1029,7 +1032,7 @@ function TextChat({
|
||||
const processed = processMessageContent(
|
||||
Array.isArray(message.content) ? "" : (message.content || ""),
|
||||
message.reasoning || "",
|
||||
textSettings?.enableThinking !== false
|
||||
deepThinkEnabled
|
||||
);
|
||||
const displayContent = Array.isArray(message.content) ? message.content : processed.content;
|
||||
const displayReasoning = processed.reasoning;
|
||||
@@ -1052,7 +1055,7 @@ function TextChat({
|
||||
<span className="chat-sender-label">
|
||||
{message.role === "user" ? "You" : "Local AI"}
|
||||
</span>
|
||||
{message.role === "assistant" && displayReasoning && textSettings?.enableThinking !== false && (
|
||||
{message.role === "assistant" && displayReasoning && deepThinkEnabled && (
|
||||
<ThinkingBlock
|
||||
reasoning={displayReasoning}
|
||||
thinkingDuration={message.thinkingDuration}
|
||||
@@ -1179,35 +1182,33 @@ function TextChat({
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
disabled={!supportsVision || isBusy}
|
||||
title={supportsVision ? "Attach files or images" : visionStatus}
|
||||
style={{ marginBottom: "2px" }}
|
||||
>
|
||||
<Paperclip size={17} />
|
||||
</button>
|
||||
<button
|
||||
className={`chat-composer-deepthink-btn ${useWebSearch ? "active" : ""}`}
|
||||
className={`chat-composer-deepthink-btn web-search-btn ${useWebSearch ? "active" : ""}`}
|
||||
onClick={() => setUseWebSearch((value) => !value)}
|
||||
disabled={!status.ready || isBusy}
|
||||
title={useWebSearch ? "Disable web search" : "Enable web search"}
|
||||
style={{ marginBottom: "2px" }}
|
||||
>
|
||||
<Globe2 size={14} />
|
||||
<span>Web</span>
|
||||
</button>
|
||||
{useWebSearch && (
|
||||
<select
|
||||
className="m3-input"
|
||||
value={webTimeFilter}
|
||||
onChange={(event) => setWebTimeFilter(event.target.value)}
|
||||
disabled={isBusy}
|
||||
title="Search recency"
|
||||
style={{ width: "116px", height: "38px", marginBottom: "2px", fontSize: "0.78rem" }}
|
||||
|
||||
|
||||
{status.ready && supportsThinking && (
|
||||
<button
|
||||
className={`chat-composer-deepthink-btn deepthink-btn ${deepThinkEnabled ? "active" : ""}`}
|
||||
onClick={handleThinkingToggle}
|
||||
title={deepThinkEnabled ? "Disable DeepThink reasoning" : "Enable DeepThink reasoning"}
|
||||
>
|
||||
<option value="any">Any time</option>
|
||||
<option value="day">Past day</option>
|
||||
<option value="week">Past week</option>
|
||||
<option value="month">Past month</option>
|
||||
<option value="year">Past year</option>
|
||||
</select>
|
||||
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.2" strokeLinecap="round" strokeLinejoin="round" className={deepThinkEnabled ? "rotate-anim" : ""}>
|
||||
<circle cx="12" cy="12" r="3" />
|
||||
<ellipse cx="12" cy="12" rx="3" ry="9" />
|
||||
<ellipse cx="12" cy="12" rx="9" ry="3" />
|
||||
</svg>
|
||||
<span>DeepThink</span>
|
||||
</button>
|
||||
)}
|
||||
|
||||
<div className="chat-composer-middle" style={{ flex: 1, display: "flex", minWidth: 0 }}>
|
||||
@@ -1228,23 +1229,8 @@ function TextChat({
|
||||
/>
|
||||
</div>
|
||||
|
||||
{status.ready && supportsThinking && (
|
||||
<button
|
||||
className={`chat-composer-deepthink-btn ${textSettings.enableThinking !== false ? "active" : ""}`}
|
||||
onClick={handleThinkingToggle}
|
||||
title={textSettings.enableThinking !== false ? "Disable DeepThink reasoning" : "Enable DeepThink reasoning"}
|
||||
>
|
||||
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.2" strokeLinecap="round" strokeLinejoin="round" className={textSettings.enableThinking !== false ? "rotate-anim" : ""}>
|
||||
<circle cx="12" cy="12" r="3" />
|
||||
<ellipse cx="12" cy="12" rx="3" ry="9" />
|
||||
<ellipse cx="12" cy="12" rx="9" ry="3" />
|
||||
</svg>
|
||||
<span>DeepThink</span>
|
||||
</button>
|
||||
)}
|
||||
|
||||
{isBusy && status.ready ? (
|
||||
<button className="chat-composer-stop-btn" onClick={handleStopGeneration} title="Stop generation" style={{ marginBottom: "2px" }}>
|
||||
<button className="chat-composer-stop-btn" onClick={handleStopGeneration} title="Stop generation">
|
||||
<Square size={15} fill="currentColor" />
|
||||
</button>
|
||||
) : (
|
||||
@@ -1253,7 +1239,6 @@ function TextChat({
|
||||
onClick={sendMessage}
|
||||
disabled={(!input.trim() && attachments.length === 0) || !status.ready}
|
||||
title="Send message"
|
||||
style={{ marginBottom: "2px" }}
|
||||
>
|
||||
<Send size={17} />
|
||||
</button>
|
||||
|
||||
+52
-19
@@ -3653,6 +3653,22 @@ function chooseAutoContext(modelFilename, isGpu) {
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
const lowerFilename = String(modelFilename || "").toLowerCase();
|
||||
const isVisionModel = isVisionModelFilename(lowerFilename);
|
||||
let projectorSizeGb = 0;
|
||||
if (isVisionModel) {
|
||||
try {
|
||||
const files = fs.readdirSync(LLM_MODELS);
|
||||
const projector = files.find((file) => {
|
||||
const lower = file.toLowerCase();
|
||||
return lower.endsWith(".gguf") && lower.includes("mmproj");
|
||||
});
|
||||
if (projector) {
|
||||
projectorSizeGb = fs.statSync(path.join(LLM_MODELS, projector)).size / (1024 * 1024 * 1024);
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
const systemRamGb = os.totalmem() / (1024 * 1024 * 1024);
|
||||
let vramGb = 0;
|
||||
try {
|
||||
@@ -3673,7 +3689,18 @@ function chooseAutoContext(modelFilename, isGpu) {
|
||||
bufferGb = 2.5;
|
||||
}
|
||||
|
||||
const usableGb = Math.max(0.5, availableGb - modelSizeGb - bufferGb);
|
||||
if (isGpu && vramGb > 0) {
|
||||
const gpuBudgetGb = Math.max(0.5, vramGb - modelSizeGb - projectorSizeGb - bufferGb);
|
||||
const maxFastContext = vramGb < 10 ? 8192 : vramGb < 16 ? 16384 : 32768;
|
||||
if (isVisionModel) {
|
||||
return Math.min(maxFastContext, gpuBudgetGb >= 2.5 ? 8192 : 4096);
|
||||
}
|
||||
if (gpuBudgetGb < 1.5) return 4096;
|
||||
if (gpuBudgetGb < 3.0) return Math.min(maxFastContext, 8192);
|
||||
return Math.min(maxFastContext, 16384);
|
||||
}
|
||||
|
||||
const usableGb = Math.max(0.5, availableGb - modelSizeGb - projectorSizeGb - bufferGb);
|
||||
|
||||
let kvPer4096Gb = 0.55;
|
||||
if (modelSizeGb >= 5.5) {
|
||||
@@ -3693,6 +3720,27 @@ function chooseAutoContext(modelFilename, isGpu) {
|
||||
return 2048;
|
||||
}
|
||||
|
||||
function isVisionModelFilename(filename = "") {
|
||||
const lower = String(filename || "").toLowerCase();
|
||||
return lower.includes("llava") ||
|
||||
lower.includes("vision") ||
|
||||
lower.includes("qwen2vl") ||
|
||||
lower.includes("qwen2-vl") ||
|
||||
lower.includes("qwen3.5-4b-vision") ||
|
||||
lower.includes("gemma-4") ||
|
||||
lower.includes("gemma4") ||
|
||||
lower.includes("-e2b-") ||
|
||||
lower.includes("e2b-it") ||
|
||||
lower.includes("paligemma") ||
|
||||
lower.includes("minicpm-v") ||
|
||||
lower.includes("internvl") ||
|
||||
lower.includes("phi-3-vision") ||
|
||||
lower.includes("phi3-vision") ||
|
||||
lower.includes("smolvlm") ||
|
||||
lower.includes("moondream") ||
|
||||
lower === "ggml-model-q4_k.gguf";
|
||||
}
|
||||
|
||||
function buildLlmLoadProfiles(settings = {}, backend = {}) {
|
||||
const requestedContext = Number(settings.contextSize);
|
||||
const safeContext = requestedContext && requestedContext > 0
|
||||
@@ -3819,7 +3867,7 @@ async function startLlmWithBackend(settings = {}, backend) {
|
||||
const effectiveCacheTypeV = isSyclBackend ? "q4_0" : loadProfile.cacheTypeV || settings.cacheTypeV || llmSettings.cacheTypeV || "q8_0";
|
||||
const effectiveBatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512) : Number(loadProfile.batchSize ?? settings.batchSize) || llmSettings.batchSize || 512;
|
||||
const effectiveUbatchSize = isSyclBackend ? Math.min(256, Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512) : Number(loadProfile.ubatchSize ?? settings.ubatchSize) || llmSettings.ubatchSize || 512;
|
||||
const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : (loadProfile.enableThinking ?? (settings.enableThinking !== false));
|
||||
const effectiveEnableThinking = isSyclBackend && effectiveGpuLayers === 0 ? false : (loadProfile.enableThinking ?? (settings.enableThinking === true));
|
||||
|
||||
llmSettings = {
|
||||
...llmSettings,
|
||||
@@ -3848,22 +3896,7 @@ async function startLlmWithBackend(settings = {}, backend) {
|
||||
|
||||
let mmprojPath = null;
|
||||
const lowerFilename = filename.toLowerCase();
|
||||
const isMultimodal = lowerFilename.includes("llava") ||
|
||||
lowerFilename.includes("vision") ||
|
||||
lowerFilename.includes("qwen2vl") ||
|
||||
lowerFilename.includes("qwen2-vl") ||
|
||||
lowerFilename.includes("gemma-4") ||
|
||||
lowerFilename.includes("gemma4") ||
|
||||
lowerFilename.includes("-e2b-") ||
|
||||
lowerFilename.includes("e2b-it") ||
|
||||
lowerFilename.includes("paligemma") ||
|
||||
lowerFilename.includes("minicpm-v") ||
|
||||
lowerFilename.includes("internvl") ||
|
||||
lowerFilename.includes("phi-3-vision") ||
|
||||
lowerFilename.includes("phi3-vision") ||
|
||||
lowerFilename.includes("smolvlm") ||
|
||||
lowerFilename.includes("moondream") ||
|
||||
lowerFilename === "ggml-model-q4_k.gguf";
|
||||
const isMultimodal = isVisionModelFilename(lowerFilename);
|
||||
|
||||
if (isMultimodal) {
|
||||
if (settings.mmproj) {
|
||||
@@ -3958,7 +3991,7 @@ async function startLlmWithBackend(settings = {}, backend) {
|
||||
} else if (llmSettings.supportsThinking) {
|
||||
args.push("--reasoning", "on"); // ✅ Enable reasoning for supported models
|
||||
}
|
||||
if (llmSettings.enableThinking !== false && llmSettings.supportsThinking) {
|
||||
if (llmSettings.enableThinking === true && llmSettings.supportsThinking) {
|
||||
args.push("--reasoning-format", "deepseek"); // ✅ Extract thoughts into reasoning_content
|
||||
}
|
||||
if (mmprojPath) {
|
||||
|
||||
Reference in new issue
Block a user