feat: implement frontend text chat interface with core styles, messaging component, and API service integration.
This commit is contained in:
techjarves committed
1 parent
cf1b7e548a
commit
647341b11f
4 files changed
+139
-10
No files matched your search
@@ -330,6 +330,23 @@ body {
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.chat-generation-stats {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
margin-top: 9px;
|
||||
padding-top: 7px;
|
||||
border-top: 1px solid color-mix(in srgb, var(--md-sys-color-outline) 22%, transparent);
|
||||
color: var(--md-sys-color-on-surface-variant);
|
||||
font-size: 0.72rem;
|
||||
font-weight: 600;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.chat-generation-stats.complete {
|
||||
color: var(--md-sys-color-primary);
|
||||
}
|
||||
|
||||
.chat-thinking,
|
||||
.text-progress {
|
||||
display: flex;
|
||||
|
||||
@@ -25,6 +25,12 @@ function TextChat({
|
||||
setShowHistory,
|
||||
saveConversationState
|
||||
}) {
|
||||
const formatGenerationTime = (seconds) => {
|
||||
const value = Number(seconds) || 0;
|
||||
if (value < 1) return `${Math.round(value * 1000)} ms`;
|
||||
return `${value.toFixed(value < 10 ? 2 : 1)} s`;
|
||||
};
|
||||
|
||||
const [models, setModels] = useState([]);
|
||||
const [status, setStatus] = useState({ ready: false, running: false, settings: {} });
|
||||
const [selectedModel, setSelectedModel] = useState("");
|
||||
@@ -44,6 +50,7 @@ function TextChat({
|
||||
|
||||
const [attachments, setAttachments] = useState([]);
|
||||
const fileInputRef = useRef(null);
|
||||
const supportsVision = Boolean(status.ready && status.settings?.supportsVision);
|
||||
|
||||
const isImage = (file) => {
|
||||
return /\.(jpe?g|png|webp)$/i.test(file.name) || file.type.startsWith("image/");
|
||||
@@ -104,8 +111,15 @@ function TextChat({
|
||||
loadingModelRef.current = loadingModel;
|
||||
}, [loadingModel]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!supportsVision) {
|
||||
setAttachments((current) => current.filter((attachment) => attachment.type !== "image"));
|
||||
}
|
||||
}, [supportsVision]);
|
||||
|
||||
// Load conversation messages when activeConversationId changes
|
||||
useEffect(() => {
|
||||
if (isBusy) return;
|
||||
if (activeConversationId) {
|
||||
const conv = conversations.find(c => c.id === activeConversationId);
|
||||
if (conv) {
|
||||
@@ -129,7 +143,7 @@ function TextChat({
|
||||
setMessages([]);
|
||||
setTokenUsage({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 });
|
||||
}
|
||||
}, [activeConversationId, conversations, models]);
|
||||
}, [activeConversationId, conversations, models, isBusy]);
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
const [nextModels, nextStatus] = await Promise.all([listLlmModels(), getLlmStatus()]);
|
||||
@@ -279,7 +293,12 @@ function TextChat({
|
||||
const firstTitle = isNew ? (displayTitleText.slice(0, 26) + (displayTitleText.length > 26 ? "..." : "")) : null;
|
||||
saveConversationState(convId, nextMessages, selectedModel, firstTitle);
|
||||
|
||||
setMessages([...nextMessages, { role: "assistant", content: "" }]);
|
||||
const requestStartedAt = performance.now();
|
||||
setMessages([...nextMessages, {
|
||||
role: "assistant",
|
||||
content: "",
|
||||
generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 },
|
||||
}]);
|
||||
|
||||
try {
|
||||
const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant.";
|
||||
@@ -289,11 +308,15 @@ function TextChat({
|
||||
];
|
||||
|
||||
let assistantText = "";
|
||||
let streamedTokens = 0;
|
||||
|
||||
const response = await streamChatWithLlm(requestMessages, {
|
||||
temperature: textSettings?.temperature || 0.7,
|
||||
maxTokens: 768,
|
||||
}, (_token, fullText) => {
|
||||
const now = performance.now();
|
||||
streamedTokens += 1;
|
||||
const generationSeconds = Math.max(0.05, (now - requestStartedAt) / 1000);
|
||||
assistantText = fullText;
|
||||
setMessages((prev) => {
|
||||
const updated = [...prev];
|
||||
@@ -301,13 +324,37 @@ function TextChat({
|
||||
updated[updated.length - 1] = {
|
||||
...updated[updated.length - 1],
|
||||
content: fullText,
|
||||
generationStats: {
|
||||
status: "streaming",
|
||||
tokens: streamedTokens,
|
||||
tokensPerSecond: streamedTokens / generationSeconds,
|
||||
seconds: (now - requestStartedAt) / 1000,
|
||||
},
|
||||
};
|
||||
}
|
||||
return updated;
|
||||
});
|
||||
});
|
||||
|
||||
const finalMessages = [...nextMessages, { role: "assistant", content: assistantText }];
|
||||
const completedAt = performance.now();
|
||||
const exactTokens = Number(response.timings?.predicted_n) || streamedTokens;
|
||||
const backendTotalMs = Number(response.timings?.prompt_ms || 0) + Number(response.timings?.predicted_ms || 0);
|
||||
const exactSeconds = backendTotalMs > 0
|
||||
? backendTotalMs / 1000
|
||||
: (completedAt - requestStartedAt) / 1000;
|
||||
const exactTokensPerSecond = Number(response.timings?.predicted_per_second)
|
||||
|| (exactTokens / Math.max(0.001, exactSeconds));
|
||||
const generationStats = {
|
||||
status: "complete",
|
||||
tokens: exactTokens,
|
||||
tokensPerSecond: exactTokensPerSecond,
|
||||
seconds: exactSeconds,
|
||||
};
|
||||
const finalMessages = [...nextMessages, {
|
||||
role: "assistant",
|
||||
content: assistantText,
|
||||
generationStats,
|
||||
}];
|
||||
setMessages(finalMessages);
|
||||
saveConversationState(convId, finalMessages, selectedModel);
|
||||
if (response.usage) setTokenUsage(response.usage);
|
||||
@@ -547,9 +594,23 @@ function TextChat({
|
||||
<span style={{ whiteSpace: "pre-wrap" }}>{message.content}</span>
|
||||
)}
|
||||
</div>
|
||||
{message.role === "assistant" && message.generationStats && !message.error && (
|
||||
<div className={`chat-generation-stats ${message.generationStats.status}`}>
|
||||
{message.generationStats.status === "starting" ? (
|
||||
<>Waiting for first token...</>
|
||||
) : message.generationStats.status === "streaming" ? (
|
||||
<>{message.generationStats.tokensPerSecond.toFixed(1)} tokens/sec</>
|
||||
) : (
|
||||
<>
|
||||
{message.generationStats.tokens} tokens
|
||||
<span>•</span>
|
||||
{formatGenerationTime(message.generationStats.seconds)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{isBusy && status.ready && <div className="chat-thinking"><LoaderCircle className="progress-spinner" size={16} /> Generating...</div>}
|
||||
</>
|
||||
)}
|
||||
<div ref={bottomRef} />
|
||||
@@ -620,8 +681,8 @@ function TextChat({
|
||||
className="m3-btn m3-btn-tonal"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
style={{ padding: "0 12px", height: "48px", display: "flex", alignItems: "center", justifyContent: "center" }}
|
||||
disabled={!status.ready || isBusy}
|
||||
title="Attach files or images"
|
||||
disabled={!supportsVision || isBusy}
|
||||
title={supportsVision ? "Attach files or images" : "Image attachment requires a vision model with an mmproj file"}
|
||||
>
|
||||
<Paperclip size={17} />
|
||||
</button>
|
||||
|
||||
@@ -398,6 +398,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
|
||||
let buffer = "";
|
||||
let content = "";
|
||||
let usage = null;
|
||||
let timings = null;
|
||||
|
||||
const consumeEvent = (eventText) => {
|
||||
const data = eventText
|
||||
@@ -415,6 +416,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
|
||||
onToken(token, content);
|
||||
}
|
||||
if (parsed.usage) usage = parsed.usage;
|
||||
if (parsed.timings) timings = parsed.timings;
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -436,7 +438,7 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () =>
|
||||
|
||||
if (!finished && buffer.trim()) consumeEvent(buffer);
|
||||
if (!content) throw new Error("The text model returned an empty streamed response.");
|
||||
return { content, usage };
|
||||
return { content, usage, timings };
|
||||
}
|
||||
|
||||
export async function downloadLlmModel(url) {
|
||||
|
||||
+52
-3
@@ -127,6 +127,7 @@ let llmSettings = {
|
||||
gpuLayers: -1,
|
||||
backendMode: "",
|
||||
backendBinary: "",
|
||||
supportsVision: false,
|
||||
};
|
||||
let backendLoadState = {
|
||||
active: false,
|
||||
@@ -219,10 +220,48 @@ function detectLinuxGpuFromSysfs() {
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectLlamaVulkanGpu() {
|
||||
const backendPath = osPlatform === "win32"
|
||||
? LLM_BACKEND_PATHS.winVulkan
|
||||
: osPlatform === "linux"
|
||||
? LLM_BACKEND_PATHS.linuxVulkan
|
||||
: "";
|
||||
if (!backendPath || !fs.existsSync(backendPath)) return null;
|
||||
|
||||
try {
|
||||
const result = spawnSync(backendPath, ["--list-devices"], {
|
||||
encoding: "utf8",
|
||||
windowsHide: true,
|
||||
timeout: 10000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
const output = `${result.stdout || ""}\n${result.stderr || ""}`;
|
||||
const devices = [];
|
||||
const devicePattern = /Vulkan\d+\s*:\s*(.+?)\s+\(([\d.]+)\s+MiB(?:,\s*([\d.]+)\s+MiB free)?\)/gi;
|
||||
let match;
|
||||
while ((match = devicePattern.exec(output)) !== null) {
|
||||
devices.push({
|
||||
name: match[1].trim(),
|
||||
vram_gb: Math.round((Number(match[2]) / 1024) * 100) / 100,
|
||||
free_vram_gb: match[3] ? Math.round((Number(match[3]) / 1024) * 100) / 100 : null,
|
||||
});
|
||||
}
|
||||
return devices.sort((a, b) => b.vram_gb - a.vram_gb)[0] || null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function getGpuInfo() {
|
||||
if (cachedGpuInfo) return cachedGpuInfo;
|
||||
|
||||
if (osPlatform === "win32") {
|
||||
const llamaVulkanGpu = detectLlamaVulkanGpu();
|
||||
if (llamaVulkanGpu) {
|
||||
cachedGpuInfo = llamaVulkanGpu;
|
||||
return cachedGpuInfo;
|
||||
}
|
||||
|
||||
try {
|
||||
const stdout = execSync(
|
||||
'powershell -NoProfile -Command "Get-CimInstance Win32_VideoController | ForEach-Object { [PSCustomObject]@{ Name = $_.Name; VRAM = [math]::Round($_.AdapterRAM / 1GB, 2) } } | ConvertTo-Json"',
|
||||
@@ -939,6 +978,7 @@ async function waitForLlmReady(maxAttempts = 240) {
|
||||
function killLlm() {
|
||||
return new Promise((resolve) => {
|
||||
llmReady = false;
|
||||
llmSettings.supportsVision = false;
|
||||
if (!llmProc) {
|
||||
resolve();
|
||||
return;
|
||||
@@ -1631,14 +1671,21 @@ function getTextModelFit(sizeBytes, specs = getHardwareSpecs()) {
|
||||
const fastGpuFit = vramBytes >= runtimeNeed;
|
||||
const cpuOrOffloadFit = usableRam >= runtimeNeed;
|
||||
const unifiedMemoryFit = isAppleSilicon && usableRam >= runtimeNeed;
|
||||
const hybridFit = !fastGpuFit && vramBytes > 0 && cpuOrOffloadFit;
|
||||
const recommended = fastGpuFit || cpuOrOffloadFit || unifiedMemoryFit;
|
||||
|
||||
if (recommended) {
|
||||
const mode = fastGpuFit ? "GPU memory" : isAppleSilicon ? "unified memory" : "system RAM with CPU/GPU offload";
|
||||
const mode = fastGpuFit
|
||||
? "GPU memory"
|
||||
: isAppleSilicon
|
||||
? "unified memory"
|
||||
: hybridFit
|
||||
? "combined GPU and system memory"
|
||||
: "system RAM";
|
||||
return {
|
||||
recommended: true,
|
||||
mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : "ram",
|
||||
label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : "Fits in RAM",
|
||||
mode: fastGpuFit ? "gpu" : isAppleSilicon ? "unified" : hybridFit ? "hybrid" : "ram",
|
||||
label: fastGpuFit ? "GPU Fit" : isAppleSilicon ? "Fits Unified Memory" : hybridFit ? "GPU + RAM Fit" : "Fits in RAM",
|
||||
reason: `${formatBytes(sizeBytes)} weights fit the estimated ${mode} budget with runtime headroom.`,
|
||||
};
|
||||
}
|
||||
@@ -2029,6 +2076,8 @@ async function startLlm(settings = {}) {
|
||||
if (mmprojPath) {
|
||||
args.push("--mmproj", mmprojPath);
|
||||
}
|
||||
llmSettings.supportsVision = Boolean(mmprojPath);
|
||||
llmSettings.mmproj = mmprojPath ? path.basename(mmprojPath) : null;
|
||||
const spawnEnv = { ...process.env };
|
||||
const backendDir = path.dirname(backend.path);
|
||||
if (osPlatform === "linux") {
|
||||
|
||||
Reference in new issue
Block a user