diff --git a/app/frontend/src/components/ModelManager.jsx b/app/frontend/src/components/ModelManager.jsx index 0719bac..3eadc55 100644 --- a/app/frontend/src/components/ModelManager.jsx +++ b/app/frontend/src/components/ModelManager.jsx @@ -456,7 +456,7 @@ function ModelManager({ if (activeModelType === "image") { res = await downloadModel(url); } else { - res = await downloadLlmModel(url); + res = await downloadLlmModel(url, expectedFilename); } if (res && res.ok) { startProgressPolling(expectedFilename); @@ -1241,13 +1241,17 @@ function ModelManager({ )} - {model.format} • approx. {model.approxSize} {model.resolution !== "N/A" && `• ${model.resolution}`} + {model.format} {model.resolution && model.resolution !== "N/A" && `• ${model.resolution}`} - {model.size && model.size !== "Unknown" && ( + {model.size && model.size !== "Unknown" ? ( File size: {model.size} - )} + ) : model.approxSize ? ( + + File size: approx. {model.approxSize} + + ) : null}

{model.notes} diff --git a/app/frontend/src/components/TextChat.jsx b/app/frontend/src/components/TextChat.jsx index 2627105..59e53f8 100644 --- a/app/frontend/src/components/TextChat.jsx +++ b/app/frontend/src/components/TextChat.jsx @@ -47,6 +47,9 @@ function TextChat({ const bottomRef = useRef(null); const completedDownloadRef = useRef(""); const loadingModelRef = useRef(null); + const chatMessagesRef = useRef(null); + const prevMessagesLengthRef = useRef(0); + const abortControllerRef = useRef(null); const [attachments, setAttachments] = useState([]); const fileInputRef = useRef(null); @@ -181,8 +184,33 @@ function TextChat({ return () => clearInterval(timer); }, [refresh]); + const handleStopGeneration = () => { + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + abortControllerRef.current = null; + } + }; + useEffect(() => { - bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + const container = chatMessagesRef.current; + if (!container) return; + + const len = messages.length; + const prevLen = prevMessagesLengthRef.current; + prevMessagesLengthRef.current = len; + + const lastMessage = messages[len - 1]; + const isNewUserMessage = len > prevLen && lastMessage?.role === "user"; + + if (isNewUserMessage) { + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + } else { + const threshold = 150; + const isNearBottom = container.scrollHeight - container.scrollTop - container.clientHeight <= threshold; + if (isNearBottom) { + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + } + } }, [messages, isBusy, loadingModel]); const handleModelChange = async (filename) => { @@ -300,6 +328,9 @@ function TextChat({ generationStats: { status: "starting", tokens: 0, tokensPerSecond: 0, seconds: 0 }, }]); + const controller = new AbortController(); + abortControllerRef.current = controller; + try { const systemPrompt = textSettings?.systemPrompt || "You are a helpful local AI assistant."; const requestMessages = [ @@ -309,14 +340,21 @@ function TextChat({ let assistantText = ""; let streamedTokens = 0; + let firstTokenAt = null; const response = await streamChatWithLlm(requestMessages, { temperature: textSettings?.temperature || 0.7, maxTokens: 768, + signal: controller.signal, }, (_token, fullText) => { const now = performance.now(); + if (streamedTokens === 0) { + firstTokenAt = now; + } streamedTokens += 1; - const generationSeconds = Math.max(0.05, (now - requestStartedAt) / 1000); + const generationSeconds = firstTokenAt + ? Math.max(0.05, (now - firstTokenAt) / 1000) + : Math.max(0.05, (now - requestStartedAt) / 1000); assistantText = fullText; setMessages((prev) => { const updated = [...prev]; @@ -362,11 +400,30 @@ function TextChat({ // Clean up attached files on success setAttachments([]); } catch (err) { - const finalMessages = [...nextMessages, { role: "assistant", content: `Error: ${err.message}`, error: true }]; - setMessages(finalMessages); - saveConversationState(convId, finalMessages, selectedModel); + if (err.name === "AbortError") { + setMessages((prev) => { + const updated = [...prev]; + if (updated.length > 0 && updated[updated.length - 1].role === "assistant") { + const lastMsg = updated[updated.length - 1]; + updated[updated.length - 1] = { + ...lastMsg, + generationStats: lastMsg.generationStats ? { + ...lastMsg.generationStats, + status: "complete", + } : null + }; + saveConversationState(convId, updated, selectedModel); + } + return updated; + }); + } else { + const finalMessages = [...nextMessages, { role: "assistant", content: `Error: ${err.message}`, error: true }]; + setMessages(finalMessages); + saveConversationState(convId, finalMessages, selectedModel); + } } finally { setIsBusy(false); + abortControllerRef.current = null; } }; @@ -520,7 +577,7 @@ function TextChat({ -

+
{loadingModel ? (
@@ -575,7 +632,7 @@ function TextChat({
{message.content.map((item, idx) => { if (item.type === "text") { - return {item.text}; + return ; } if (item.type === "image_url") { return ( @@ -591,7 +648,7 @@ function TextChat({ })}
) : ( - {message.content} + )}
{message.role === "assistant" && message.generationStats && !message.error && ( @@ -700,14 +757,24 @@ function TextChat({ disabled={!status.ready || isBusy} style={{ flex: 1 }} /> - + {isBusy && status.ready ? ( + + ) : ( + + )}
@@ -715,4 +782,159 @@ function TextChat({ ); } +function parseInlineMarkdown(text) { + const regex = /(\*\*.*?\*\*|`.*?`|\*.*?\*|\[.*?\]\(.*?\))/g; + const parts = text.split(regex); + + return parts.map((part, idx) => { + if (part.startsWith("**") && part.endsWith("**")) { + return {part.slice(2, -2)}; + } + if (part.startsWith("`") && part.endsWith("`")) { + return {part.slice(1, -1)}; + } + if (part.startsWith("*") && part.endsWith("*")) { + return {part.slice(1, -1)}; + } + if (part.startsWith("[") && part.includes("](") && part.endsWith(")")) { + const match = part.match(/\[(.*?)\]\((.*?)\)/); + if (match) { + return ( + + {match[1]} + + ); + } + } + return part; + }); +} + +export function MarkdownRenderer({ content }) { + if (typeof content !== 'string') return null; + + const parts = content.split(/(```[\s\S]*?```)/g); + + return ( +
+ {parts.map((part, index) => { + if (part.startsWith("```") && part.endsWith("```")) { + const match = part.match(/```(\w*)\n([\s\S]*?)```/); + const lang = match ? match[1] : ""; + const code = match ? match[2] : part.slice(3, -3); + return ( +
+              {lang && 
{lang}
} + {code.trim()} +
+ ); + } else { + const rawLines = part.split(/\r?\n/); + const blocks = []; + let currentBlock = null; + + for (let i = 0; i < rawLines.length; i++) { + const line = rawLines[i]; + const trimmed = line.trim(); + + if (trimmed.startsWith("### ")) { + if (currentBlock) { blocks.push(currentBlock); currentBlock = null; } + blocks.push({ type: "h4", content: trimmed.slice(4) }); + } else if (trimmed.startsWith("## ")) { + if (currentBlock) { blocks.push(currentBlock); currentBlock = null; } + blocks.push({ type: "h3", content: trimmed.slice(3) }); + } else if (trimmed.startsWith("# ")) { + if (currentBlock) { blocks.push(currentBlock); currentBlock = null; } + blocks.push({ type: "h2", content: trimmed.slice(2) }); + } else if (trimmed.startsWith("* ") || trimmed.startsWith("- ")) { + const itemContent = trimmed.slice(2); + if (currentBlock && currentBlock.type === "ul") { + currentBlock.items.push(itemContent); + } else { + if (currentBlock) { blocks.push(currentBlock); } + currentBlock = { type: "ul", items: [itemContent] }; + } + } else { + const numMatch = trimmed.match(/^(\d+)\.\s+(.*)/); + if (numMatch) { + const itemContent = numMatch[2]; + if (currentBlock && currentBlock.type === "ol") { + currentBlock.items.push(itemContent); + } else { + if (currentBlock) { blocks.push(currentBlock); } + currentBlock = { type: "ol", items: [itemContent] }; + } + } else if (trimmed === "") { + if (currentBlock) { blocks.push(currentBlock); currentBlock = null; } + blocks.push({ type: "spacer" }); + } else { + if (currentBlock && currentBlock.type === "p") { + currentBlock.lines.push(line); + } else { + if (currentBlock) { blocks.push(currentBlock); } + currentBlock = { type: "p", lines: [line] }; + } + } + } + } + if (currentBlock) { + blocks.push(currentBlock); + } + + return blocks.map((block, blockIdx) => { + switch (block.type) { + case "h4": + return

{parseInlineMarkdown(block.content)}

; + case "h3": + return

{parseInlineMarkdown(block.content)}

; + case "h2": + return

{parseInlineMarkdown(block.content)}

; + case "ul": + return ( + + ); + case "ol": + return ( +
    + {block.items.map((item, itemIdx) => ( +
  1. {parseInlineMarkdown(item)}
  2. + ))} +
+ ); + case "spacer": + return
; + case "p": + return

{parseInlineMarkdown(block.lines.join(" "))}

; + default: + return null; + } + }); + } + })} +
+ ); +} + export default TextChat; diff --git a/app/frontend/src/services/api.js b/app/frontend/src/services/api.js index 9b7d463..b1f7858 100644 --- a/app/frontend/src/services/api.js +++ b/app/frontend/src/services/api.js @@ -376,7 +376,7 @@ export async function chatWithLlm(messages, options = {}) { } export async function streamChatWithLlm(messages, options = {}, onToken = () => {}) { - const res = await fetch("/api/llm/chat", { + const fetchOpts = { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -385,7 +385,11 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => max_tokens: options.maxTokens, stream: true, }), - }); + }; + if (options.signal) { + fetchOpts.signal = options.signal; + } + const res = await fetch("/api/llm/chat", fetchOpts); if (!res.ok) { const data = await readJsonResponse(res, `Chat request failed (HTTP ${res.status}).`); @@ -441,11 +445,11 @@ export async function streamChatWithLlm(messages, options = {}, onToken = () => return { content, usage, timings }; } -export async function downloadLlmModel(url) { +export async function downloadLlmModel(url, filename = null) { const res = await fetch("/api/llm/download-model", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ url }), + body: JSON.stringify({ url, filename }), }); return await readJsonResponse(res, "The local server returned an invalid text download response."); } diff --git a/scripts/serve.cjs b/scripts/serve.cjs index 6b9aa27..c5f492d 100644 --- a/scripts/serve.cjs +++ b/scripts/serve.cjs @@ -1848,7 +1848,7 @@ async function searchHuggingFaceModels(query, filters) { models.push({ id: model.id, name: String(model.id || "").split("/").pop().replace(/[-_]+/g, " "), - filename: path.basename(filename), + filename: `${model.id.replace(/\//g, "--")}--${path.basename(filename)}`, repositoryFilename: filename, format: "GGUF", approxSize: traits.parameters ? `~${traits.parameters}B parameters` : "Size shown before download", @@ -2202,6 +2202,9 @@ async function startLlm(settings = {}) { "--ctx-size", String(llmSettings.contextSize), "--threads", String(llmSettings.threads), "--n-gpu-layers", String(llmSettings.gpuLayers), + "--parallel", "1", + "--cache-ram", "0", + "--no-warmup", ]; if (mmprojPath) { args.push("--mmproj", mmprojPath); @@ -3604,7 +3607,8 @@ async function doLlmChat(req, res, body, retryCount = 0) { if (downloadState.active) return json(res, 409, { ok: false, error: "Another model download is already active." }); try { const directUrl = url.includes("huggingface.co") ? url.replace("/blob/", "/resolve/") : url; - const filename = path.basename(new URL(directUrl).pathname); + const overrideFilename = body.filename ? path.basename(String(body.filename)) : null; + const filename = overrideFilename || path.basename(new URL(directUrl).pathname); if (!filename.toLowerCase().endsWith(".gguf")) { return json(res, 400, { ok: false, error: "Text model URL must point to a .gguf file." }); }