diff --git a/app/frontend/src/components/Sidebar.jsx b/app/frontend/src/components/Sidebar.jsx
index db86544..4b27604 100644
--- a/app/frontend/src/components/Sidebar.jsx
+++ b/app/frontend/src/components/Sidebar.jsx
@@ -1,5 +1,11 @@
import React, { memo } from "react";
-import { Image, FolderDown, MessageSquare, Settings, Shield, Terminal, ChevronDown, ChevronUp, Trash2 } from "lucide-react";
+import { Image, FolderDown, MessageSquare, Mic, Settings, Shield, Terminal, ChevronDown, ChevronUp, Trash2 } from "lucide-react";
+
+function formatSidebarDate(value) {
+ const date = new Date(value);
+ if (Number.isNaN(date.getTime())) return "";
+ return date.toLocaleDateString();
+}
function Sidebar({
activeTab,
@@ -10,7 +16,12 @@ function Sidebar({
setActiveConversationId,
showHistory,
setShowHistory,
- onDeleteConversation
+ onDeleteConversation,
+ speechTranscriptions = [],
+ selectedSpeechTranscript,
+ setSelectedSpeechTranscript,
+ showSpeechHistory,
+ setShowSpeechHistory
}) {
return (
setActiveTab("models")}
diff --git a/app/frontend/src/components/SpeechTranscriber.jsx b/app/frontend/src/components/SpeechTranscriber.jsx
new file mode 100644
index 0000000..d72aee3
--- /dev/null
+++ b/app/frontend/src/components/SpeechTranscriber.jsx
@@ -0,0 +1,478 @@
+import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import {
+ FileAudio,
+ LoaderCircle,
+ Mic,
+ Play,
+ Save,
+ Square,
+ Upload,
+ X,
+} from "lucide-react";
+import {
+ getSpeechStatus,
+ listSpeechModels,
+ startSpeech,
+ stopSpeech,
+ transcribeSpeech,
+} from "../services/api";
+
+const LANGUAGE_OPTIONS = [
+ { value: "auto", label: "Auto detect" },
+ { value: "en", label: "English" },
+ { value: "es", label: "Spanish" },
+ { value: "fr", label: "French" },
+ { value: "de", label: "German" },
+ { value: "it", label: "Italian" },
+ { value: "pt", label: "Portuguese" },
+ { value: "hi", label: "Hindi" },
+ { value: "ja", label: "Japanese" },
+ { value: "ko", label: "Korean" },
+ { value: "zh", label: "Chinese" },
+];
+
+function flattenFloat32(chunks) {
+ const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+ const merged = new Float32Array(total);
+ let offset = 0;
+ chunks.forEach((chunk) => {
+ merged.set(chunk, offset);
+ offset += chunk.length;
+ });
+ return merged;
+}
+
+function resampleLinear(input, sourceRate, targetRate) {
+ if (sourceRate === targetRate) return input;
+ const ratio = sourceRate / targetRate;
+ const newLength = Math.max(1, Math.round(input.length / ratio));
+ const output = new Float32Array(newLength);
+ for (let i = 0; i < newLength; i += 1) {
+ const src = i * ratio;
+ const left = Math.floor(src);
+ const right = Math.min(input.length - 1, left + 1);
+ const weight = src - left;
+ output[i] = input[left] * (1 - weight) + input[right] * weight;
+ }
+ return output;
+}
+
+function encodeWavPcm16(samples, sampleRate = 16000) {
+ const bytesPerSample = 2;
+ const buffer = new ArrayBuffer(44 + samples.length * bytesPerSample);
+ const view = new DataView(buffer);
+
+ const writeString = (offset, value) => {
+ for (let i = 0; i < value.length; i += 1) {
+ view.setUint8(offset + i, value.charCodeAt(i));
+ }
+ };
+
+ writeString(0, "RIFF");
+ view.setUint32(4, 36 + samples.length * bytesPerSample, true);
+ writeString(8, "WAVE");
+ writeString(12, "fmt ");
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, 1, true);
+ view.setUint32(24, sampleRate, true);
+ view.setUint32(28, sampleRate * bytesPerSample, true);
+ view.setUint16(32, bytesPerSample, true);
+ view.setUint16(34, 16, true);
+ writeString(36, "data");
+ view.setUint32(40, samples.length * bytesPerSample, true);
+
+ let offset = 44;
+ for (let i = 0; i < samples.length; i += 1, offset += 2) {
+ const sample = Math.max(-1, Math.min(1, samples[i]));
+ view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+ }
+ return new Blob([view], { type: "audio/wav" });
+}
+
+function downloadBlob(filename, content, type) {
+ const blob = content instanceof Blob ? content : new Blob([content], { type });
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement("a");
+ link.href = url;
+ link.download = filename;
+ document.body.appendChild(link);
+ link.click();
+ link.remove();
+ URL.revokeObjectURL(url);
+}
+
+function normalizeTimestamp(value) {
+ if (!value) return "";
+ if (typeof value === "string") {
+ const from = value.match(/from=([^;},]+)/)?.[1];
+ const to = value.match(/to=([^;},]+)/)?.[1];
+ return from && to ? `${from} - ${to}` : value;
+ }
+ if (typeof value === "object" && value.from && value.to) {
+ return `${value.from} - ${value.to}`;
+ }
+ return "";
+}
+
+function formatTranscriptWithTimes(transcription) {
+ const segments = transcription?.raw?.transcription;
+ if (!Array.isArray(segments) || segments.length === 0) {
+ return transcription?.text || "";
+ }
+ return segments
+ .map((segment) => {
+ const time = normalizeTimestamp(segment.timestamps);
+ const text = String(segment.text || "").trim();
+ return time ? `[${time}] ${text}` : text;
+ })
+ .filter(Boolean)
+ .join("\n");
+}
+
+export default function SpeechTranscriber({ showAlert, showConfirm, selectedTranscript, onTranscriptionsChanged }) {
+ const [status, setStatus] = useState({ ready: false, running: false, backendInstalled: false, settings: {} });
+ const [models, setModels] = useState([]);
+ const [selectedModel, setSelectedModel] = useState("");
+ const [language, setLanguage] = useState("auto");
+ const [threads, setThreads] = useState(Math.max(1, Math.min(8, navigator.hardwareConcurrency || 4)));
+ const [audioBlob, setAudioBlob] = useState(null);
+ const [audioName, setAudioName] = useState("");
+ const [audioUrl, setAudioUrl] = useState("");
+ const [isRecording, setIsRecording] = useState(false);
+ const [isTranscribing, setIsTranscribing] = useState(false);
+ const [result, setResult] = useState(null);
+ const [resultText, setResultText] = useState("");
+ const [isLoadingModel, setIsLoadingModel] = useState(false);
+ const recorderRef = useRef(null);
+ const abortRef = useRef(null);
+
+ const installedModels = useMemo(() => models.filter((model) => model.installed), [models]);
+
+ const refresh = useCallback(async () => {
+ const [nextStatus, nextModels] = await Promise.all([
+ getSpeechStatus(),
+ listSpeechModels(),
+ ]);
+ setStatus(nextStatus);
+ setModels(nextModels);
+ setSelectedModel((current) => {
+ const installed = nextModels.filter((model) => model.installed);
+ if (current && installed.some((model) => model.filename === current || model.id === current)) return current;
+ const active = nextStatus.settings?.model;
+ if (active && installed.some((model) => model.filename === active)) return active;
+ return installed.find((model) => model.recommended)?.filename ||
+ installed[0]?.filename ||
+ "";
+ });
+ }, []);
+
+ useEffect(() => {
+ refresh().catch((err) => showAlert?.({ title: "Speech Status Failed", message: err.message || String(err), danger: true }));
+ }, [refresh, showAlert]);
+
+ useEffect(() => {
+ if (!selectedTranscript) return;
+ setResult(selectedTranscript);
+ setResultText(formatTranscriptWithTimes(selectedTranscript));
+ }, [selectedTranscript]);
+
+ useEffect(() => {
+ return () => {
+ if (audioUrl) URL.revokeObjectURL(audioUrl);
+ abortRef.current?.abort();
+ };
+ }, [audioUrl]);
+
+ const setCurrentAudio = useCallback((blob, name) => {
+ setAudioBlob(blob);
+ setAudioName(name);
+ setResult(null);
+ setResultText("");
+ setAudioUrl((oldUrl) => {
+ if (oldUrl) URL.revokeObjectURL(oldUrl);
+ return URL.createObjectURL(blob);
+ });
+ }, []);
+
+ const clearCurrentAudio = useCallback(() => {
+ setAudioBlob(null);
+ setAudioName("");
+ setAudioUrl((oldUrl) => {
+ if (oldUrl) URL.revokeObjectURL(oldUrl);
+ return "";
+ });
+ }, []);
+
+ const handleLoadSpeechModel = async () => {
+ if (!selectedModel) return;
+ setIsLoadingModel(true);
+ try {
+ if (!status.ready || status.settings?.model !== selectedModel) {
+ await startSpeech(selectedModel, { language, threads });
+ }
+ await refresh();
+ } catch (err) {
+ showAlert?.({ title: "Speech Model Load Failed", message: err.message || String(err), danger: true });
+ } finally {
+ setIsLoadingModel(false);
+ }
+ };
+ const startRecording = async () => {
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ const audioContext = new AudioContext();
+ const source = audioContext.createMediaStreamSource(stream);
+ const processor = audioContext.createScriptProcessor(4096, 1, 1);
+ const chunks = [];
+ processor.onaudioprocess = (event) => {
+ chunks.push(new Float32Array(event.inputBuffer.getChannelData(0)));
+ };
+ source.connect(processor);
+ processor.connect(audioContext.destination);
+ recorderRef.current = { stream, audioContext, source, processor, chunks, sampleRate: audioContext.sampleRate };
+ setIsRecording(true);
+ } catch (err) {
+ showAlert?.({ title: "Microphone Failed", message: err.message || String(err), danger: true });
+ }
+ };
+
+ const stopRecording = async () => {
+ const recorder = recorderRef.current;
+ if (!recorder) return;
+ recorder.processor.disconnect();
+ recorder.source.disconnect();
+ recorder.stream.getTracks().forEach((track) => track.stop());
+ await recorder.audioContext.close();
+ recorderRef.current = null;
+ setIsRecording(false);
+ const merged = flattenFloat32(recorder.chunks);
+ const resampled = resampleLinear(merged, recorder.sampleRate, 16000);
+ const wav = encodeWavPcm16(resampled, 16000);
+ setCurrentAudio(wav, `recording-${new Date().toISOString().replace(/[:.]/g, "-")}.wav`);
+ };
+
+ const handleUploadAudio = (event) => {
+ const file = event.target.files?.[0];
+ event.target.value = "";
+ if (!file) return;
+ if (!file.name.toLowerCase().endsWith(".wav")) {
+ showAlert?.({ title: "Unsupported Audio", message: "V1 accepts WAV files only. MP3/WebM/M4A will need the later FFmpeg path.", danger: true });
+ return;
+ }
+ setCurrentAudio(file, file.name);
+ };
+
+ const handleTranscribe = async () => {
+ if (!audioBlob || !selectedModel) return;
+ setIsTranscribing(true);
+ setResult(null);
+ setResultText("");
+ const controller = new AbortController();
+ abortRef.current = controller;
+ try {
+ if (!status.ready || status.settings?.model !== selectedModel) {
+ await startSpeech(selectedModel, { language, threads });
+ }
+ const transcription = await transcribeSpeech(audioBlob, {
+ model: selectedModel,
+ language,
+ threads,
+ filename: audioName || "audio.wav",
+ signal: controller.signal,
+ });
+ setResult(transcription);
+ setResultText(formatTranscriptWithTimes(transcription));
+ await refresh();
+ await onTranscriptionsChanged?.();
+ } catch (err) {
+ if (err.name !== "AbortError") {
+ showAlert?.({ title: "Transcription Failed", message: err.message || String(err), danger: true });
+ }
+ } finally {
+ abortRef.current = null;
+ setIsTranscribing(false);
+ }
+ };
+
+ const handleCancel = () => {
+ abortRef.current?.abort();
+ setIsTranscribing(false);
+ };
+
+ const saveText = () => {
+ if (!resultText.trim()) return;
+ downloadBlob("transcription.txt", resultText, "text/plain;charset=utf-8");
+ };
+
+ const saveJson = () => {
+ const payload = { ...(result || {}), text: resultText };
+ downloadBlob("transcription.json", JSON.stringify(payload, null, 2), "application/json;charset=utf-8");
+ };
+
+ return (
+
+
+
Speech Transcriber
+
+ Record microphone audio or upload a WAV file, then transcribe locally with whisper.cpp.
+
+
+
+
+
+
+
Runtime
+
+ {status.backendInstalled ? (status.ready ? "Ready" : "Installed") : "Backend missing"}
+
+
+
+ {status.error && {status.error}
}
+
+
+ Whisper model
+ setSelectedModel(event.target.value)}>
+ {installedModels.length === 0 && No downloaded speech models }
+ {installedModels.map((model) => (
+
+ {model.name || model.filename}
+
+ ))}
+
+
+
+
+
+ Language
+ setLanguage(event.target.value)}>
+ {LANGUAGE_OPTIONS.map((item) => {item.label} )}
+
+
+
+ Threads
+
+
+
+
+
+ {isLoadingModel ? {status.ready && status.settings?.model === selectedModel ? "Loaded" : isLoadingModel ? "Loading" : "Load"}
+
+
+ Refresh
+
+
+ Download a Whisper model from Model Manager, Speech Models first.
+
+ )}
+
+
+
+
+
Audio
+ {audioName || "No audio selected"}
+
+
+
+
+
+
{audioName || "Record or upload WAV"}
+
Microphone recordings are converted to 16 kHz mono PCM16 WAV before transcription.
+
+
+ {isRecording ? (
+
+ Stop Recording
+
+ ) : (
+
+ Record
+
+ )}
+
+ Upload WAV
+
+ {audioBlob && (
+
+ Remove Audio
+
+ )}
+
+
+
+ {isTranscribing ? (
+
+ Cancel
+
+ ) : (
+
+ Transcribe
+
+ )}
+ {isTranscribing &&
+
+
+
+
+
+
Transcript
+
+
+ TXT
+
+
+ JSON
+
+ {status.ready && (
+ stopSpeech().then(refresh)}>
+ Stop Runtime
+
+ )}
+
+