feat: workspaces, agentic action tools, local Whisper STT, vec recall
- Projects/workspaces: documents grouped into projects; chat RAG scopes to the active project. Switcher in the Documents page. - Agentic action tools: web_search, fetch_url, and remember (first write tool), allowlist-gated per playbook. - Local Whisper STT (faster-whisper, no torch): on-device dictation replacing the browser Web Speech API. POST /stt + GET /stt/status; browser fallback. - Vector index extended to conversation recall (message_vectors), with the brute-force cosine scan kept as the fallback. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -19,23 +19,64 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
|
||||
const [editingIdx, setEditingIdx] = useState(null); // user message being edited
|
||||
const [editText, setEditText] = useState("");
|
||||
const [listening, setListening] = useState(false); // mic dictation active
|
||||
const [transcribing, setTranscribing] = useState(false); // local STT running
|
||||
const [sttLocal, setSttLocal] = useState(false); // backend Whisper available
|
||||
const [speakingIdx, setSpeakingIdx] = useState(null); // message being read aloud
|
||||
const recognitionRef = useRef(null);
|
||||
const mediaRecRef = useRef(null);
|
||||
|
||||
// Web Speech API — browser-native, no backend/model. Absent on unsupported browsers.
|
||||
const SpeechRec = typeof window !== "undefined" && (window.SpeechRecognition || window.webkitSpeechRecognition);
|
||||
const ttsSupported = typeof window !== "undefined" && "speechSynthesis" in window;
|
||||
const canRecord = typeof navigator !== "undefined" && navigator.mediaDevices && window.MediaRecorder;
|
||||
|
||||
const toggleMic = () => {
|
||||
if (!SpeechRec) return;
|
||||
if (listening) { recognitionRef.current?.stop(); return; }
|
||||
// Prefer local Whisper (on-device) over browser speech (Chrome routes audio to Google).
|
||||
useEffect(() => {
|
||||
fetch(`${API_BASE}/stt/status`).then(r => r.ok ? r.json() : null)
|
||||
.then(d => setSttLocal(Boolean(d && d.available))).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const _appendTranscript = (text) => {
|
||||
if (text) setInput(prev => (prev ? prev + " " : "") + text);
|
||||
};
|
||||
|
||||
// Local path: record audio, POST to /stt (faster-whisper transcribes on-device).
|
||||
const startLocalDictation = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
const rec = new MediaRecorder(stream);
|
||||
const chunks = [];
|
||||
rec.ondataavailable = e => e.data.size && chunks.push(e.data);
|
||||
rec.onstop = async () => {
|
||||
stream.getTracks().forEach(t => t.stop());
|
||||
setListening(false);
|
||||
setTranscribing(true);
|
||||
try {
|
||||
const b64 = await new Promise((res) => {
|
||||
const fr = new FileReader();
|
||||
fr.onload = () => res(String(fr.result).split(",")[1]);
|
||||
fr.readAsDataURL(new Blob(chunks, { type: rec.mimeType }));
|
||||
});
|
||||
const r = await fetch(`${API_BASE}/stt`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ audio: b64 }),
|
||||
});
|
||||
if (r.ok) _appendTranscript((await r.json()).text);
|
||||
} catch { /* ignore */ } finally {
|
||||
setTranscribing(false);
|
||||
}
|
||||
};
|
||||
mediaRecRef.current = rec;
|
||||
setListening(true);
|
||||
rec.start();
|
||||
} catch { setListening(false); }
|
||||
};
|
||||
|
||||
const startBrowserDictation = () => {
|
||||
const rec = new SpeechRec();
|
||||
rec.lang = "en-US";
|
||||
rec.interimResults = false;
|
||||
rec.onresult = (e) => {
|
||||
const text = Array.from(e.results).map(r => r[0].transcript).join(" ").trim();
|
||||
if (text) setInput(prev => (prev ? prev + " " : "") + text);
|
||||
};
|
||||
rec.onresult = (e) => _appendTranscript(Array.from(e.results).map(r => r[0].transcript).join(" ").trim());
|
||||
rec.onend = () => setListening(false);
|
||||
rec.onerror = () => setListening(false);
|
||||
recognitionRef.current = rec;
|
||||
@@ -43,6 +84,17 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
|
||||
rec.start();
|
||||
};
|
||||
|
||||
const toggleMic = () => {
|
||||
if (transcribing) return;
|
||||
if (listening) {
|
||||
if (sttLocal && mediaRecRef.current) mediaRecRef.current.stop();
|
||||
else recognitionRef.current?.stop();
|
||||
return;
|
||||
}
|
||||
if (sttLocal && canRecord) startLocalDictation();
|
||||
else if (SpeechRec) startBrowserDictation();
|
||||
};
|
||||
|
||||
// Read a reply aloud (light markdown strip so symbols aren't spoken).
|
||||
const speak = (idx, text) => {
|
||||
if (!ttsSupported) return;
|
||||
@@ -617,10 +669,11 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
|
||||
📎
|
||||
<input type="file" accept="image/*" multiple onChange={onImagePick} disabled={loading} style={{ display: "none" }} />
|
||||
</label>
|
||||
{SpeechRec && (
|
||||
<button onClick={toggleMic} title={listening ? "Stop dictation" : "Dictate (speech to text)"}
|
||||
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
|
||||
{listening ? "🔴" : "🎤"}
|
||||
{((sttLocal && canRecord) || SpeechRec) && (
|
||||
<button onClick={toggleMic}
|
||||
title={transcribing ? "Transcribing…" : listening ? "Stop dictation" : sttLocal ? "Dictate (on-device Whisper)" : "Dictate (browser speech)"}
|
||||
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: transcribing ? "default" : "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
|
||||
{transcribing ? "⏳" : listening ? "🔴" : "🎤"}
|
||||
</button>
|
||||
)}
|
||||
<textarea
|
||||
|
||||
Reference in New Issue
Block a user