feat: workspaces, agentic action tools, local Whisper STT, vec recall

- Projects/workspaces: documents grouped into projects; chat RAG scopes to the
  active project. Switcher in the Documents page.
- Agentic action tools: web_search, fetch_url, and remember (first write tool),
  allowlist-gated per playbook.
- Local Whisper STT (faster-whisper, no torch): on-device dictation replacing
  the browser Web Speech API. POST /stt + GET /stt/status; browser fallback.
- Vector index extended to conversation recall (message_vectors), with the
  brute-force cosine scan kept as the fallback.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jon
2026-07-23 14:22:46 -05:00
co-authored by Claude Opus 4.8
parent f4aea78b55
commit ba6a4ac4e4
12 changed files with 586 additions and 53 deletions
+64 -11
View File
@@ -19,23 +19,64 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
const [editingIdx, setEditingIdx] = useState(null); // user message being edited
const [editText, setEditText] = useState("");
const [listening, setListening] = useState(false); // mic dictation active
const [transcribing, setTranscribing] = useState(false); // local STT running
const [sttLocal, setSttLocal] = useState(false); // backend Whisper available
const [speakingIdx, setSpeakingIdx] = useState(null); // message being read aloud
const recognitionRef = useRef(null);
const mediaRecRef = useRef(null);
// Web Speech API — browser-native, no backend/model. Absent on unsupported browsers.
const SpeechRec = typeof window !== "undefined" && (window.SpeechRecognition || window.webkitSpeechRecognition);
const ttsSupported = typeof window !== "undefined" && "speechSynthesis" in window;
const canRecord = typeof navigator !== "undefined" && navigator.mediaDevices && window.MediaRecorder;
const toggleMic = () => {
if (!SpeechRec) return;
if (listening) { recognitionRef.current?.stop(); return; }
// Prefer local Whisper (on-device) over browser speech (Chrome routes audio to Google).
useEffect(() => {
fetch(`${API_BASE}/stt/status`).then(r => r.ok ? r.json() : null)
.then(d => setSttLocal(Boolean(d && d.available))).catch(() => {});
}, []);
const _appendTranscript = (text) => {
if (text) setInput(prev => (prev ? prev + " " : "") + text);
};
// Local path: record audio, POST to /stt (faster-whisper transcribes on-device).
const startLocalDictation = async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const rec = new MediaRecorder(stream);
const chunks = [];
rec.ondataavailable = e => e.data.size && chunks.push(e.data);
rec.onstop = async () => {
stream.getTracks().forEach(t => t.stop());
setListening(false);
setTranscribing(true);
try {
const b64 = await new Promise((res) => {
const fr = new FileReader();
fr.onload = () => res(String(fr.result).split(",")[1]);
fr.readAsDataURL(new Blob(chunks, { type: rec.mimeType }));
});
const r = await fetch(`${API_BASE}/stt`, {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ audio: b64 }),
});
if (r.ok) _appendTranscript((await r.json()).text);
} catch { /* ignore */ } finally {
setTranscribing(false);
}
};
mediaRecRef.current = rec;
setListening(true);
rec.start();
} catch { setListening(false); }
};
const startBrowserDictation = () => {
const rec = new SpeechRec();
rec.lang = "en-US";
rec.interimResults = false;
rec.onresult = (e) => {
const text = Array.from(e.results).map(r => r[0].transcript).join(" ").trim();
if (text) setInput(prev => (prev ? prev + " " : "") + text);
};
rec.onresult = (e) => _appendTranscript(Array.from(e.results).map(r => r[0].transcript).join(" ").trim());
rec.onend = () => setListening(false);
rec.onerror = () => setListening(false);
recognitionRef.current = rec;
@@ -43,6 +84,17 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
rec.start();
};
const toggleMic = () => {
if (transcribing) return;
if (listening) {
if (sttLocal && mediaRecRef.current) mediaRecRef.current.stop();
else recognitionRef.current?.stop();
return;
}
if (sttLocal && canRecord) startLocalDictation();
else if (SpeechRec) startBrowserDictation();
};
// Read a reply aloud (light markdown strip so symbols aren't spoken).
const speak = (idx, text) => {
if (!ttsSupported) return;
@@ -617,10 +669,11 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
📎
<input type="file" accept="image/*" multiple onChange={onImagePick} disabled={loading} style={{ display: "none" }} />
</label>
{SpeechRec && (
<button onClick={toggleMic} title={listening ? "Stop dictation" : "Dictate (speech to text)"}
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
{listening ? "🔴" : "🎤"}
{((sttLocal && canRecord) || SpeechRec) && (
<button onClick={toggleMic}
title={transcribing ? "Transcribing…" : listening ? "Stop dictation" : sttLocal ? "Dictate (on-device Whisper)" : "Dictate (browser speech)"}
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: transcribing ? "default" : "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
{transcribing ? "⏳" : listening ? "🔴" : "🎤"}
</button>
)}
<textarea