feat: workspaces, agentic action tools, local Whisper STT, vec recall
- Projects/workspaces: documents grouped into projects; chat RAG scopes to the active project. Switcher in the Documents page. - Agentic action tools: web_search, fetch_url, and remember (first write tool), allowlist-gated per playbook. - Local Whisper STT (faster-whisper, no torch): on-device dictation replacing the browser Web Speech API. POST /stt + GET /stt/status; browser fallback. - Vector index extended to conversation recall (message_vectors), with the brute-force cosine scan kept as the fallback. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -19,23 +19,64 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
|
||||
const [editingIdx, setEditingIdx] = useState(null); // user message being edited
|
||||
const [editText, setEditText] = useState("");
|
||||
const [listening, setListening] = useState(false); // mic dictation active
|
||||
const [transcribing, setTranscribing] = useState(false); // local STT running
|
||||
const [sttLocal, setSttLocal] = useState(false); // backend Whisper available
|
||||
const [speakingIdx, setSpeakingIdx] = useState(null); // message being read aloud
|
||||
const recognitionRef = useRef(null);
|
||||
const mediaRecRef = useRef(null);
|
||||
|
||||
// Web Speech API — browser-native, no backend/model. Absent on unsupported browsers.
|
||||
const SpeechRec = typeof window !== "undefined" && (window.SpeechRecognition || window.webkitSpeechRecognition);
|
||||
const ttsSupported = typeof window !== "undefined" && "speechSynthesis" in window;
|
||||
const canRecord = typeof navigator !== "undefined" && navigator.mediaDevices && window.MediaRecorder;
|
||||
|
||||
const toggleMic = () => {
|
||||
if (!SpeechRec) return;
|
||||
if (listening) { recognitionRef.current?.stop(); return; }
|
||||
// Prefer local Whisper (on-device) over browser speech (Chrome routes audio to Google).
|
||||
useEffect(() => {
|
||||
fetch(`${API_BASE}/stt/status`).then(r => r.ok ? r.json() : null)
|
||||
.then(d => setSttLocal(Boolean(d && d.available))).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const _appendTranscript = (text) => {
|
||||
if (text) setInput(prev => (prev ? prev + " " : "") + text);
|
||||
};
|
||||
|
||||
// Local path: record audio, POST to /stt (faster-whisper transcribes on-device).
|
||||
const startLocalDictation = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
const rec = new MediaRecorder(stream);
|
||||
const chunks = [];
|
||||
rec.ondataavailable = e => e.data.size && chunks.push(e.data);
|
||||
rec.onstop = async () => {
|
||||
stream.getTracks().forEach(t => t.stop());
|
||||
setListening(false);
|
||||
setTranscribing(true);
|
||||
try {
|
||||
const b64 = await new Promise((res) => {
|
||||
const fr = new FileReader();
|
||||
fr.onload = () => res(String(fr.result).split(",")[1]);
|
||||
fr.readAsDataURL(new Blob(chunks, { type: rec.mimeType }));
|
||||
});
|
||||
const r = await fetch(`${API_BASE}/stt`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ audio: b64 }),
|
||||
});
|
||||
if (r.ok) _appendTranscript((await r.json()).text);
|
||||
} catch { /* ignore */ } finally {
|
||||
setTranscribing(false);
|
||||
}
|
||||
};
|
||||
mediaRecRef.current = rec;
|
||||
setListening(true);
|
||||
rec.start();
|
||||
} catch { setListening(false); }
|
||||
};
|
||||
|
||||
const startBrowserDictation = () => {
|
||||
const rec = new SpeechRec();
|
||||
rec.lang = "en-US";
|
||||
rec.interimResults = false;
|
||||
rec.onresult = (e) => {
|
||||
const text = Array.from(e.results).map(r => r[0].transcript).join(" ").trim();
|
||||
if (text) setInput(prev => (prev ? prev + " " : "") + text);
|
||||
};
|
||||
rec.onresult = (e) => _appendTranscript(Array.from(e.results).map(r => r[0].transcript).join(" ").trim());
|
||||
rec.onend = () => setListening(false);
|
||||
rec.onerror = () => setListening(false);
|
||||
recognitionRef.current = rec;
|
||||
@@ -43,6 +84,17 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
|
||||
rec.start();
|
||||
};
|
||||
|
||||
const toggleMic = () => {
|
||||
if (transcribing) return;
|
||||
if (listening) {
|
||||
if (sttLocal && mediaRecRef.current) mediaRecRef.current.stop();
|
||||
else recognitionRef.current?.stop();
|
||||
return;
|
||||
}
|
||||
if (sttLocal && canRecord) startLocalDictation();
|
||||
else if (SpeechRec) startBrowserDictation();
|
||||
};
|
||||
|
||||
// Read a reply aloud (light markdown strip so symbols aren't spoken).
|
||||
const speak = (idx, text) => {
|
||||
if (!ttsSupported) return;
|
||||
@@ -617,10 +669,11 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
|
||||
📎
|
||||
<input type="file" accept="image/*" multiple onChange={onImagePick} disabled={loading} style={{ display: "none" }} />
|
||||
</label>
|
||||
{SpeechRec && (
|
||||
<button onClick={toggleMic} title={listening ? "Stop dictation" : "Dictate (speech to text)"}
|
||||
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
|
||||
{listening ? "🔴" : "🎤"}
|
||||
{((sttLocal && canRecord) || SpeechRec) && (
|
||||
<button onClick={toggleMic}
|
||||
title={transcribing ? "Transcribing…" : listening ? "Stop dictation" : sttLocal ? "Dictate (on-device Whisper)" : "Dictate (browser speech)"}
|
||||
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: transcribing ? "default" : "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
|
||||
{transcribing ? "⏳" : listening ? "🔴" : "🎤"}
|
||||
</button>
|
||||
)}
|
||||
<textarea
|
||||
|
||||
@@ -10,6 +10,43 @@ export function Documents() {
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [message, setMessage] = useState("");
|
||||
const [viewing, setViewing] = useState(null); // {title, chunks} being previewed
|
||||
const [projects, setProjects] = useState([]);
|
||||
const [activeProject, setActiveProject] = useState(""); // "" = All
|
||||
|
||||
const loadProjects = async () => {
|
||||
try {
|
||||
const r = await fetch(`${API_BASE}/projects`);
|
||||
if (r.ok) { const d = await r.json(); setProjects(d.projects || []); setActiveProject(d.active || ""); }
|
||||
} catch { /* offline */ }
|
||||
};
|
||||
|
||||
// Switch the active workspace (persisted in settings; scopes chat RAG too).
|
||||
const setActive = async (pid) => {
|
||||
await fetch(`${API_BASE}/settings`, {
|
||||
method: "PUT", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ active_project: pid }),
|
||||
});
|
||||
setActiveProject(pid);
|
||||
load();
|
||||
loadProjects();
|
||||
};
|
||||
|
||||
const newProject = async () => {
|
||||
const name = window.prompt("New project name:");
|
||||
if (!name || !name.trim()) return;
|
||||
const r = await fetch(`${API_BASE}/projects`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ name: name.trim() }),
|
||||
});
|
||||
if (r.ok) { const p = await r.json(); await setActive(p.id); }
|
||||
};
|
||||
|
||||
const removeProject = async () => {
|
||||
if (!activeProject) return;
|
||||
if (!window.confirm("Delete this project? Its documents are kept but become unscoped.")) return;
|
||||
await fetch(`${API_BASE}/projects/${encodeURIComponent(activeProject)}`, { method: "DELETE" });
|
||||
await setActive("");
|
||||
};
|
||||
|
||||
const openDoc = async (doc) => {
|
||||
try {
|
||||
@@ -25,7 +62,7 @@ export function Documents() {
|
||||
} catch { /* offline — leave list as-is */ }
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, []);
|
||||
useEffect(() => { load(); loadProjects(); }, []);
|
||||
|
||||
// Files (pdf/docx/txt/md) upload straight to the server, which extracts the
|
||||
// text. Base64 in JSON — no multipart dependency.
|
||||
@@ -106,6 +143,30 @@ export function Documents() {
|
||||
material when a message is relevant.
|
||||
</p>
|
||||
|
||||
<div style={{ display: "flex", alignItems: "center", gap: "0.6rem", marginBottom: "1.2rem", flexWrap: "wrap" }}>
|
||||
<span style={{ color: "#888", fontSize: "0.85rem" }}>Workspace:</span>
|
||||
<select
|
||||
value={activeProject}
|
||||
onChange={e => (e.target.value === "__new__" ? newProject() : setActive(e.target.value))}
|
||||
style={{ padding: "0.5rem 0.7rem", background: "#222", color: "#eee", border: "1px solid #333", borderRadius: "8px" }}
|
||||
>
|
||||
<option value="">All documents</option>
|
||||
{projects.map(p => (
|
||||
<option key={p.id} value={p.id}>{p.name} ({p.docs})</option>
|
||||
))}
|
||||
<option value="__new__">+ New project…</option>
|
||||
</select>
|
||||
{activeProject && (
|
||||
<button onClick={removeProject} title="Delete this project (documents kept)"
|
||||
style={{ padding: "0.4rem 0.7rem", background: "#2a1a1a", color: "#ff8a80", border: "1px solid #5a2a2a", borderRadius: "8px", cursor: "pointer", fontSize: "0.8rem" }}>
|
||||
Delete project
|
||||
</button>
|
||||
)}
|
||||
<span style={{ color: "#666", fontSize: "0.78rem" }}>
|
||||
{activeProject ? "Chat scopes to this project's documents." : "Chat searches all documents."}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: "0.7rem", marginBottom: "1.5rem" }}>
|
||||
<input type="text" placeholder="Title" value={title}
|
||||
onChange={e => setTitle(e.target.value)} style={input} />
|
||||
|
||||
@@ -360,7 +360,7 @@ export function Playbook() {
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Tools (comma separated): search_memory, search_history, search_documents, list_models, get_time"
|
||||
placeholder="Tools: search_memory, search_history, search_documents, list_models, get_time, web_search, fetch_url, remember"
|
||||
value={form.tools}
|
||||
onChange={e => setForm(prev => ({ ...prev, tools: e.target.value }))}
|
||||
style={{ padding: "0.9rem", background: "#222", color: "#eee", border: "1px solid #333", borderRadius: "10px" }}
|
||||
|
||||
Reference in New Issue
Block a user