enderofwings/NexusOS
1
0
Fork 2
Code Issues Pull Requests 1 Actions 13 Packages Projects Releases Wiki Activity

feat: workspaces, agentic action tools, local Whisper STT, vec recall

Browse Source 
- Projects/workspaces: documents grouped into projects; chat RAG scopes to the
  active project. Switcher in the Documents page.
- Agentic action tools: web_search, fetch_url, and remember (first write tool),
  allowlist-gated per playbook.
- Local Whisper STT (faster-whisper, no torch): on-device dictation replacing
  the browser Web Speech API. POST /stt + GET /stt/status; browser fallback.
- Vector index extended to conversation recall (message_vectors), with the
  brute-force cosine scan kept as the fallback.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jon
2026-07-23 14:22:46 -05:00
co-authored by Claude Opus 4.8
parent f4aea78b55
commit ba6a4ac4e4
12 changed files with 586 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
interface/web/src/Chatbot.jsx
+64 -11
View File
@@ -19,23 +19,64 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
const [editingIdx, setEditingIdx] = useState(null); // user message being edited
const [editText, setEditText] = useState("");
const [listening, setListening] = useState(false); // mic dictation active
const [transcribing, setTranscribing] = useState(false); // local STT running
const [sttLocal, setSttLocal] = useState(false); // backend Whisper available
const [speakingIdx, setSpeakingIdx] = useState(null); // message being read aloud
const recognitionRef = useRef(null);
const mediaRecRef = useRef(null);
// Web Speech API — browser-native, no backend/model. Absent on unsupported browsers.
const SpeechRec = typeof window !== "undefined" && (window.SpeechRecognition || window.webkitSpeechRecognition);
const ttsSupported = typeof window !== "undefined" && "speechSynthesis" in window;
const canRecord = typeof navigator !== "undefined" && navigator.mediaDevices && window.MediaRecorder;
const toggleMic = () => {
if (!SpeechRec) return;
if (listening) { recognitionRef.current?.stop(); return; }
// Prefer local Whisper (on-device) over browser speech (Chrome routes audio to Google).
useEffect(() => {
fetch(`${API_BASE}/stt/status`).then(r => r.ok ? r.json() : null)
.then(d => setSttLocal(Boolean(d && d.available))).catch(() => {});
}, []);
const _appendTranscript = (text) => {
if (text) setInput(prev => (prev ? prev + " " : "") + text);
};
// Local path: record audio, POST to /stt (faster-whisper transcribes on-device).
const startLocalDictation = async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const rec = new MediaRecorder(stream);
const chunks = [];
rec.ondataavailable = e => e.data.size && chunks.push(e.data);
rec.onstop = async () => {
stream.getTracks().forEach(t => t.stop());
setListening(false);
setTranscribing(true);
try {
const b64 = await new Promise((res) => {
const fr = new FileReader();
fr.onload = () => res(String(fr.result).split(",")[1]);
fr.readAsDataURL(new Blob(chunks, { type: rec.mimeType }));
});
const r = await fetch(`${API_BASE}/stt`, {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ audio: b64 }),
});
if (r.ok) _appendTranscript((await r.json()).text);
} catch { /* ignore */ } finally {
setTranscribing(false);
}
};
mediaRecRef.current = rec;
setListening(true);
rec.start();
} catch { setListening(false); }
};
const startBrowserDictation = () => {
const rec = new SpeechRec();
rec.lang = "en-US";
rec.interimResults = false;
rec.onresult = (e) => {
const text = Array.from(e.results).map(r => r[0].transcript).join(" ").trim();
if (text) setInput(prev => (prev ? prev + " " : "") + text);
};
rec.onresult = (e) => _appendTranscript(Array.from(e.results).map(r => r[0].transcript).join(" ").trim());
rec.onend = () => setListening(false);
rec.onerror = () => setListening(false);
recognitionRef.current = rec;
@@ -43,6 +84,17 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
rec.start();
};
const toggleMic = () => {
if (transcribing) return;
if (listening) {
if (sttLocal && mediaRecRef.current) mediaRecRef.current.stop();
else recognitionRef.current?.stop();
return;
}
if (sttLocal && canRecord) startLocalDictation();
else if (SpeechRec) startBrowserDictation();
};
// Read a reply aloud (light markdown strip so symbols aren't spoken).
const speak = (idx, text) => {
if (!ttsSupported) return;
@@ -617,10 +669,11 @@ export function Chatbot({ conversationId, setConversationId, onConversationChang
📎
<input type="file" accept="image/*" multiple onChange={onImagePick} disabled={loading} style={{ display: "none" }} />
</label>
{SpeechRec && (
<button onClick={toggleMic} title={listening ? "Stop dictation" : "Dictate (speech to text)"}
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
{listening ? "🔴" : "🎤"}
{((sttLocal && canRecord) || SpeechRec) && (
<button onClick={toggleMic}
title={transcribing ? "Transcribing…" : listening ? "Stop dictation" : sttLocal ? "Dictate (on-device Whisper)" : "Dictate (browser speech)"}
style={{ padding: "0.9rem", background: listening ? "#3a1a1a" : "#222", border: "1px solid " + (listening ? "#c0392b" : "#333"), borderRadius: "10px", cursor: transcribing ? "default" : "pointer", color: listening ? "#ff8a80" : "#ccc" }}>
{transcribing ? "⏳" : listening ? "🔴" : "🎤"}
</button>
)}
<textarea
interface/web/src/Documents.jsx
+62 -1
View File
@@ -10,6 +10,43 @@ export function Documents() {
const [busy, setBusy] = useState(false);
const [message, setMessage] = useState("");
const [viewing, setViewing] = useState(null); // {title, chunks} being previewed
const [projects, setProjects] = useState([]);
const [activeProject, setActiveProject] = useState(""); // "" = All
const loadProjects = async () => {
try {
const r = await fetch(`${API_BASE}/projects`);
if (r.ok) { const d = await r.json(); setProjects(d.projects || []); setActiveProject(d.active || ""); }
} catch { /* offline */ }
};
// Switch the active workspace (persisted in settings; scopes chat RAG too).
const setActive = async (pid) => {
await fetch(`${API_BASE}/settings`, {
method: "PUT", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ active_project: pid }),
});
setActiveProject(pid);
load();
loadProjects();
};
const newProject = async () => {
const name = window.prompt("New project name:");
if (!name || !name.trim()) return;
const r = await fetch(`${API_BASE}/projects`, {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name: name.trim() }),
});
if (r.ok) { const p = await r.json(); await setActive(p.id); }
};
const removeProject = async () => {
if (!activeProject) return;
if (!window.confirm("Delete this project? Its documents are kept but become unscoped.")) return;
await fetch(`${API_BASE}/projects/${encodeURIComponent(activeProject)}`, { method: "DELETE" });
await setActive("");
};
const openDoc = async (doc) => {
try {
@@ -25,7 +62,7 @@ export function Documents() {
} catch { /* offline — leave list as-is */ }
};
useEffect(() => { load(); }, []);
useEffect(() => { load(); loadProjects(); }, []);
// Files (pdf/docx/txt/md) upload straight to the server, which extracts the
// text. Base64 in JSON — no multipart dependency.
@@ -106,6 +143,30 @@ export function Documents() {
material when a message is relevant.
</p>
<div style={{ display: "flex", alignItems: "center", gap: "0.6rem", marginBottom: "1.2rem", flexWrap: "wrap" }}>
<span style={{ color: "#888", fontSize: "0.85rem" }}>Workspace:</span>
<select
value={activeProject}
onChange={e => (e.target.value === "__new__" ? newProject() : setActive(e.target.value))}
style={{ padding: "0.5rem 0.7rem", background: "#222", color: "#eee", border: "1px solid #333", borderRadius: "8px" }}
>
<option value="">All documents</option>
{projects.map(p => (
<option key={p.id} value={p.id}>{p.name} ({p.docs})</option>
))}
<option value="__new__">＋ New project…</option>
</select>
{activeProject && (
<button onClick={removeProject} title="Delete this project (documents kept)"
style={{ padding: "0.4rem 0.7rem", background: "#2a1a1a", color: "#ff8a80", border: "1px solid #5a2a2a", borderRadius: "8px", cursor: "pointer", fontSize: "0.8rem" }}>
Delete project
</button>
)}
<span style={{ color: "#666", fontSize: "0.78rem" }}>
{activeProject ? "Chat scopes to this project's documents." : "Chat searches all documents."}
</span>
</div>
<div style={{ display: "flex", flexDirection: "column", gap: "0.7rem", marginBottom: "1.5rem" }}>
<input type="text" placeholder="Title" value={title}
onChange={e => setTitle(e.target.value)} style={input} />
interface/web/src/Playbook.jsx
+1 -1
View File
@@ -360,7 +360,7 @@ export function Playbook() {
/>
<input
type="text"
placeholder="Tools (comma separated): search_memory, search_history, search_documents, list_models, get_time"
placeholder="Tools: search_memory, search_history, search_documents, list_models, get_time, web_search, fetch_url, remember"
value={form.tools}
onChange={e => setForm(prev => ({ ...prev, tools: e.target.value }))}
style={{ padding: "0.9rem", background: "#222", color: "#eee", border: "1px solid #333", borderRadius: "10px" }}