feat: workspaces, agentic action tools, local Whisper STT, vec recall
- Projects/workspaces: documents grouped into projects; chat RAG scopes to the active project. Switcher in the Documents page. - Agentic action tools: web_search, fetch_url, and remember (first write tool), allowlist-gated per playbook. - Local Whisper STT (faster-whisper, no torch): on-device dictation replacing the browser Web Speech API. POST /stt + GET /stt/status; browser fallback. - Vector index extended to conversation recall (message_vectors), with the brute-force cosine scan kept as the fallback. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jon
+162
-31
@@ -191,6 +191,20 @@ class PersistentMemoryStore:
|
||||
""")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_documents_doc_id ON documents (doc_id)")
|
||||
|
||||
# Projects / workspaces: group documents so RAG can scope to one set.
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
# documents.project_id — "" (or missing) means unscoped / All.
|
||||
try:
|
||||
cur.execute("ALTER TABLE documents ADD COLUMN project_id TEXT NOT NULL DEFAULT ''")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
@@ -569,26 +583,31 @@ class PersistentMemoryStore:
|
||||
"INSERT OR REPLACE INTO message_vectors (message_id, embedding) VALUES (?, ?)",
|
||||
(row["id"], json.dumps(vec)),
|
||||
)
|
||||
self._vec_upsert_msg(conn, row["id"], vec) # mirror into the ANN index
|
||||
if missing:
|
||||
conn.commit()
|
||||
|
||||
# Score every stored message against the query vector.
|
||||
cur.execute("""
|
||||
SELECT v.message_id, v.embedding, m.conversation_id
|
||||
FROM message_vectors v
|
||||
JOIN messages m ON m.id = v.message_id
|
||||
""")
|
||||
scored = []
|
||||
for row in cur.fetchall():
|
||||
try:
|
||||
vec = json.loads(row["embedding"])
|
||||
except Exception:
|
||||
continue
|
||||
score = _cosine(query_vec, vec)
|
||||
if score >= min_score:
|
||||
scored.append((score, row["message_id"], row["conversation_id"]))
|
||||
|
||||
scored.sort(reverse=True)
|
||||
# Rank messages by similarity. Fast path: the sqlite-vec ANN index over an
|
||||
# over-fetch (conversation dedup below thins it); else brute-force cosine.
|
||||
scored = None
|
||||
if self.vec_enabled:
|
||||
scored = self._vec_search_messages(conn, query_vec, max(limit * 5, 20), min_score)
|
||||
if scored is None:
|
||||
cur.execute("""
|
||||
SELECT v.message_id, v.embedding, m.conversation_id
|
||||
FROM message_vectors v
|
||||
JOIN messages m ON m.id = v.message_id
|
||||
""")
|
||||
scored = []
|
||||
for row in cur.fetchall():
|
||||
try:
|
||||
vec = json.loads(row["embedding"])
|
||||
except Exception:
|
||||
continue
|
||||
score = _cosine(query_vec, vec)
|
||||
if score >= min_score:
|
||||
scored.append((score, row["message_id"], row["conversation_id"]))
|
||||
scored.sort(reverse=True)
|
||||
|
||||
results: List[dict] = []
|
||||
seen_convs: set = set()
|
||||
@@ -616,6 +635,70 @@ class PersistentMemoryStore:
|
||||
|
||||
return results
|
||||
|
||||
# --- message vector index (sqlite-vec) — same pattern as documents --------
|
||||
def _ensure_vec_msgs(self, conn, dim: int) -> None:
|
||||
conn.execute(
|
||||
f"CREATE VIRTUAL TABLE IF NOT EXISTS vec_messages "
|
||||
f"USING vec0(embedding float[{dim}] distance_metric=cosine)"
|
||||
)
|
||||
|
||||
def _vec_upsert_msg(self, conn, message_id: int, vec: list) -> None:
|
||||
if not (self.vec_enabled and vec):
|
||||
return
|
||||
try:
|
||||
import sqlite_vec
|
||||
self._ensure_vec_msgs(conn, len(vec))
|
||||
conn.execute("DELETE FROM vec_messages WHERE rowid = ?", (message_id,))
|
||||
conn.execute(
|
||||
"INSERT INTO vec_messages(rowid, embedding) VALUES (?, ?)",
|
||||
(message_id, sqlite_vec.serialize_float32(vec)),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _backfill_vec_msgs(self, conn, dim: int) -> None:
|
||||
"""Index any message_vectors rows missing from vec_messages."""
|
||||
try:
|
||||
import sqlite_vec
|
||||
self._ensure_vec_msgs(conn, dim)
|
||||
rows = conn.execute(
|
||||
"SELECT mv.message_id AS mid, mv.embedding AS emb FROM message_vectors mv "
|
||||
"LEFT JOIN vec_messages v ON v.rowid = mv.message_id WHERE v.rowid IS NULL"
|
||||
).fetchall()
|
||||
for r in rows:
|
||||
try:
|
||||
vec = json.loads(r["emb"])
|
||||
if len(vec) == dim:
|
||||
conn.execute(
|
||||
"INSERT INTO vec_messages(rowid, embedding) VALUES (?, ?)",
|
||||
(r["mid"], sqlite_vec.serialize_float32(vec)),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _vec_search_messages(self, conn, query_vec: list, fetch: int, min_score: float):
|
||||
"""Top message hits via the vec index as sorted [(score, msg_id, conv_id)],
|
||||
or None to fall back to the brute-force scan. Stale rows for deleted
|
||||
messages are dropped by the inner join, so they never surface."""
|
||||
try:
|
||||
import sqlite_vec
|
||||
self._backfill_vec_msgs(conn, len(query_vec))
|
||||
rows = conn.execute(
|
||||
"SELECT v.rowid AS mid, v.distance AS distance, m.conversation_id AS cid "
|
||||
"FROM vec_messages v JOIN messages m ON m.id = v.rowid "
|
||||
"WHERE v.embedding MATCH ? ORDER BY v.distance LIMIT ?",
|
||||
(sqlite_vec.serialize_float32(query_vec), fetch),
|
||||
).fetchall()
|
||||
return [
|
||||
(1.0 - r["distance"], r["mid"], r["cid"])
|
||||
for r in rows if (1.0 - r["distance"]) >= min_score
|
||||
] # distance-asc == score-desc, already sorted
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _exchange_pair(self, cur, conv_id: str, message_id: int) -> Optional[dict]:
|
||||
"""Build a {id, updated_at, matches} record with the full user+assistant
|
||||
pair surrounding `message_id`, in the shape search callers expect."""
|
||||
@@ -677,7 +760,7 @@ class PersistentMemoryStore:
|
||||
chunks.append(buf)
|
||||
return chunks
|
||||
|
||||
async def add_document(self, title: str, content: str, embed_fn) -> dict:
|
||||
async def add_document(self, title: str, content: str, embed_fn, project_id: str = "") -> dict:
|
||||
"""Chunk, embed, and store a document. Returns {doc_id, chunks}."""
|
||||
import uuid as _uuid
|
||||
doc_id = str(_uuid.uuid4())
|
||||
@@ -688,16 +771,47 @@ class PersistentMemoryStore:
|
||||
for i, piece in enumerate(pieces):
|
||||
vec = await embed_fn(self._EMBED_DOC_PREFIX + piece[:2000])
|
||||
cur.execute(
|
||||
"INSERT INTO documents (id, doc_id, title, chunk_idx, text, embedding, created_at)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
"INSERT INTO documents (id, doc_id, title, chunk_idx, text, embedding, created_at, project_id)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(str(_uuid.uuid4()), doc_id, title, i, piece,
|
||||
json.dumps(vec) if vec else None, now),
|
||||
json.dumps(vec) if vec else None, now, project_id or ""),
|
||||
)
|
||||
self._vec_upsert(conn, cur.lastrowid, vec) # mirror into the ANN index
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return {"doc_id": doc_id, "title": title, "chunks": len(pieces)}
|
||||
|
||||
# --- projects / workspaces --------------------------------------------------
|
||||
def create_project(self, name: str) -> dict:
|
||||
import uuid as _uuid
|
||||
pid = str(_uuid.uuid4())
|
||||
conn = self._connect()
|
||||
conn.execute("INSERT INTO projects (id, name, created_at) VALUES (?, ?, ?)",
|
||||
(pid, name.strip(), time.time()))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return {"id": pid, "name": name.strip()}
|
||||
|
||||
def list_projects(self) -> List[dict]:
|
||||
conn = self._connect()
|
||||
rows = conn.execute("""
|
||||
SELECT p.id, p.name, p.created_at,
|
||||
(SELECT COUNT(DISTINCT doc_id) FROM documents d WHERE d.project_id = p.id) AS docs
|
||||
FROM projects p ORDER BY p.created_at ASC
|
||||
""").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def delete_project(self, project_id: str) -> bool:
|
||||
"""Delete a project; its documents survive but become unscoped ("")."""
|
||||
conn = self._connect()
|
||||
conn.execute("UPDATE documents SET project_id = '' WHERE project_id = ?", (project_id,))
|
||||
cur = conn.execute("DELETE FROM projects WHERE id = ?", (project_id,))
|
||||
deleted = cur.rowcount
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return deleted > 0
|
||||
|
||||
# --- vector index (sqlite-vec) — accelerator over the JSON embedding column,
|
||||
# --- with brute-force cosine below as the guaranteed fallback ---------------
|
||||
def _ensure_vec_docs(self, conn, dim: int) -> None:
|
||||
@@ -768,13 +882,21 @@ class PersistentMemoryStore:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def list_documents(self) -> List[dict]:
|
||||
def list_documents(self, project_id: Optional[str] = None) -> List[dict]:
|
||||
"""All documents, or just one project's when project_id is given."""
|
||||
conn = self._connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT doc_id, title, COUNT(*) AS chunks, MIN(created_at) AS created_at
|
||||
FROM documents GROUP BY doc_id, title ORDER BY created_at DESC
|
||||
""")
|
||||
if project_id is not None:
|
||||
cur.execute("""
|
||||
SELECT doc_id, title, COUNT(*) AS chunks, MIN(created_at) AS created_at
|
||||
FROM documents WHERE project_id = ?
|
||||
GROUP BY doc_id, title ORDER BY created_at DESC
|
||||
""", (project_id,))
|
||||
else:
|
||||
cur.execute("""
|
||||
SELECT doc_id, title, COUNT(*) AS chunks, MIN(created_at) AS created_at
|
||||
FROM documents GROUP BY doc_id, title ORDER BY created_at DESC
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
@@ -807,23 +929,30 @@ class PersistentMemoryStore:
|
||||
return deleted > 0
|
||||
|
||||
async def search_documents(
|
||||
self, query: str, embed_fn, limit: int = 3, min_score: float = 0.6
|
||||
self, query: str, embed_fn, limit: int = 3, min_score: float = 0.6,
|
||||
project_id: Optional[str] = None,
|
||||
) -> List[dict]:
|
||||
"""Top-`limit` document chunks most similar to `query`. Returns
|
||||
[{title, text, score}]. Empty on no query / embeddings down."""
|
||||
[{title, text, score}]. Empty on no query / embeddings down.
|
||||
When project_id is given, only that project's docs are searched.
|
||||
ponytail: scoped search uses the brute-force path (easy SQL filter, few
|
||||
docs per project); the vec index accelerates the unscoped "All" case."""
|
||||
if not query or not query.strip():
|
||||
return []
|
||||
query_vec = await embed_fn(self._EMBED_QUERY_PREFIX + query.strip())
|
||||
if not query_vec:
|
||||
return []
|
||||
# Fast path: the sqlite-vec ANN index. None => fall through to brute force.
|
||||
if self.vec_enabled:
|
||||
# Fast path (unscoped only): the sqlite-vec ANN index.
|
||||
if not project_id and self.vec_enabled:
|
||||
hits = self._vec_search(query_vec, limit, min_score)
|
||||
if hits is not None:
|
||||
return hits
|
||||
conn = self._connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT title, text, embedding FROM documents WHERE embedding IS NOT NULL")
|
||||
if project_id:
|
||||
cur.execute("SELECT title, text, embedding FROM documents WHERE embedding IS NOT NULL AND project_id = ?", (project_id,))
|
||||
else:
|
||||
cur.execute("SELECT title, text, embedding FROM documents WHERE embedding IS NOT NULL")
|
||||
scored = []
|
||||
for row in cur.fetchall():
|
||||
try:
|
||||
@@ -852,6 +981,8 @@ class PersistentMemoryStore:
|
||||
# cosine similarity (0-1) a chunk must clear to count as relevant.
|
||||
"rag_top_k": 3,
|
||||
"rag_min_score": 0.6,
|
||||
# Active project/workspace; "" = all documents (unscoped).
|
||||
"active_project": "",
|
||||
"system_prompt": "",
|
||||
"timeout": 120,
|
||||
# How long Ollama keeps the model resident in VRAM between messages.
|
||||
|
||||
Reference in New Issue
Block a user
GitNexus · part of the Nexus stack