feat: playbook tools, RAG, vision, voice, chat controls, faster boot

- Tool-using playbooks: read-only tool registry (search_memory,
  search_history, search_documents, list_models, get_time), per-playbook
  allowlist, agentic loop, and a "running tool" status indicator.
- Document ingest / RAG: documents table + chunker + embed/cosine retrieval
  reusing the existing stack; Documents page (upload/paste, viewer, delete);
  top-k chunks injected into the system prompt.
- Vision chat: attach images, base64 into /api/chat.
- Voice I/O: Web Speech dictation + read-aloud (browser-native, no backend).
- Chat controls: stop, regenerate, edit-and-resend; num_ctx knob in Settings.
- Per-playbook model override.
- History polish: per-conversation + bulk ShareGPT export.
- Faster ncp start: UI up first, Ollama warms in the background, with a
  per-phase timing readout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jon
2026-07-23 13:31:48 -05:00
co-authored by Claude Opus 4.8
parent f514d3dbe5
commit f509034fdd
15 changed files with 1061 additions and 96 deletions
+118
View File
@@ -150,6 +150,22 @@ class PersistentMemoryStore:
)
""")
# RAG: one row per chunk. doc_id groups the chunks of a single uploaded
# document; embedding is a JSON vector (search_document-prefixed), set at
# ingest so retrieval needs no backfill.
cur.execute("""
CREATE TABLE IF NOT EXISTS documents (
id TEXT PRIMARY KEY,
doc_id TEXT NOT NULL,
title TEXT NOT NULL,
chunk_idx INTEGER NOT NULL,
text TEXT NOT NULL,
embedding TEXT,
created_at REAL NOT NULL
)
""")
cur.execute("CREATE INDEX IF NOT EXISTS idx_documents_doc_id ON documents (doc_id)")
cur.execute("""
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
@@ -602,6 +618,106 @@ class PersistentMemoryStore:
"matches": matches,
}
# -----------------------------
# Documents (RAG)
# -----------------------------
@staticmethod
def _chunk_text(text: str, size: int = 800) -> List[str]:
"""Split on blank lines, then pack paragraphs into ~`size`-char chunks.
ponytail: naive char-based packing, no token counting or overlap — good
enough for local recall; add overlap if retrieval misses boundaries."""
chunks: List[str] = []
buf = ""
for para in (p.strip() for p in text.split("\n\n")):
if not para:
continue
if buf and len(buf) + len(para) + 2 > size:
chunks.append(buf)
buf = para
else:
buf = f"{buf}\n\n{para}" if buf else para
if buf:
chunks.append(buf)
return chunks
async def add_document(self, title: str, content: str, embed_fn) -> dict:
"""Chunk, embed, and store a document. Returns {doc_id, chunks}."""
import uuid as _uuid
doc_id = str(_uuid.uuid4())
pieces = self._chunk_text(content)
now = time.time()
conn = self._connect()
cur = conn.cursor()
for i, piece in enumerate(pieces):
vec = await embed_fn(self._EMBED_DOC_PREFIX + piece[:2000])
cur.execute(
"INSERT INTO documents (id, doc_id, title, chunk_idx, text, embedding, created_at)"
" VALUES (?, ?, ?, ?, ?, ?, ?)",
(str(_uuid.uuid4()), doc_id, title, i, piece,
json.dumps(vec) if vec else None, now),
)
conn.commit()
conn.close()
return {"doc_id": doc_id, "title": title, "chunks": len(pieces)}
def list_documents(self) -> List[dict]:
conn = self._connect()
cur = conn.cursor()
cur.execute("""
SELECT doc_id, title, COUNT(*) AS chunks, MIN(created_at) AS created_at
FROM documents GROUP BY doc_id, title ORDER BY created_at DESC
""")
rows = cur.fetchall()
conn.close()
return [dict(r) for r in rows]
def get_document(self, doc_id: str) -> List[dict]:
"""Ordered chunks of one document: [{chunk_idx, text}]."""
conn = self._connect()
cur = conn.cursor()
cur.execute(
"SELECT chunk_idx, text FROM documents WHERE doc_id = ? ORDER BY chunk_idx",
(doc_id,),
)
rows = cur.fetchall()
conn.close()
return [dict(r) for r in rows]
def delete_document(self, doc_id: str) -> bool:
conn = self._connect()
cur = conn.cursor()
cur.execute("DELETE FROM documents WHERE doc_id = ?", (doc_id,))
deleted = cur.rowcount
conn.commit()
conn.close()
return deleted > 0
async def search_documents(
self, query: str, embed_fn, limit: int = 3, min_score: float = 0.6
) -> List[dict]:
"""Top-`limit` document chunks most similar to `query`. Returns
[{title, text, score}]. Empty on no query / embeddings down."""
if not query or not query.strip():
return []
query_vec = await embed_fn(self._EMBED_QUERY_PREFIX + query.strip())
if not query_vec:
return []
conn = self._connect()
cur = conn.cursor()
cur.execute("SELECT title, text, embedding FROM documents WHERE embedding IS NOT NULL")
scored = []
for row in cur.fetchall():
try:
vec = json.loads(row["embedding"])
except Exception:
continue
score = _cosine(query_vec, vec)
if score >= min_score:
scored.append({"title": row["title"], "text": row["text"], "score": score})
conn.close()
scored.sort(key=lambda d: d["score"], reverse=True)
return scored[:limit]
# -----------------------------
# Settings API
# -----------------------------
@@ -611,6 +727,8 @@ class PersistentMemoryStore:
# latency for chat/memory. Turn on for hard multi-step problems.
"think": False,
"temperature": 0.7,
# Context window (tokens Ollama keeps in view). 0 → Ollama's model default.
"num_ctx": 0,
"system_prompt": "",
"timeout": 120,
# How long Ollama keeps the model resident in VRAM between messages.