enderofwings/NexusOS
1
0
Fork 1
Code Issues Pull Requests 1 Actions 11 Packages Projects Releases Wiki Activity

feat: playbook tools, RAG, vision, voice, chat controls, faster boot

Browse Source 
- Tool-using playbooks: read-only tool registry (search_memory,
  search_history, search_documents, list_models, get_time), per-playbook
  allowlist, agentic loop, and a "running tool" status indicator.
- Document ingest / RAG: documents table + chunker + embed/cosine retrieval
  reusing the existing stack; Documents page (upload/paste, viewer, delete);
  top-k chunks injected into the system prompt.
- Vision chat: attach images, base64 into /api/chat.
- Voice I/O: Web Speech dictation + read-aloud (browser-native, no backend).
- Chat controls: stop, regenerate, edit-and-resend; num_ctx knob in Settings.
- Per-playbook model override.
- History polish: per-conversation + bulk ShareGPT export.
- Faster ncp start: UI up first, Ollama warms in the background, with a
  per-phase timing readout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jon
2026-07-23 13:31:48 -05:00
co-authored by Claude Opus 4.8
parent f514d3dbe5
commit f509034fdd
15 changed files with 1061 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
synapse/main.py
+112 -19
View File
@@ -19,6 +19,7 @@ from .nexus_config import settings, VERSION, DEFAULT_CHAT_MODEL
from .chat import generate_chat_response, stream_chat_response, _synapse_trace
from .ollama_manager import initialize_ollama, initialize_ollama_async, get_ollama_manager
from .playbook_manager import PlaybookManager
from . import tools as _tools
def _render_memory_block(facts) -> str:
"""Render memory items as grouped ## Section / - bullet markdown.
@@ -230,11 +231,15 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
try:
message = payload.get("message", "")
app_settings = store.get_settings()
model = payload.get("model") or await _auto_select_model(message)
# Model precedence: explicit request > active playbook's pinned model > auto-select.
_active_pb = playbooks.get_main_playbook()
_pb_model = _active_pb.model if (_active_pb and _active_pb.model) else ""
model = payload.get("model") or _pb_model or await _auto_select_model(message)
context = payload.get("context", {})
conversation_id = payload.get("conversation_id") or str(_uuid.uuid4())
history = payload.get("history", [])
temperature = payload.get("temperature", app_settings.get("temperature"))
num_ctx = payload.get("num_ctx", app_settings.get("num_ctx", 0))
think = payload.get("think", app_settings.get("think", False))
gpu_offload = payload.get("gpu_offload", app_settings.get("gpu_offload", -1))
num_gpu = await get_ollama_manager().resolve_num_gpu(gpu_offload, model)
@@ -280,6 +285,13 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
separator = "\n\n---\nRelevant past exchanges (use as background context only):\n\n"
system_prompt = (system_prompt + separator + memory_block) if system_prompt else memory_block
# Retrieve relevant uploaded documents (RAG) and inject the top chunks.
doc_hits = await store.search_documents(message, get_ollama_manager().embed, limit=3)
if doc_hits:
doc_block = "\n\n".join(f"[{d['title']}]\n{d['text']}" for d in doc_hits)
separator = "\n\n---\nRelevant documents (cite as source material):\n\n"
system_prompt = (system_prompt + separator + doc_block) if system_prompt else doc_block
# Fetch web search results for time-sensitive queries
search_results = ""
if needs_web_search(message):
@@ -338,7 +350,19 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
_synapse_trace(f"{'─' * 55}\n")
# ── end MindTrace pre-flight ──────────────────────────────────────
metadata: Dict[str, Any] = {"model": model, "context": context, "system": system_prompt, "temperature": temperature, "num_gpu": num_gpu, "think": think}
metadata: Dict[str, Any] = {"model": model, "context": context, "system": system_prompt, "temperature": temperature, "num_gpu": num_gpu, "num_ctx": num_ctx, "think": think}
# Vision: base64 images (data: prefix stripped by the client) ride on the user turn.
images = payload.get("images")
if images:
metadata["images"] = images
# Tool-using playbook: advertise the active playbook's allowlisted tools.
if _main_pb and getattr(_main_pb, "tools", None):
schemas = _tools.schemas_for(_main_pb.tools)
if schemas:
metadata["tools"] = schemas
_synapse_trace(f" TOOLS : {', '.join(_main_pb.tools)}\n")
# Persist conversation and user message before streaming
store.create_conversation(conversation_id)
@@ -363,6 +387,9 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
pass
yield f"event: meta\ndata: {chunk[8:]}\n\n"
continue
if chunk.startswith("__status__"):
yield f"event: status\ndata: {_json.dumps({'tool': chunk[10:]})}\n\n"
continue
response_chunks.append(chunk)
yield f"data: {_json.dumps(chunk)}\n\n"
except _asyncio.TimeoutError:
@@ -637,13 +664,26 @@ async def ollama_status_endpoint():
# -------------------------
# Ollama Start
# -------------------------
_warm_tasks: set = set()
async def _warm_default_model():
"""Load the default model into RAM/VRAM so the first chat isn't a cold read.
Returns the model name. Raises are the caller's to swallow."""
s = store.get_settings()
warm_model = s.get("model") or await ollama.select_best_model()
num_gpu = await ollama.resolve_num_gpu(s.get("gpu_offload", -1), warm_model)
await ollama.warm(warm_model, num_gpu)
return warm_model
@app.post("/ollama/start")
async def ollama_start_endpoint():
async def ollama_start_endpoint(background: bool = False):
try:
global ollama
if ollama is None:
ollama = initialize_ollama()
# start_async, NOT start: the sync one polls with time.sleep(1) up to 30
# times, which blocks the event loop — the whole backend (including the
# UI's 5s status poll) goes dead while Ollama boots, so a slow start
@@ -653,19 +693,24 @@ async def ollama_start_endpoint():
is_running = ollama.is_running() if hasattr(ollama, "is_running") else True
# Warm the default model so the first chat isn't a cold load. This blocks
# until the model is resident, so the UI's Start finishes only once the AI
# is actually ready to answer. Best-effort — warm() never raises.
# Warm the default model so the first chat isn't a cold load.
# background=False (UI "Start AI"): block until resident, so the button
# finishes only once the AI can actually answer.
# background=True (`ncp start`): fire-and-forget so boot returns fast and
# the model warms concurrently — Ollama serialises the load, so a first
# chat that arrives mid-warm simply waits on the same load.
warmed = None
if is_running:
try:
s = store.get_settings()
warm_model = s.get("model") or await ollama.select_best_model()
num_gpu = await ollama.resolve_num_gpu(s.get("gpu_offload", -1), warm_model)
await ollama.warm(warm_model, num_gpu)
warmed = warm_model
except Exception:
pass
if background:
task = _asyncio.create_task(_warm_default_model())
_warm_tasks.add(task) # hold a ref (asyncio only weak-refs tasks)
task.add_done_callback(_warm_tasks.discard)
warmed = "background"
else:
try:
warmed = await _warm_default_model()
except Exception:
pass
return {"status": "started" if is_running else "failed", "running": is_running, "warmed": warmed}
except Exception as e:
@@ -701,6 +746,8 @@ async def get_playbooks():
"goal": p.goal,
"instructions": getattr(p, "instructions", ""),
"tags": getattr(p, "tags", []),
"tools": getattr(p, "tools", []),
"model": getattr(p, "model", ""),
}
for p in playbook_list
]
@@ -749,11 +796,13 @@ def _persist_playbook(playbook_dict: Dict[str, Any]) -> Dict[str, Any]:
goal=playbook_dict.get("goal", ""),
instructions=playbook_dict.get("instructions", ""),
tags=playbook_dict.get("tags", []),
tools=playbook_dict.get("tools", []),
model=playbook_dict.get("model", ""),
order=order
)
playbook_store.add_playbook(playbook_item)
# Return as dict
return {
"id": playbook_item.id,
@@ -761,6 +810,8 @@ def _persist_playbook(playbook_dict: Dict[str, Any]) -> Dict[str, Any]:
"goal": playbook_item.goal,
"instructions": playbook_item.instructions,
"tags": playbook_item.tags,
"tools": playbook_item.tools,
"model": playbook_item.model,
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to persist playbook: {str(e)}")
@@ -810,6 +861,8 @@ async def get_playbook(id: UUID):
"goal": getattr(pb, "goal", None) or (pb.get("goal") if isinstance(pb, dict) else None),
"instructions": getattr(pb, "instructions", "") or (pb.get("instructions") if isinstance(pb, dict) else ""),
"tags": getattr(pb, "tags", []) or (pb.get("tags") if isinstance(pb, dict) else []),
"tools": getattr(pb, "tools", []) or (pb.get("tools") if isinstance(pb, dict) else []),
"model": getattr(pb, "model", "") or (pb.get("model") if isinstance(pb, dict) else ""),
}
return result
except HTTPException:
@@ -880,6 +933,41 @@ async def delete_playbook_endpoint(id: UUID):
raise HTTPException(status_code=500, detail=str(e))
# -------------------------
# Documents (RAG)
# -------------------------
@app.get("/documents")
async def list_documents():
return {"documents": store.list_documents()}
@app.post("/documents")
async def add_document(payload: Dict[str, Any] = Body(...)):
title = (payload.get("title") or "").strip()
content = (payload.get("content") or "").strip()
if not title or not content:
raise HTTPException(status_code=400, detail="title and content are required")
result = await store.add_document(title, content, get_ollama_manager().embed)
if result["chunks"] == 0:
raise HTTPException(status_code=400, detail="no text to index")
return result
@app.get("/documents/{doc_id}")
async def get_document(doc_id: str):
chunks = store.get_document(doc_id)
if not chunks:
raise HTTPException(status_code=404, detail="Not Found")
return {"doc_id": doc_id, "chunks": chunks}
@app.delete("/documents/{doc_id}")
async def delete_document(doc_id: str):
if not store.delete_document(doc_id):
raise HTTPException(status_code=404, detail="Not Found")
return {"status": "deleted"}
# -------------------------
# Unified Search
# -------------------------
@@ -914,19 +1002,24 @@ async def get_conversations(q: Optional[str] = None):
@app.get("/conversations/export")
async def export_conversations(min_turns: int = 1):
async def export_conversations(min_turns: int = 1, conversation_id: Optional[str] = None):
"""Export conversations as ShareGPT JSONL for fine-tuning.
Each line is one conversation:
{"conversations": [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]}
Query params:
min_turns — minimum user/assistant exchanges to include (default 1)
min_turns — minimum user/assistant exchanges to include (default 1)
conversation_id — export just this one conversation (default: all)
"""
from fastapi.responses import Response
import datetime
conversations = store.all_conversations()
if conversation_id:
one = store.get_conversation(conversation_id)
conversations = [one] if one else []
else:
conversations = store.all_conversations()
lines = []
for conv in conversations: