enderofwings/NexusOS
1
0
Fork 1
Code Issues Pull Requests 1 Actions 10 Packages Projects Releases Wiki Activity

feat: sync with upstream — v1.2.0, in-app updates, Projects, modules

Browse Source 
Brings the public tree back in line with the development repo after several
weeks of drift caused by a stale publish include list.

New:
- In-app update path: GET /update/check compares the checkout against
  origin/main and POST /update/apply runs `ncp upgrade` detached (pull,
  rebuild, restart). The sidebar shows the version, checks on click, and
  offers an "update available" pill.
- Projects: a project workspace groups chats and RAG documents, with
  per-project instructions and document retrieval scoped to the active
  project. Replaces the standalone Documents page.
- modules/: auto-discovered feature plugins (mail, network) with their
  frontend counterparts and tests.
- Memory curation runs in-process (synapse/memory/curator.py) on the chat
  model when a conversation goes idle. The separate memory service on :8001
  is gone, along with the launcher lines that started it.

Also: the KDE theme, panel and Promethean terminal assets, the full test
suite, and VERSION 1.2.0.

🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
janvanwan
2026-08-25 09:13:55 -05:00
parent fe5d18afa7
commit 42eaed647a
88 changed files with 4280 additions and 1888 deletions
Download Patch File Download Diff File Expand all files Collapse all files
synapse/main.py
+244 -61
View File
@@ -13,6 +13,7 @@ from uuid import UUID
import httpx
import os as _os
import platform as _platform
import sys as _sys
from pathlib import Path
from fastapi import FastAPI, HTTPException, Body, Request
from fastapi.middleware.cors import CORSMiddleware
@@ -28,8 +29,9 @@ from .chat import generate_chat_response, stream_chat_response, _synapse_trace
from . import chat as _chat
from .ollama_manager import initialize_ollama, initialize_ollama_async, get_ollama_manager
from . import frontend_manager as _frontend_manager
from .playbook_manager import PlaybookManager
from . import playbook_manager
from . import tools as _tools
from .memory.curator import extract_for_conversation
def _render_memory_block(facts) -> str:
"""Render memory items as grouped ## Section / - bullet markdown.
@@ -61,8 +63,11 @@ def _render_memory_block(facts) -> str:
# the user. (A minimal header is kept so the facts have context; drop it entirely
# to inject the raw facts block with no framing.)
_MEMORY_PREAMBLE = (
"\n\n---\nWhat you know about the user — use these facts freely and naturally to "
"inform and personalize your replies:\n\n"
"\n\n---\nThe user is the person you are talking to right now — every user turn in "
"this conversation is his. The facts below are about him, written in the third "
"person only because that is how they are stored; address him as \"you\", never "
"discuss him as an absent third party. Use them freely and naturally to inform "
"and personalize your replies:\n\n"
)
@@ -179,7 +184,6 @@ from .memory.store import store, MemoryItem
from .playbooks.store import playbook_store, PlaybookItem
from .search import needs_web_search, web_search
MEMORY_SERVICE = "http://localhost:8001"
app = FastAPI(title="Synapse Backend", version=VERSION)
@@ -205,6 +209,13 @@ app.add_middleware(
allow_headers=["*"],
)
# --- Modules ---
# Installed feature modules (mail, and whatever comes next) live under the
# repo-root modules/ package and mount their own APIRouter here.
from modules.registry import ROUTERS as _MODULE_ROUTERS
for _module_router in _MODULE_ROUTERS:
app.include_router(_module_router)
# --- Request-size cap ---
# Reject oversized bodies before they are buffered/decoded (a base64 upload or a
@@ -263,7 +274,6 @@ _CHAT_INFLIGHT = _InFlightLimiter(MAX_CONCURRENT_CHATS, "chat")
_UPLOAD_INFLIGHT = _InFlightLimiter(MAX_CONCURRENT_UPLOADS, "document ingest")
# --- GLOBALS ---
playbooks = PlaybookManager()
ollama = None
@@ -284,6 +294,17 @@ async def startup_event():
print("[Synapse] Ready. Ollama not auto-started (manual control).")
async def _ollama_status_async() -> Optional[str]:
"""to_thread, not a direct call: get_status() does blocking IO (a sync httpx
request and, once, a subprocess). Awaiting it inline stalls the whole event
loop for its duration - and both callers below are polled continuously by
the UI, so an inline call froze the entire server in lockstep with its own
health check."""
if ollama is None or not hasattr(ollama, "get_status"):
return None
return await _asyncio.to_thread(ollama.get_status)
# -------------------------
# Status
# -------------------------
@@ -291,17 +312,150 @@ async def startup_event():
@app.get("/status")
async def root():
try:
# to_thread, not a direct call: get_status() does blocking IO (an httpx
# request and, once, a subprocess). Awaiting it inline stalled the whole
# event loop on every poll - and the UI polls /status continuously, so
# the server froze in lockstep with its own health check.
status = (await _asyncio.to_thread(ollama.get_status)
if (ollama is not None and hasattr(ollama, "get_status")) else None)
status = await _ollama_status_async()
except Exception:
status = None
return {"status": "online", "version": VERSION, "ollama": status, "platform": _platform.system().lower()}
# -------------------------
# Update check
# -------------------------
# The update mechanism is git: `git pull` + rebuild, i.e. `python bin/sync.py
# restore` (./install.sh on Linux). So "is a newer build out there" is just
# "how many commits is origin/main ahead of HEAD" — no version server needed.
def _git(*args: str, timeout: int = 30) -> str:
import subprocess
out = subprocess.run(
["git", *args], cwd=str(settings.project_root),
capture_output=True, text=True, timeout=timeout,
)
if out.returncode:
raise RuntimeError((out.stderr or out.stdout).strip() or "git failed")
return out.stdout.strip()
def _check_update() -> Dict[str, Any]:
_git("fetch", "--quiet", "origin", timeout=60)
behind = int(_git("rev-list", "--count", "HEAD..origin/main") or 0)
remote_version = VERSION
if behind:
with _contextlib.suppress(Exception):
remote_version = _git("show", "origin/main:VERSION").strip() or VERSION
return {
"version": VERSION,
"remote_version": remote_version,
"behind": behind,
"latest": _git("log", "-1", "--format=%h %s", "origin/main") if behind else "",
}
@app.get("/update/check")
async def update_check():
"""Compare this checkout against origin/main. Network call + subprocess, so
it runs off the event loop like the Ollama health check does."""
try:
return await _asyncio.to_thread(_check_update)
except Exception as e:
return {"version": VERSION, "behind": 0, "error": str(e)[:300]}
_update_running = False
@app.post("/update/apply")
async def update_apply():
"""Run `ncp upgrade` (git pull, rebuild, restart) fully detached.
It stops this very process, so it cannot be a child of it: a child would be
killed halfway through its own upgrade. Output goes to runtime/logs/update.log
because the UI cannot read /logs while the backend is down - that file is the
only record if the restart fails.
"""
global _update_running
import subprocess
if _update_running:
return {"started": False, "error": "An update is already running."}
log_path = settings.logs_dir / "update.log"
kwargs = ({"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP
| getattr(subprocess, "CREATE_NO_WINDOW", 0)}
if _os.name == "nt" else {"start_new_session": True})
try:
log = open(log_path, "wb")
subprocess.Popen(
[_sys.executable, str(settings.project_root / "management" / "ncp.py"), "upgrade"],
cwd=str(settings.project_root), stdout=log, stderr=subprocess.STDOUT,
stdin=subprocess.DEVNULL, **kwargs,
)
except Exception as e:
return {"started": False, "error": str(e)[:300]}
_update_running = True
return {"started": True, "log": str(log_path)}
# --- Deferred memory extraction ------------------------------------------
# The curator reads a conversation when it has been quiet for a while, rather
# than after every exchange. Each new message reschedules, so "idle" means the
# user actually stopped — which is also the only reliable signal that a
# half-said fact is now complete.
_pending_extractions: Dict[str, _asyncio.Task] = {}
def _extract_idle_seconds() -> float:
try:
return max(5.0, float(store.get_settings().get("memory_extract_idle", 120)))
except (TypeError, ValueError):
return 120.0
async def _extract_after_idle(conversation_id: str, project_id: str, delay: float) -> None:
try:
await _asyncio.sleep(delay)
saved = await extract_for_conversation(conversation_id, project_id)
if saved:
_synapse_trace(
f"\n◆ CURATOR (idle sweep): {len(saved)} fact(s) — "
+ "; ".join(f"[{i['section']}] {i['text']}" for i in saved) + "\n"
)
except _asyncio.CancelledError:
raise # rescheduled by a newer message; the watermark still has the work
except Exception as e:
_synapse_trace(f"\n⚠ idle extraction failed: {e}\n")
finally:
_pending_extractions.pop(conversation_id, None)
def _schedule_extraction(conversation_id: str, project_id: str) -> None:
"""(Re)arm the idle sweep for one conversation."""
previous = _pending_extractions.pop(conversation_id, None)
if previous:
previous.cancel()
_pending_extractions[conversation_id] = _asyncio.create_task(
_extract_after_idle(conversation_id, project_id, _extract_idle_seconds())
)
@app.on_event("startup")
async def _resume_dropped_extractions() -> None:
"""Pick up conversations whose sweep was lost to a restart. The watermark
makes this idempotent, so a conversation already read is skipped without
ever reaching the model."""
async def _bg():
await _asyncio.sleep(5) # let the app finish coming up first
try:
stale = store.conversations_awaiting_extraction(_extract_idle_seconds())
except Exception:
return
for cid in stale:
try:
# One at a time: Ollama serialises anyway, and a burst here would
# sit in front of the user's first message of the session.
await extract_for_conversation(cid, store.conversation_project(cid) or "")
except Exception:
continue
_asyncio.create_task(_bg())
# -------------------------
# Chat (streaming)
# -------------------------
@@ -316,7 +470,7 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
message = payload.get("message", "")
app_settings = store.get_settings()
# Model precedence: explicit request > active playbook's pinned model > auto-select.
_active_pb = playbooks.get_main_playbook()
_active_pb = playbook_manager.get_main_playbook()
_pb_model = _active_pb.model if (_active_pb and _active_pb.model) else ""
model = payload.get("model") or _pb_model or await _auto_select_model(message)
context = payload.get("context", {})
@@ -331,14 +485,28 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
if not message:
raise HTTPException(status_code=400, detail="Missing 'message'")
rendered_message = message # chat has no template vars; render_prompt is for the playbook path
system_prompt = playbooks.get_system_prompt() or app_settings.get("system_prompt", "")
# Resolve the project scope: an existing conversation keeps its bound project;
# a brand-new one inherits the current workspace (active_project setting).
# Everything project-scoped below (instructions, memory facts, RAG) uses it.
_conv_proj = store.conversation_project(conversation_id)
rag_scope = _conv_proj if _conv_proj is not None else app_settings.get("active_project", "")
# Fetch memory facts once — used for both playbook routing and system prompt injection
memory_facts = store.all()
rendered_message = message # chat has no template vars; render_prompt is for the playbook path
system_prompt = playbook_manager.get_system_prompt() or app_settings.get("system_prompt", "")
# Fetch memory facts once — used for both playbook routing and system prompt
# injection. Global facts ("") always apply; the rest only inside their project.
memory_facts = [m for m in store.all() if m.project_id in ("", rag_scope)]
# Per-project instructions sit right under the playbook: they say how the
# assistant should behave for this project specifically.
project_instructions = store.project_instructions(rag_scope)
if project_instructions:
separator = "\n\n---\nProject instructions (follow these for this project):\n\n"
system_prompt = (system_prompt + separator + project_instructions) if system_prompt else project_instructions
# Append the best-matching reference playbook(s) to the system prompt
context_pbs = _route_playbooks(rendered_message, playbooks.get_context_playbooks())
context_pbs = _route_playbooks(rendered_message, playbook_manager.get_context_playbooks())
if context_pbs:
refs = "\n\n".join(
f"### {pb.title}\nGoal: {pb.goal}\n\n{pb.instructions}"
@@ -369,11 +537,6 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
separator = "\n\n---\nRelevant past exchanges (use as background context only):\n\n"
system_prompt = (system_prompt + separator + memory_block) if system_prompt else memory_block
# Resolve the RAG scope: an existing conversation keeps its bound project;
# a brand-new one inherits the current workspace (active_project setting).
_conv_proj = store.conversation_project(conversation_id)
rag_scope = _conv_proj if _conv_proj is not None else app_settings.get("active_project", "")
# Retrieve relevant uploaded documents (RAG) and inject the top chunks.
doc_hits = await store.search_documents(
message, get_ollama_manager().embed,
@@ -414,7 +577,7 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
else:
_synapse_trace(f" INTENT: chat\n")
_main_pb = playbooks.get_main_playbook()
_main_pb = playbook_manager.get_main_playbook()
if _main_pb:
_synapse_trace(f" PLAYBOOK: {_main_pb.title}\n")
if _main_pb.goal:
@@ -442,6 +605,8 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
if past_context:
_synapse_trace(f" CONTEXT : {len(past_context)} past conversation match(es) injected\n")
if rag_scope:
_synapse_trace(f" PROJECT : {rag_scope}{' [+instructions]' if project_instructions else ''}\n")
_synapse_trace(f" SYS LEN : {len(system_prompt)} chars\n")
_synapse_trace(f"{'─' * 55}\n")
# ── end MindTrace pre-flight ──────────────────────────────────────
@@ -453,19 +618,26 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
if images:
metadata["images"] = images
# Tool-using playbook: advertise the active playbook's allowlisted tools.
# Action tools follow action_tool_policy: off (withheld) / ask (per-call
# approval, handled in the tool loop) / allow (run freely).
# Tool-using playbook: advertise the allowlisted tools of the active
# playbook AND of the reference playbooks _route_playbooks picked for
# this message — a routed playbook's instructions are already in the
# prompt, so its abilities have to come with them or the model narrates
# tools it was never given. Action tools follow action_tool_policy:
# off (withheld) / ask (per-call approval, in the tool loop) / allow.
_policy = app_settings.get("action_tool_policy", "off")
if _main_pb and getattr(_main_pb, "tools", None):
_pb_tools = list(dict.fromkeys(
(getattr(_main_pb, "tools", None) or [] if _main_pb else [])
+ [t for pb in context_pbs for t in (getattr(pb, "tools", None) or [])]
))
if _pb_tools:
allow_actions = _policy != "off"
schemas = _tools.schemas_for(_main_pb.tools, allow_actions)
schemas = _tools.schemas_for(_pb_tools, allow_actions)
if schemas:
metadata["tools"] = schemas
metadata["action_tool_policy"] = _policy
metadata["conversation_id"] = conversation_id
_granted = [t for t in _main_pb.tools if not _tools.is_action(t) or allow_actions]
_withheld = [t for t in _main_pb.tools if _tools.is_action(t) and not allow_actions]
_granted = [t for t in _pb_tools if not _tools.is_action(t) or allow_actions]
_withheld = [t for t in _pb_tools if _tools.is_action(t) and not allow_actions]
_synapse_trace(f" TOOLS : {', '.join(_granted)} [actions: {_policy}]\n")
if _withheld:
_synapse_trace(f" WITHHELD: {', '.join(_withheld)} (action tools off)\n")
@@ -546,24 +718,12 @@ async def chat_stream_endpoint(payload: Dict[str, Any]):
except Exception:
pass
# Ask the memory service curator to evaluate this exchange
# Hand the conversation to the curator once it goes quiet. Not now:
# the curator is the chat model, and Ollama runs one request at a
# time (OLLAMA_NUM_PARALLEL=1), so extracting here would put the
# next message in a queue behind it.
if response_chunks:
try:
async with httpx.AsyncClient(timeout=310.0) as _mc:
r = await _mc.post(
f"{MEMORY_SERVICE}/memories/extract",
json={
"user_message": rendered_message,
"assistant_response": "".join(response_chunks),
},
)
if r.status_code == 200:
data = r.json()
for it in data.get("items", []):
mem_result = {"section": it["section"], "text": it["text"]}
yield f"event: memory\ndata: {_json.dumps(mem_result)}\n\n"
except Exception as e:
_synapse_trace(f"\n⚠ memory extraction call failed: {e}\n")
_schedule_extraction(conversation_id, rag_scope)
# Hold the concurrency slot for the stream's lifetime, then release it
# exactly once when the generator is exhausted or closed (client
@@ -671,8 +831,13 @@ async def put_settings_endpoint(payload: Dict[str, Any] = Body(...)):
# Memory
# -------------------------
@app.get("/memory")
async def get_memory():
return {"items": [{"id": m.id, "section": m.section, "text": m.text, "tags": m.tags} for m in store.all()]}
async def get_memory(project: Optional[str] = None):
"""All facts, or — with ?project=<id> — just that scope's ('' = global)."""
items = store.all()
if project is not None:
items = [m for m in items if m.project_id == project]
return {"items": [{"id": m.id, "section": m.section, "text": m.text, "tags": m.tags,
"project_id": m.project_id} for m in items]}
@app.post("/memory")
@@ -687,9 +852,11 @@ async def add_memory(payload: Dict[str, Any] = Body(...)):
section=(payload.get("section") or "General").strip(),
text=text,
tags=payload.get("tags", []),
project_id=payload.get("project_id") or "",
)
store.add(item)
return {"id": item.id, "section": item.section, "text": item.text, "tags": item.tags}
return {"id": item.id, "section": item.section, "text": item.text, "tags": item.tags,
"project_id": item.project_id}
@app.patch("/memory/{item_id}")
@@ -703,9 +870,11 @@ async def update_memory(item_id: str, payload: Dict[str, Any] = Body(...)):
section=(payload.get("section") or existing.section or "General").strip(),
text=(payload.get("text") or existing.text).rstrip(),
tags=payload.get("tags", existing.tags),
project_id=payload.get("project_id", existing.project_id),
)
store.update(updated)
return {"id": updated.id, "section": updated.section, "text": updated.text, "tags": updated.tags}
return {"id": updated.id, "section": updated.section, "text": updated.text,
"tags": updated.tags, "project_id": updated.project_id}
@app.delete("/memory/{item_id}")
@@ -818,10 +987,7 @@ async def delete_model(name: str):
@app.get("/ollama/status")
async def ollama_status_endpoint():
try:
if ollama is not None and hasattr(ollama, "get_status"):
status = ollama.get_status()
else:
status = None
status = await _ollama_status_async()
return {"status": status}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@@ -1148,6 +1314,8 @@ async def stt_transcribe(payload: Dict[str, Any] = Body(...)):
return {"text": text}
# -------------------------
# Projects / workspaces
# -------------------------
@@ -1169,6 +1337,15 @@ async def create_project(payload: Dict[str, Any] = Body(...)):
return store.create_project(name)
@app.patch("/projects/{project_id}")
async def update_project(project_id: str, payload: Dict[str, Any] = Body(...)):
"""Set the project's instructions — layered into the system prompt of every
chat bound to this project."""
if not store.set_project_instructions(project_id, payload.get("instructions") or ""):
raise HTTPException(status_code=404, detail="Project not found")
return {"id": project_id, "instructions": payload.get("instructions") or ""}
@app.delete("/projects/{project_id}")
async def delete_project(project_id: str):
if not store.delete_project(project_id):
@@ -1281,9 +1458,11 @@ async def delete_document(doc_id: str):
# Conversations
# -------------------------
@app.get("/conversations")
async def get_conversations(q: Optional[str] = None):
async def get_conversations(q: Optional[str] = None, project: Optional[str] = None):
try:
conversations = store.all_conversations()
if project is not None:
conversations = [c for c in conversations if c.project_id == project]
if q:
q_lower = q.lower()
conversations = [
@@ -1299,6 +1478,7 @@ async def get_conversations(q: Optional[str] = None):
"updated_at": c.updated_at,
"preview": c.preview,
"title": c.title,
"project_id": c.project_id,
}
for c in conversations
]
@@ -1376,17 +1556,20 @@ async def get_conversation(conversation_id: str):
@app.patch("/conversations/{conversation_id}")
async def rename_conversation(conversation_id: str, payload: Dict[str, Any] = Body(...)):
"""Manually set a conversation's title."""
"""Set a conversation's title and/or move it into a project."""
try:
conv = store.get_conversation(conversation_id)
if not conv:
raise HTTPException(status_code=404, detail="Conversation not found")
if "project_id" in payload:
store.set_conversation_project(conversation_id, payload.get("project_id") or "")
title = (payload.get("title") or "").strip()
if not title:
if title:
title = " ".join(title.split())[:120]
store.set_conversation_title(conversation_id, title)
elif "project_id" not in payload:
raise HTTPException(status_code=400, detail="Missing 'title'")
title = " ".join(title.split())[:120]
store.set_conversation_title(conversation_id, title)
return {"id": conversation_id, "title": title}
return {"id": conversation_id, "title": title or conv.title}
except HTTPException:
raise
except Exception as e: