Athena/NexusOS
1
0
Fork 0
forked from enderofwings/NexusOS
Code Pull Requests Activity

feat: sync with upstream — v1.2.0, in-app updates, Projects, modules

Browse Source 
Brings the public tree back in line with the development repo after several
weeks of drift caused by a stale publish include list.

New:
- In-app update path: GET /update/check compares the checkout against
  origin/main and POST /update/apply runs `ncp upgrade` detached (pull,
  rebuild, restart). The sidebar shows the version, checks on click, and
  offers an "update available" pill.
- Projects: a project workspace groups chats and RAG documents, with
  per-project instructions and document retrieval scoped to the active
  project. Replaces the standalone Documents page.
- modules/: auto-discovered feature plugins (mail, network) with their
  frontend counterparts and tests.
- Memory curation runs in-process (synapse/memory/curator.py) on the chat
  model when a conversation goes idle. The separate memory service on :8001
  is gone, along with the launcher lines that started it.

Also: the KDE theme, panel and Promethean terminal assets, the full test
suite, and VERSION 1.2.0.

🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
janvanwan
2026-08-25 09:13:55 -05:00
parent fe5d18afa7
commit 42eaed647a
88 changed files with 4280 additions and 1888 deletions
Download Patch File Download Diff File Expand all files Collapse all files
synapse/memory/store.py
+160 -14
View File
@@ -30,6 +30,7 @@ class MemoryItem(BaseModel):
text: str
tags: List[str] = []
position: int = 0
project_id: str = "" # "" = global: injected into every chat
class MessageItem(BaseModel):
role: str # "user" or "assistant"
@@ -44,6 +45,7 @@ class ConversationItem(BaseModel):
created_at: float
updated_at: float
title: Optional[str] = None
project_id: str = ""
@property
def preview(self) -> str:
@@ -127,6 +129,12 @@ class PersistentMemoryStore:
if cur.fetchone()[0] == 0:
cur.execute("UPDATE memory SET position = rowid")
# Migrate: scope a fact to a project ("" = global, applies everywhere).
try:
cur.execute("ALTER TABLE memory ADD COLUMN project_id TEXT NOT NULL DEFAULT ''")
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS conversations (
id TEXT PRIMARY KEY,
@@ -145,6 +153,24 @@ class PersistentMemoryStore:
cur.execute("ALTER TABLE conversations ADD COLUMN project_id TEXT NOT NULL DEFAULT ''")
except Exception:
pass
# Migrate: high-water mark for memory extraction — the id of the last
# message the curator has already read. Extraction runs once the
# conversation goes idle rather than after every exchange, so this is
# what makes it idempotent and restart-safe: a backend that dies with a
# pending sweep resumes from here instead of re-reading the whole
# transcript and re-saving facts it already saved.
try:
cur.execute("ALTER TABLE conversations ADD COLUMN extracted_through INTEGER NOT NULL DEFAULT 0")
# Only reached the first time the column is added. Everything already
# in the database was extracted per-exchange under the old design, so
# watermark it as read — without this the first idle sweep would
# re-read every historical conversation and re-save its facts.
cur.execute(
"UPDATE conversations SET extracted_through = "
"COALESCE((SELECT MAX(id) FROM messages WHERE conversation_id = conversations.id), 0)"
)
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS messages (
@@ -204,6 +230,11 @@ class PersistentMemoryStore:
created_at REAL NOT NULL
)
""")
# projects.instructions — per-project system prompt ("" = none).
try:
cur.execute("ALTER TABLE projects ADD COLUMN instructions TEXT NOT NULL DEFAULT ''")
except Exception:
pass
# documents.project_id — "" (or missing) means unscoped / All.
try:
cur.execute("ALTER TABLE documents ADD COLUMN project_id TEXT NOT NULL DEFAULT ''")
@@ -229,7 +260,7 @@ class PersistentMemoryStore:
def _load_all_memory(self) -> Dict[str, MemoryItem]:
conn = self._connect()
cur = conn.cursor()
cur.execute("SELECT id, section, text, tags, position FROM memory ORDER BY position ASC, rowid ASC")
cur.execute("SELECT id, section, text, tags, position, project_id FROM memory ORDER BY position ASC, rowid ASC")
rows = cur.fetchall()
conn.close()
@@ -245,6 +276,7 @@ class PersistentMemoryStore:
text=row["text"],
tags=tags,
position=row["position"] or 0,
project_id=row["project_id"] or "",
)
return cache
@@ -271,8 +303,10 @@ class PersistentMemoryStore:
try:
cur = conn.cursor()
cur.execute(
"INSERT OR REPLACE INTO memory (id, section, text, tags, position) VALUES (?, ?, ?, ?, ?)",
(item.id, item.section or "General", item.text, json.dumps(item.tags), item.position)
"INSERT OR REPLACE INTO memory (id, section, text, tags, position, project_id)"
" VALUES (?, ?, ?, ?, ?, ?)",
(item.id, item.section or "General", item.text, json.dumps(item.tags),
item.position, item.project_id or "")
)
conn.commit()
finally:
@@ -318,12 +352,12 @@ class PersistentMemoryStore:
return item
def all(self) -> List[MemoryItem]:
# Always read from DB — the memory service and backend run in separate processes
# with separate caches, so the cache can be stale for facts extracted by the
# memory service after this process started.
# Always read from DB, never the cache: the CLI, the control panel and
# the backend are separate processes with separate caches, so a fact
# written by one is invisible to another's cache.
conn = self._connect()
cur = conn.cursor()
cur.execute("SELECT id, section, text, tags, position FROM memory ORDER BY position ASC, rowid ASC")
cur.execute("SELECT id, section, text, tags, position, project_id FROM memory ORDER BY position ASC, rowid ASC")
rows = cur.fetchall()
conn.close()
items = []
@@ -338,6 +372,7 @@ class PersistentMemoryStore:
text=row["text"],
tags=tags,
position=row["position"] or 0,
project_id=row["project_id"] or "",
))
return items
@@ -405,6 +440,18 @@ class PersistentMemoryStore:
finally:
conn.close()
def set_conversation_project(self, conversation_id: str, project_id: str):
"""Move a conversation into a project ('' = unscoped)."""
conn = self._connect()
try:
conn.execute(
"UPDATE conversations SET project_id = ? WHERE id = ?",
(project_id or "", conversation_id),
)
conn.commit()
finally:
conn.close()
def add_message(self, conversation_id: str, role: str, content: str, model: Optional[str] = None, tokens: Optional[int] = None):
now = time.time()
conn = self._connect()
@@ -452,7 +499,7 @@ class PersistentMemoryStore:
conn = self._connect()
cur = conn.cursor()
cur.execute("""
SELECT c.id, c.created_at, c.updated_at, c.title,
SELECT c.id, c.created_at, c.updated_at, c.title, c.project_id,
m.role, m.content, m.timestamp, m.model, m.tokens
FROM conversations c
LEFT JOIN messages m ON m.conversation_id = c.id
@@ -471,6 +518,7 @@ class PersistentMemoryStore:
created_at=row["created_at"],
updated_at=row["updated_at"],
title=row["title"],
project_id=row["project_id"] or "",
)
order.append(cid)
if row["role"] is not None:
@@ -479,10 +527,79 @@ class PersistentMemoryStore:
)
return [convs[cid] for cid in order]
def pending_extraction(self, conversation_id: str) -> tuple[list, int]:
"""Messages the curator has not read yet, and the id to watermark to.
Returns ([{role, content}], last_id). An empty list means nothing new,
so callers can skip the model call entirely.
"""
conn = self._connect()
try:
row = conn.execute(
"SELECT extracted_through FROM conversations WHERE id = ?", (conversation_id,)
).fetchone()
if row is None:
return [], 0
rows = conn.execute(
"SELECT id, role, content FROM messages "
"WHERE conversation_id = ? AND id > ? ORDER BY id ASC",
(conversation_id, row["extracted_through"] or 0),
).fetchall()
finally:
conn.close()
if not rows:
return [], 0
return ([{"role": r["role"], "content": r["content"]} for r in rows], rows[-1]["id"])
def set_extracted_through(self, conversation_id: str, message_id: int) -> None:
conn = self._connect()
try:
conn.execute(
"UPDATE conversations SET extracted_through = ? WHERE id = ?",
(int(message_id), conversation_id),
)
conn.commit()
finally:
conn.close()
def conversations_awaiting_extraction(self, idle_seconds: float) -> list[str]:
"""Conversations with unread messages that have been quiet long enough
to count as finished. Used to resume sweeps dropped by a restart."""
conn = self._connect()
try:
rows = conn.execute(
"SELECT c.id FROM conversations c JOIN messages m ON m.conversation_id = c.id "
"WHERE m.id > c.extracted_through GROUP BY c.id "
"HAVING MAX(m.timestamp) < ?",
(time.time() - idle_seconds,),
).fetchall()
finally:
conn.close()
return [r["id"] for r in rows]
def delete_conversation(self, conversation_id: str):
conn = self._connect()
try:
cur = conn.cursor()
# Drop the embeddings first, while the message ids still resolve.
# Stale vectors are inert (the search joins messages) but they still
# occupy slots in the ANN over-fetch, so leaving them behind quietly
# thins recall of the conversations that are still here.
cur.execute(
"DELETE FROM message_vectors WHERE message_id IN "
"(SELECT id FROM messages WHERE conversation_id = ?)",
(conversation_id,),
)
# vec_enabled only says the extension loaded; the virtual table is
# created lazily on the first semantic search, so check for it.
if self.vec_enabled and cur.execute(
"SELECT 1 FROM sqlite_master WHERE name = 'vec_messages'"
).fetchone():
cur.execute(
"DELETE FROM vec_messages WHERE rowid IN "
"(SELECT id FROM messages WHERE conversation_id = ?)",
(conversation_id,),
)
cur.execute("DELETE FROM messages WHERE conversation_id = ?", (conversation_id,))
cur.execute("DELETE FROM conversations WHERE id = ?", (conversation_id,))
conn.commit()
@@ -810,17 +927,40 @@ class PersistentMemoryStore:
def list_projects(self) -> List[dict]:
conn = self._connect()
rows = conn.execute("""
SELECT p.id, p.name, p.created_at,
(SELECT COUNT(DISTINCT doc_id) FROM documents d WHERE d.project_id = p.id) AS docs
SELECT p.id, p.name, p.created_at, p.instructions,
(SELECT COUNT(DISTINCT doc_id) FROM documents d WHERE d.project_id = p.id) AS docs,
(SELECT COUNT(*) FROM conversations c WHERE c.project_id = p.id) AS chats
FROM projects p ORDER BY p.created_at ASC
""").fetchall()
conn.close()
return [dict(r) for r in rows]
def delete_project(self, project_id: str) -> bool:
"""Delete a project; its documents survive but become unscoped ("")."""
def set_project_instructions(self, project_id: str, instructions: str) -> bool:
"""Per-project system prompt, layered into chats bound to the project."""
conn = self._connect()
conn.execute("UPDATE documents SET project_id = '' WHERE project_id = ?", (project_id,))
try:
cur = conn.execute("UPDATE projects SET instructions = ? WHERE id = ?",
(instructions or "", project_id))
conn.commit()
return cur.rowcount > 0
finally:
conn.close()
def project_instructions(self, project_id: str) -> str:
"""'' when the project has none, or doesn't exist."""
if not project_id:
return ""
conn = self._connect()
row = conn.execute("SELECT instructions FROM projects WHERE id = ?", (project_id,)).fetchone()
conn.close()
return (row["instructions"] or "") if row else ""
def delete_project(self, project_id: str) -> bool:
"""Delete a project; its documents, chats and facts survive but become
unscoped ("")."""
conn = self._connect()
for table in ("documents", "conversations", "memory"):
conn.execute(f"UPDATE {table} SET project_id = '' WHERE project_id = ?", (project_id,))
cur = conn.execute("DELETE FROM projects WHERE id = ?", (project_id,))
deleted = cur.rowcount
conn.commit()
@@ -1024,7 +1164,10 @@ class PersistentMemoryStore:
# Curator CPU/GPU offload — same scale as gpu_offload above. Default 0
# (all CPU/RAM): OS-neutral and never evicts the chat model from a small
# GPU. Boxes with spare VRAM can set -1 (Auto) or a percent to use the GPU.
"memory_gpu_offload": 0,
# -1 = Auto (let Ollama fit it). The curator is the resident chat model
# now, so there is nothing to keep off the GPU; pinning it to CPU (0)
# only bought coexistence with a second, separate curator model.
"memory_gpu_offload": -1,
# Similar-fact merge: when a newly extracted fact's embedding is at least
# this cosine-similar to an existing fact, UPDATE that fact in place
# instead of appending a duplicate ("edit with new info"). 0 disables
@@ -1033,6 +1176,9 @@ class PersistentMemoryStore:
# top out ~0.61 — so 0.80 catches updates and never merges unrelated
# facts. Lower to catch looser rephrases; raise toward 1.0 to be stricter.
"memory_merge_threshold": 0.80,
# Seconds of quiet before the curator reads a conversation. This is the
# "conversation is over" signal; each new message restarts the clock.
"memory_extract_idle": 120,
}
def get_settings(self) -> Dict[str, Any]: