NexusOS/management/controlpanel.py

# /home/jon/nexus-core/management/controlpanel.py

import time
import tkinter as tk
import subprocess
import threading
import os
import signal
import sys
import webbrowser
import httpx
from pathlib import Path

# --- CONFIGURATION ---
PROJECT_ROOT = Path(__file__).resolve().parent.parent

# Single source of truth: the VERSION file at the repo root
try:
    VERSION = (PROJECT_ROOT / "VERSION").read_text(encoding="utf-8").strip() or "0.0.0"
except Exception:
    VERSION = "0.0.0"

# Force 0.0.0.0 to ensure the Windows Host bridge works for Project Nexus
FRONTEND_DIR = PROJECT_ROOT / "interface" / "web"
def _find_npm():
    """Prefer nvm's newest node. When the panel is launched from the desktop
    (not a shell), nvm isn't on PATH, so shutil.which('npm') returns the SYSTEM
    node — often too old for the installed Vite (e.g. Node 18 -> "CustomEvent is
    not defined"). nvm exists precisely to shadow the system node, and ncp
    already prefers it, so resolve it here too. _spawn_process prepends the
    returned npm's bin dir to PATH, so its node wins for the child process."""
    import shutil
    nvm_dir = Path.home() / ".nvm" / "versions" / "node"
    if nvm_dir.exists():
        def _ver(p):
            try:
                return tuple(int(x) for x in p.name.lstrip("v").split("."))
            except ValueError:
                return (0,)
        for v in sorted((d for d in nvm_dir.iterdir() if d.is_dir()), key=_ver, reverse=True):
            candidate = v / "bin" / "npm"
            if candidate.exists():
                return str(candidate)
    return shutil.which("npm") or "npm"

FRONTEND_CMD = [_find_npm(), "run", "dev", "--", "--host", "0.0.0.0"]
FRONTEND_URL = "http://127.0.0.1:5173"

BACKEND_DIR = PROJECT_ROOT
BACKEND_CMD = [
    str(PROJECT_ROOT / "Promethean" / "bin" / "python"),
    "-m", "uvicorn",
    "synapse.main:sio_app",
    "--host", "0.0.0.0",
    "--port", "8000",
    "--reload",
    "--reload-dir", str(PROJECT_ROOT / "synapse"),
    "--log-level", "info"
]

MEMORY_DIR = PROJECT_ROOT
MEMORY_CMD = [
    str(PROJECT_ROOT / "Promethean" / "bin" / "python"),
    "-m", "uvicorn",
    "synapse.memory.service:app",
    "--host", "0.0.0.0",
    "--port", "8001",
    "--reload",
    "--reload-dir", str(PROJECT_ROOT / "synapse"),
    "--log-level", "info"
]

FRONTEND_PID = PROJECT_ROOT / "runtime" / "pids" / "frontend.pid"
BACKEND_PID = PROJECT_ROOT / "runtime" / "pids" / "backend.pid"
MEMORY_PID = PROJECT_ROOT / "runtime" / "pids" / "memory.pid"
CHAT_LOG = PROJECT_ROOT / "runtime" / "logs" / "chat.log"

class NexusControlPanel:
    def __init__(self, root):
        self.root = root
        
        if os.name != 'nt' and not os.environ.get('DISPLAY'):
            print("ERROR: No DISPLAY variable. Check WSLg.")
            sys.exit(1)

        self.root.title("Nexus Control Panel")
        # Example icon path; replace with your actual icon file path
        try:
            self.root.iconphoto(True, tk.PhotoImage(file=str(PROJECT_ROOT / "assets" / "n-small.png")))
        except Exception as e:
            print(f"Icon load failed: {e}")

        if not sys.platform.startswith("linux"):
            # Keep the custom borderless titlebar on Windows/WSL,
            # but allow normal stacking on Linux Mint.
            self.root.overrideredirect(True)
        self.root.configure(background="#1e1e1e")
        
        window_width = 1800
        window_height = 720

        try:
            self.root.update_idletasks()
            sw = self.root.winfo_screenwidth()
            sh = self.root.winfo_screenheight()
            self.root.geometry(f'{window_width}x{window_height}+{int(sw/2 - window_width/2)}+{int(sh/2 - window_height/2)}')
        except:
            self.root.geometry(f'{window_width}x{window_height}+100+100')

        self.frontend_process = None
        self.backend_process = None
        self.memory_process = None
        self._frontend_starting = False
        self._backend_starting = False
        self._memory_starting = False
        self._closing = False

        self._setup_custom_titlebar()
        self._setup_ui()
        self._init_log_tags()
        self._load_existing_logs()

        # Set up window close handler for graceful shutdown
        self.root.protocol("WM_DELETE_WINDOW", self.on_close)

        backend_pid = PROJECT_ROOT / "runtime/pids/backend.pid"
        frontend_pid = PROJECT_ROOT / "runtime/pids/frontend.pid"

        if self._is_running(backend_pid):
            threading.Thread(target=self._monitor_backend, daemon=True).start()

        if self._is_running(frontend_pid):
            threading.Thread(target=self._monitor_frontend, daemon=True).start()

        if self._is_running(MEMORY_PID):
            threading.Thread(target=self._monitor_memory, daemon=True).start()

        threading.Thread(target=self._monitor_synapses, daemon=True).start()

        self.update_master_ui()
        self._poll_status()
        threading.Thread(target=self._detect_gpu, daemon=True).start()


    def _load_existing_logs(self):
        backend_pid = PROJECT_ROOT / "runtime/pids/backend.pid"
        frontend_pid = PROJECT_ROOT / "runtime/pids/frontend.pid"

        # BACKEND
        if self._is_running(backend_pid):
            try:
                with open(PROJECT_ROOT / "runtime/backend.log", "r") as f:
                    for line in f.readlines()[-200:]:
                        self.log("BACKEND", line.rstrip())
            except:
                pass

        # FRONTEND
        if self._is_running(frontend_pid):
            try:
                with open(PROJECT_ROOT / "runtime/frontend.log", "r") as f:
                    for line in f.readlines()[-200:]:
                        self.log("FRONTEND", line.rstrip())
            except:
                pass



    def _init_log_tags(self):
        for console in [self.sys_console, self.front_console, self.back_console, self.mind_console, self.mem_console]:
            console.tag_config("SYSTEM", foreground="#ff00ea")
            console.tag_config("FRONTEND", foreground="#61dbfb")
            console.tag_config("BACKEND", foreground="#ffd43b")
            console.tag_config("MINDTRACE", foreground="#cc88ff")
            console.tag_config("MEMORY", foreground="#78e08f")
            console.tag_config("ERROR", foreground="#ff0000")
            console.tag_config("GPU_NVIDIA", foreground="#76b900")
            console.tag_config("GPU_AMD", foreground="#ed1c24")
            console.tag_config("GPU_VULKAN", foreground="#7b68ee")

    def _setup_custom_titlebar(self):
        self.title_bar = tk.Frame(self.root, bg="#2d2d2d", bd=0)
        self.title_bar.pack(fill=tk.X, side=tk.TOP)
        tk.Label(self.title_bar, text="NexusOS // AI CONTROL PANEL", bg="#2d2d2d", fg="#00ff00", font=("Consolas", 10, "bold"), padx=8, pady=3).pack(side=tk.LEFT)
        tk.Label(self.title_bar, text=f"v{VERSION}", bg="#2d2d2d", fg="#666666", font=("Consolas", 9), padx=8, pady=3).pack(side=tk.RIGHT)
        self.title_bar.bind("<Button-1>", self.start_move)
        self.title_bar.bind("<B1-Motion>", self.do_move)

    def start_move(self, event):
        self.x, self.y = event.x, event.y

    def do_move(self, event):
        self.root.geometry(f"+{self.root.winfo_x() + (event.x - self.x)}+{self.root.winfo_y() + (event.y - self.y)}")

    def _clear_all_logs(self):
        for console in [self.sys_console, self.front_console, self.back_console, self.mem_console, self.mind_console]:
            console.delete("1.0", tk.END)

    def _copy_selection(self, widget):
        """Copy selection or full console contents to the system clipboard."""
        try:
            content = widget.get(tk.SEL_FIRST, tk.SEL_LAST) if widget.tag_ranges(tk.SEL) else widget.get("1.0", tk.END)
            content = content.strip()
            if content:
                self.root.clipboard_clear()
                self.root.clipboard_append(content)
                self.root.update()
                self.log("SYSTEM", "Copied log contents to clipboard.")
        except Exception as e:
            self.log("ERROR", f"Clipboard Error: {e}")
        return "break"

    def _create_terminal(self, parent, label_text, height=10, text_color="#cccccc"):
        frame = tk.Frame(parent, bg="#1e1e1e", bd=0)
        header = tk.Frame(frame, bg="#1e1e1e", bd=0)
        header.pack(fill=tk.X, pady=(0, 1))
        tk.Label(header, text=label_text, bg="#1e1e1e", fg=text_color, font=("Consolas", 8, "bold")).pack(side=tk.LEFT)
        console = tk.Text(frame, bg="#000000", fg=text_color, font=("Consolas", 8), height=height, borderwidth=3, relief="flat", exportselection=True)
        tk.Button(header, text="CLEAR LOG", bg="#2d2d2d", fg="#ff0000", font=("Consolas", 7, "bold"), relief="flat", pady=2, command=lambda c=console: c.delete("1.0", tk.END)).pack(side=tk.RIGHT, padx=0)
        tk.Button(header, text="COPY LOG", bg="#2d2d2d", fg=text_color, font=("Consolas", 7, "bold"), relief="flat", pady=2, command=lambda c=console: self._copy_selection(c)).pack(side=tk.RIGHT, padx=4)
        console.pack(fill=tk.BOTH, expand=True)
        return frame, console

    def _setup_ui(self):
        controls = tk.Frame(self.root, bg="#1e1e1e")
        controls.pack(fill=tk.X, padx=10, pady=8)

        # Style shared across all STOP buttons
        STOP_STYLE = {"bg": "#331111", "fg": "#ff4444", "disabledforeground": "#552222", "relief": "flat", "font": ("Consolas", 9, "bold"), "pady": 3}
        START_STYLE = {"bg": "#2d2d2d", "relief": "flat", "font": ("Consolas", 9, "bold"), "pady": 3}

        # Clear All Logs
        tk.Button(controls, text="CLEAR ALL LOGS", bg="#2d2d2d", fg="#ff0000", relief="flat",
                  font=("Consolas", 9, "bold"), pady=3,
                  command=self._clear_all_logs).pack(side=tk.RIGHT, padx=(5, 0))

        # Master Controls
        self.btn_master_start = tk.Button(controls, text="SYSTEM START", fg="#00ff00", **START_STYLE, command=self.start_all)
        self.btn_master_start.pack(side=tk.LEFT, padx=4)
        self.btn_master_stop = tk.Button(controls, text="SYSTEM STOP", **STOP_STYLE, command=self.stop_all)
        self.btn_master_stop.pack(side=tk.LEFT, padx=4)
        self.btn_master_kill = tk.Button(controls, text="SYSTEM KILL", bg="#4a0000", fg="#ff2222",
                                         disabledforeground="#552222", relief="flat",
                                         font=("Consolas", 9, "bold"), pady=3, command=self.kill_all)
        self.btn_master_kill.pack(side=tk.LEFT, padx=4)

        tk.Label(controls, text=" | ", bg="#1e1e1e", fg="#444444").pack(side=tk.LEFT, padx=5)

        # Interface Controls
        self.btn_front_start = tk.Button(controls, text="INTERFACE", fg="#61dbfb", **START_STYLE, command=self.start_frontend)
        self.btn_front_start.pack(side=tk.LEFT, padx=2)
        self.btn_front_stop = tk.Button(controls, text="OFF", **STOP_STYLE, command=self.stop_frontend)
        self.btn_front_stop.pack(side=tk.LEFT, padx=2)
        self.btn_front_view = tk.Button(controls, text="VIEW", fg="#61dbfb", **START_STYLE, command=self.open_frontend_view)
        self.btn_front_view.pack(side=tk.LEFT, padx=2)

        # Synapse Controls
        self.btn_back_start = tk.Button(controls, text="SYNAPSE", fg="#ffd43b", **START_STYLE, command=self.start_backend)
        self.btn_back_start.pack(side=tk.LEFT, padx=(8, 2))
        self.btn_back_stop = tk.Button(controls, text="OFF", **STOP_STYLE, command=self.stop_backend)
        self.btn_back_stop.pack(side=tk.LEFT, padx=2)

        # Memory Service Controls
        self.btn_mem_start = tk.Button(controls, text="MEMORY", fg="#78e08f", **START_STYLE, command=self.start_memory)
        self.btn_mem_start.pack(side=tk.LEFT, padx=(8, 2))
        self.btn_mem_stop = tk.Button(controls, text="OFF", **STOP_STYLE, command=self.stop_memory)
        self.btn_mem_stop.pack(side=tk.LEFT, padx=2)

        # Terminal Displays
        sys_f, self.sys_console = self._create_terminal(self.root, " >> SYSTEM STATUS", height=5, text_color="#ff00ea")
        sys_f.pack(fill=tk.X, padx=10, pady=(0, 8))

        bottom_f = tk.Frame(self.root, bg="#1e1e1e")
        bottom_f.pack(fill=tk.BOTH, expand=True, padx=10, pady=(0, 8))
        for col in range(4):
            bottom_f.columnconfigure(col, weight=1, uniform="term")
        bottom_f.rowconfigure(0, weight=1)

        f_f, self.front_console = self._create_terminal(bottom_f, " >> INTERFACE (NPM)", text_color="#61dbfb")
        f_f.grid(row=0, column=0, sticky="nsew", padx=(0, 4))
        b_f, self.back_console = self._create_terminal(bottom_f, " >> SYNAPSE (UVICORN)", text_color="#ffd43b")
        b_f.grid(row=0, column=1, sticky="nsew", padx=4)
        mm_f, self.mem_console = self._create_terminal(bottom_f, " >> MEMORY SERVICE", text_color="#78e08f")
        mm_f.grid(row=0, column=2, sticky="nsew", padx=4)
        m_f, self.mind_console = self._create_terminal(bottom_f, " >> MINDTRACE", text_color="#cc88ff")
        m_f.grid(row=0, column=3, sticky="nsew", padx=(4, 0))

    def _safe_after(self, delay, func):
        """Schedule a Tk callback without ever raising.

        tkinter is not thread-safe: the worker/monitor threads call in here,
        and during shutdown (root being destroyed) an .after() from a thread
        raises RuntimeError/TclError. Swallowing it keeps a thread's `finally`
        (which unlinks its PID file) from being aborted mid-way — the cause of
        the crash-on-close and the stale runtime/pids/*.pid files it left."""
        if self._closing:
            return
        try:
            self.root.after(delay, func)
        except (RuntimeError, tk.TclError):
            pass

    def log(self, tag, message):
        def _log():
            con = self.sys_console
            if tag == "FRONTEND": con = self.front_console
            elif tag == "BACKEND": con = self.back_console
            elif tag == "MEMORY": con = self.mem_console
            elif tag == "MINDTRACE": con = self.mind_console
            con.insert(tk.END, f"[{tag}] {message}\n", tag)
            con.see(tk.END)
        self._safe_after(0, _log)

    def _detect_gpu(self):
        """Log a cosmetic colored GPU banner. Best-effort — label only."""
        def _run(cmd):
            try:
                r = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
                return r.stdout if r.returncode == 0 else None
            except (FileNotFoundError, subprocess.TimeoutExpired):
                return None

        out = _run(["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"])
        if out and out.strip():
            self.log("GPU_NVIDIA", f"GPU: NVIDIA — {out.strip().splitlines()[0]} (CUDA)")
            return

        out = _run(["rocm-smi", "--showproductname"])
        if out is not None:
            name = next((l.strip() for l in out.splitlines() if l.strip() and not l.startswith("=")), "")
            self.log("GPU_AMD", f"GPU: AMD — {name} (ROCm)" if name else "GPU: AMD (ROCm)")
            return

        out = _run(["vulkaninfo", "--summary"])
        if out is not None:
            name = next((l.split("=", 1)[1].strip() for l in out.splitlines()
                         if l.strip().startswith("deviceName")), "GPU")
            self.log("GPU_VULKAN", f"GPU: {name} (Vulkan)")
            return

        out = _run(["lspci"])  # hardware present but no usable drivers
        for line in (out or "").splitlines():
            ll = line.lower()
            if any(x in ll for x in ("vga", "3d controller", "display")):
                if "nvidia" in ll:
                    self.log("GPU_NVIDIA", "GPU: NVIDIA detected (no drivers)")
                    return
                if "amd" in ll or "radeon" in ll or "advanced micro" in ll:
                    self.log("GPU_AMD", "GPU: AMD detected (no drivers)")
                    return

        self.log("SYSTEM", "GPU: not detected")

    def _poll_status(self):
        self.update_master_ui()
        self._safe_after(2000, self._poll_status)

    def update_master_ui(self):
        if self._closing:  # window may be destroyed; stop_* still calls this
            return
        f, b, m = self._frontend_is_active(), self._backend_is_active(), self._memory_is_active()
        self.btn_master_start.config(state=tk.DISABLED if (f and b and m) else tk.NORMAL)
        self.btn_master_stop.config(state=tk.NORMAL if (f or b or m) else tk.DISABLED)
        self.btn_front_start.config(state=tk.DISABLED if f else tk.NORMAL, bg="#0b3d0b" if f else "#2d2d2d")
        self.btn_back_start.config(state=tk.DISABLED if b else tk.NORMAL, bg="#3d3d0b" if b else "#2d2d2d")
        self.btn_mem_start.config(state=tk.DISABLED if m else tk.NORMAL, bg="#0b2d14" if m else "#2d2d2d")
        self.btn_front_stop.config(state=tk.NORMAL if f else tk.DISABLED)
        self.btn_front_view.config(state=tk.NORMAL if f else tk.DISABLED)
        self.btn_back_stop.config(state=tk.NORMAL if b else tk.DISABLED)
        self.btn_mem_stop.config(state=tk.NORMAL if m else tk.DISABLED)

    def open_frontend_view(self):
        webbrowser.open_new_tab(FRONTEND_URL)

    def _spawn_process(self, cmd, cwd):
        env = os.environ.copy()
        env["PYTHONUNBUFFERED"] = "1"
        # If the command uses an absolute path (e.g. nvm npm), prepend its
        # bin directory to PATH so child processes (node, vite shebang) resolve
        # from the same installation instead of falling back to /usr/bin/node.
        if cmd and os.path.isabs(cmd[0]):
            bin_dir = str(Path(cmd[0]).parent)
            env["PATH"] = bin_dir + os.pathsep + env.get("PATH", "")
        kwargs = {
            "cwd": str(cwd),
            "stdout": subprocess.PIPE,
            "stderr": subprocess.STDOUT,
            "text": True,
            "bufsize": 1,
            "env": env,
        }
        if os.name == "nt":
            kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
        else:
            kwargs["preexec_fn"] = os.setsid
        return subprocess.Popen(cmd, **kwargs)

    def _terminate_process(self, process):
        try:
            if os.name == "nt":
                try:
                    process.send_signal(signal.CTRL_BREAK_EVENT)
                except Exception:
                    pass
                process.terminate()
            else:
                os.killpg(os.getpgid(process.pid), signal.SIGKILL)
        except Exception:
            try:
                process.kill()
            except Exception:
                pass

    def _write_pid(self, pid_file, pid):
        try:
            pid_file.parent.mkdir(parents=True, exist_ok=True)
            with open(pid_file, "w") as f:
                f.write(str(pid))
        except Exception as e:
            self.log("ERROR", f"PID write failed ({pid_file.name}): {e}")

    def _worker(self, name, cmd, cwd):
        try:
            p = self._spawn_process(cmd, cwd)
            if name == "FRONTEND":
                self._frontend_starting = False
                self.frontend_process = p
                self._write_pid(FRONTEND_PID, p.pid)
            elif name == "MEMORY":
                self._memory_starting = False
                self.memory_process = p
                self._write_pid(MEMORY_PID, p.pid)
            else:
                self._backend_starting = False
                self.backend_process = p
                self._write_pid(BACKEND_PID, p.pid)
            self._safe_after(0, self.update_master_ui)

            if not p.stdout:
                raise RuntimeError("Process has no stdout stream")

            while True:
                try:
                    line = p.stdout.readline()
                except ValueError:
                    break
                if not line:
                    break

                line = line.strip()
                if not line:
                    continue

                # Silence routine HTTP access logs
                if any(x in line for x in ["GET /", "POST /", "OPTIONS /"]):
                    continue

                # Normalize prefixes to consistent casing
                line = (line
                    .replace("[Ollama]", "[OLLAMA]")
                    .replace("[Synapse]", "[SYNAPSE]")
                )

                self.log(name, line)

            p.wait()
        except Exception as e:
            self.log("ERROR", f"{name} fault: {e}")
        finally:
            self.log("SYSTEM", f"{name} connection closed.")
            if name == "FRONTEND":
                self._frontend_starting = False
                self.frontend_process = None
                FRONTEND_PID.unlink(missing_ok=True)
            elif name == "MEMORY":
                self._memory_starting = False
                self.memory_process = None
                MEMORY_PID.unlink(missing_ok=True)
            else:
                self._backend_starting = False
                self.backend_process = None
                BACKEND_PID.unlink(missing_ok=True)
            self._safe_after(0, self.update_master_ui)

    def start_frontend(self):
        if self._frontend_is_active():
            return
        self._frontend_starting = True
        self.update_master_ui()
        self.log("SYSTEM", "Starting INTERFACE...")
        threading.Thread(target=self._worker, args=("FRONTEND", FRONTEND_CMD, FRONTEND_DIR), daemon=True).start()

    def stop_frontend(self):
        if self.frontend_process:
            self._terminate_process(self.frontend_process)
            self.frontend_process = None
        elif self._is_running(FRONTEND_PID):
            self._terminate_pid_file(FRONTEND_PID)
        self.update_master_ui()

    def start_backend(self):
        if self._backend_is_active():
            return
        self._backend_starting = True
        self.update_master_ui()
        self.log("SYSTEM", "Starting SYNAPSE...")
        threading.Thread(target=self._worker, args=("BACKEND", BACKEND_CMD, BACKEND_DIR), daemon=True).start()
        # Wait for backend to start, then start Ollama
        threading.Thread(target=self._start_ollama_after_backend, daemon=True).start()

    def _start_ollama_after_backend(self):
        """Wait for backend to be ready, then start Ollama.
        If the backend process dies before becoming reachable, stop Ollama."""
        max_retries = 30
        for attempt in range(max_retries):
            # Once the _worker has had a chance to set backend_process, check
            # whether it has already exited (import error, port conflict, etc.)
            if attempt >= 2 and self.backend_process is not None:
                if self.backend_process.poll() is not None:
                    self.log("SYSTEM", "Backend exited before becoming reachable — stopping OLLAMA.")
                    self._stop_ollama_direct()
                    return

            try:
                response = httpx.get("http://localhost:8000/", timeout=2)
                if response.status_code == 200:
                    time.sleep(1)
                    self.log("SYSTEM", "Starting OLLAMA...")
                    try:
                        httpx.post("http://localhost:8000/ollama/start", timeout=2)
                        self.log("SYSTEM", "OLLAMA startup command sent.")
                    except Exception as e:
                        self.log("SYSTEM", f"OLLAMA startup error: {e}")
                    return
            except (httpx.RequestError, ConnectionError):
                time.sleep(1)

        self.log("SYSTEM", "Backend failed to become reachable — stopping OLLAMA.")
        self._stop_ollama_direct()

    def _stop_ollama_direct(self):
        """Kill Ollama without the backend API (used when backend is not running)."""
        try:
            r = httpx.get("http://localhost:11434/api/tags", timeout=2.0)
            ollama_running = r.status_code == 200
        except Exception:
            ollama_running = False

        if not ollama_running:
            self.log("SYSTEM", "OLLAMA is not running — nothing to stop.")
            return

        try:
            result = subprocess.run(
                ["pkill", "-f", "ollama serve"],
                capture_output=True, timeout=5,
            )
            if result.returncode == 0:
                self.log("SYSTEM", "OLLAMA stopped.")
            else:
                self.log("SYSTEM", "OLLAMA process not found via pkill.")
        except Exception as e:
            self.log("SYSTEM", f"Could not stop OLLAMA: {e}")

    def stop_backend(self):
        self.log("SYSTEM", "Stopping OLLAMA...")
        try:
            httpx.post("http://localhost:8000/ollama/stop", timeout=5)
        except Exception:
            self._stop_ollama_direct()
        
        if self.backend_process:
            self._terminate_process(self.backend_process)
            self.backend_process = None
        elif self._is_running(BACKEND_PID):
            self._terminate_pid_file(BACKEND_PID)
        self.update_master_ui()

    def start_memory(self):
        if self._memory_is_active():
            return
        self._memory_starting = True
        self.update_master_ui()
        self.log("SYSTEM", "Starting MEMORY SERVICE...")
        threading.Thread(target=self._worker, args=("MEMORY", MEMORY_CMD, MEMORY_DIR), daemon=True).start()

    def stop_memory(self):
        if self.memory_process:
            self._terminate_process(self.memory_process)
            self.memory_process = None
        elif self._is_running(MEMORY_PID):
            self._terminate_pid_file(MEMORY_PID)
        self.update_master_ui()

    def _memory_is_active(self):
        return self._memory_starting or self.memory_process is not None or self._is_running(MEMORY_PID)

    def _monitor_memory(self):
        log_path = PROJECT_ROOT / "runtime/memory.log"
        try:
            with open(log_path, "r") as f:
                f.seek(0, os.SEEK_END)
                while self._is_running(MEMORY_PID):
                    line = f.readline()
                    if line:
                        self.log("MEMORY", line.rstrip())
                    else:
                        time.sleep(0.1)
        except Exception as e:
            self.log("ERROR", f"MEMORY monitor fault: {e}")

    def start_all(self):
        self.start_memory()
        self.start_frontend()
        self.start_backend()

    def kill_all(self):
        self.log("SYSTEM", "FORCE KILL — terminating all Nexus processes by port...")
        ports = {8000: "SYNAPSE", 8001: "MEMORY", 5173: "INTERFACE"}
        for port, name in ports.items():
            try:
                r = subprocess.run(["fuser", "-k", f"{port}/tcp"], capture_output=True, timeout=5)
                self.log("SYSTEM", f"{name} (:{port}) {'killed' if r.returncode == 0 else 'not running'}")
            except Exception as e:
                self.log("ERROR", f"  Kill :{port} failed: {e}")
        for pat in ["uvicorn synapse", "npm run dev"]:
            try:
                subprocess.run(["pkill", "-9", "-f", pat], capture_output=True, timeout=5)
            except Exception:
                pass
        for pid_file in [FRONTEND_PID, BACKEND_PID, MEMORY_PID]:
            pid_file.unlink(missing_ok=True)
        self.frontend_process = None
        self.backend_process = None
        self.memory_process = None
        self._frontend_starting = False
        self._backend_starting = False
        self._memory_starting = False
        self._safe_after(0, self.update_master_ui)
        self.log("SYSTEM", "Force kill complete.")

    def stop_all(self):
        self.stop_frontend()
        self.stop_backend()
        self.stop_memory()

    def on_close(self):
        if self._closing:
            return
        self._closing = True
        # Destroy the window FIRST so closing is instant. The teardown used to
        # run with the window still up, and stop_backend's blocking calls (a 5s
        # httpx.post to a backend stop_frontend had just killed, plus httpx +
        # pkill) froze it into "not responding" — the crash you saw with
        # services running. Destroy, then tear down fast and non-blocking.
        try:
            self.root.destroy()
        except Exception:
            pass
        try:
            self._force_teardown()
        except Exception:
            pass
        os._exit(0)

    def _force_teardown(self):
        """Fast, non-blocking shutdown for window-close. No HTTP calls: the
        window is already gone, and the graceful /ollama/stop endpoint (used by
        the STOP button) blocks for seconds against a backend we're killing."""
        # Ollama runs in its own session (start_new_session=True), so the group
        # kills below won't reach it — signal it directly, by binary path so we
        # don't match unrelated processes. Popen (no wait) keeps this instant.
        try:
            subprocess.Popen(["pkill", "-TERM", "-f", "ollama/bin/ollama"])
        except Exception:
            pass
        # Services the panel spawned itself (each in its own setsid group):
        for proc in (self.frontend_process, self.backend_process, self.memory_process):
            if proc:
                self._terminate_process(proc)
        # Services attached via pid file (ncp-started, one shared group);
        # _terminate_pid_file verifies ownership and unlinks. Guarantee the
        # unlink even for a pid that's already dead so no stale files remain.
        for pf in (FRONTEND_PID, BACKEND_PID, MEMORY_PID):
            if self._is_running(pf):
                self._terminate_pid_file(pf)
            else:
                pf.unlink(missing_ok=True)
        
    def _frontend_is_active(self):
        return self._frontend_starting or self.frontend_process is not None or self._is_running(FRONTEND_PID)

    def _backend_is_active(self):
        return self._backend_starting or self.backend_process is not None or self._is_running(BACKEND_PID)

    def _pid_is_ours(self, pid):
        """True only if the live PID is actually one of our services. PID files
        outlive the process; the OS recycles the number onto something unrelated
        (a desktop app, a shell — or this panel), and killpg on a recycled PID's
        group takes down the wrong processes. Mirrors nexus-cli.sh's guard."""
        try:
            with open(f"/proc/{pid}/cmdline", "rb") as f:
                cmd = f.read().replace(b"\0", b" ").decode("utf-8", "replace")
        except OSError:
            return False
        return any(m in cmd for m in ("uvicorn synapse", "npm run dev", "vite"))

    def _terminate_pid_file(self, pid_file):
        try:
            with open(pid_file, "r") as f:
                pid = int(f.read().strip())
        except Exception:
            pid_file.unlink(missing_ok=True)
            return
        try:
            if os.name == "nt":
                try:
                    os.kill(pid, signal.CTRL_BREAK_EVENT)
                except Exception:
                    pass
                os.kill(pid, signal.SIGKILL)
            elif self._pid_is_ours(pid):
                try:
                    pgid = os.getpgid(pid)
                    # Never killpg our OWN group — if a service was launched into
                    # the panel's process group, nuking the group SIGKILLs the
                    # panel mid-close. Kill just the service pid in that case.
                    if pgid == os.getpgrp():
                        os.kill(pid, signal.SIGKILL)
                    else:
                        os.killpg(pgid, signal.SIGKILL)
                except (ProcessLookupError, OSError):
                    os.kill(pid, signal.SIGKILL)
            # else: PID recycled onto a non-service — don't kill the wrong thing
        except Exception:
            pass
        pid_file.unlink(missing_ok=True)

    def _is_running(self, pid_file):
        try:
            with open(pid_file, "r") as f:
                pid = int(f.read().strip())
            os.kill(pid, 0)  # Check if process exists
            return True
        except:
            return False
    
    def _monitor_backend(self):
        log_path = PROJECT_ROOT / "runtime/backend.log"
        try:
            with open(log_path, "r") as f:
                f.seek(0, os.SEEK_END)  # Start at end of file
                while self._is_running(PROJECT_ROOT / "runtime/pids/backend.pid"):
                    line = f.readline()
                    if line:
                        self.log("BACKEND", line.rstrip())
                    else:
                        time.sleep(0.1)
        except Exception as e:
            self.log("ERROR", f"BACKEND fault: {e}")

    def _monitor_frontend(self):
        log_path = PROJECT_ROOT / "runtime/frontend.log"
        try:
            with open(log_path, "r") as f:
                f.seek(0, os.SEEK_END)
                while self._is_running(PROJECT_ROOT / "runtime/pids/frontend.pid"):
                    line = f.readline()
                    if line:
                        self.log("FRONTEND", line.rstrip())
                    else:
                        time.sleep(0.1)
        except Exception as e:
            self.log("ERROR", f"FRONTEND fault: {e}")

    def _monitor_synapses(self):
        while True:
            try:
                if not CHAT_LOG.exists():
                    time.sleep(2)
                    continue
                with open(CHAT_LOG, "r", encoding="utf-8") as f:
                    f.seek(0, os.SEEK_END)
                    while True:
                        line = f.readline()
                        if line:
                            stripped = line.rstrip()
                            if stripped:
                                self.log("MINDTRACE", stripped)
                        else:
                            time.sleep(0.05)
            except Exception as e:
                self.log("ERROR", f"MINDTRACE monitor fault: {e}")
                time.sleep(2)

PANEL_LOG = PROJECT_ROOT / "runtime" / "logs" / "panel.log"

def _log_crash(where, exc_type, exc_value, exc_tb):
    """Persist tracebacks — the .desktop launcher runs Terminal=false, so
    stderr is discarded and crashes are otherwise invisible. Covers the three
    surfaces tk hides: main thread, worker threads, and Tk callbacks."""
    import traceback, datetime
    try:
        PANEL_LOG.parent.mkdir(parents=True, exist_ok=True)
        with open(PANEL_LOG, "a") as f:
            f.write(f"\n[{datetime.datetime.now().isoformat()}] {where}\n")
            traceback.print_exception(exc_type, exc_value, exc_tb, file=f)
    except Exception:
        pass
    traceback.print_exception(exc_type, exc_value, exc_tb)

if __name__ == "__main__":
    sys.excepthook = lambda et, ev, tb: _log_crash("main-thread uncaught", et, ev, tb)
    threading.excepthook = lambda a: _log_crash("thread uncaught", a.exc_type, a.exc_value, a.exc_traceback)
    root = tk.Tk()
    root.report_callback_exception = lambda et, ev, tb: _log_crash("tk callback", et, ev, tb)
    app = NexusControlPanel(root)
    root.mainloop()