diff --git a/.gitea/workflows/package.yml b/.gitea/workflows/package.yml new file mode 100644 index 0000000..37782b3 --- /dev/null +++ b/.gitea/workflows/package.yml @@ -0,0 +1,74 @@ +name: package + +on: + push: + branches: [main, code-preview] + tags: ["v*"] + pull_request: + +jobs: + wheel: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: interface/web/package-lock.json + + - name: Build and test web UI + working-directory: interface/web + run: | + npm ci + npm run lint + npm test + npm run build + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Test Python runtime + run: | + python -m pip install -e ".[dev]" + python -m pytest -q tests management + for f in scripts/install-termux.sh bin/check.sh management/nexus-cli.sh; do bash -n "$f"; done + + - name: Build wheel and sdist + run: | + python -m build + python -m twine check dist/* + + - name: Verify clean wheel install + run: | + python -m venv "$RUNNER_TEMP/nexus-wheel" + "$RUNNER_TEMP/nexus-wheel/bin/python" -m pip install dist/*.whl + cd "$RUNNER_TEMP" + export NEXUS_HOME="$RUNNER_TEMP/nexus-home" + export NEXUS_CONFIG_DIR="$RUNNER_TEMP/nexus-config" + "$RUNNER_TEMP/nexus-wheel/bin/nexus" init --json + "$RUNNER_TEMP/nexus-wheel/bin/nexus" doctor --json + "$RUNNER_TEMP/nexus-wheel/bin/python" -c "from synapse.main import sio_app; assert sio_app" + # The wheel must carry the compiled UI, not just import cleanly. + "$RUNNER_TEMP/nexus-wheel/bin/python" - <<'PY' + from synapse.nexus_config import settings + index = settings.web_dist_dir / "index.html" + assert index.is_file(), f"wheel shipped no web UI at {index}" + PY + + - uses: actions/upload-artifact@v4 + with: + name: nexusos-python-dist + path: dist/* + + - name: Publish tagged release to PyPI + if: startsWith(gitea.ref, 'refs/tags/v') + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: python -m twine upload --non-interactive dist/* diff --git a/.gitignore b/.gitignore index 47fbcc3..b88e465 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ Promethean/ ollama/ interface/web/node_modules/ interface/web/dist/ +/.build-check/ +/dist/ +/*.egg-info/ runtime/ __pycache__/ *.pyc diff --git a/CLAUDE.md b/CLAUDE.md index d917432..824d045 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ access to the same features as the web UI (all via the REST API on `:8000`): ./management/nexus-cli.sh stop ./management/nexus-cli.sh start --backend|-b / --frontend|-f / --memory|-m -# Feature commands (dispatch to management/nexus_api.py — httpx, no TUI): +# Feature commands (dispatch to nexusos_cli/nexus_api.py — httpx, no TUI): ncp chat "" # stream a reply (POST /chat/stream) ncp memory list|add |rm ncp playbook list|show # first playbook (*) is the active system prompt @@ -66,12 +66,15 @@ ncp history [query] # recent conversations The old curses TUIs (`nexus-chat.py`, `nexus-playbook.py`) were removed in favor of these API-backed subcommands. The CLI covers chat, memory, playbooks, and history; the web UI and control panel expose the remaining management features. +The CLI itself lives in `nexusos_cli/` (that is what the wheel ships and what +`nexus`/`ncp`/`nexusos` dispatch to); `management/` keeps the desktop-only +pieces — the shell wrappers, the Tk control panel, and the XFCE panel wiring. `management/controlpanel.py` (tkinter GUI, wired into the XFCE panel via `bin/panel/nexus-popup.py`) stays. **Checks (the release gate):** ```bash -./bin/check.sh # pytest (tests/ + management/) + eslint + .ps1 parse check +./bin/check.sh # pytest + eslint + frontend tests + .ps1/.sh parse + wheel build ``` There is no hosted CI — the remote is self-hosted Gitea with no act_runner — so this script *is* the gate. Run it before tagging a release. diff --git a/README.md b/README.md index 3a674bf..f2d22a4 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,10 @@ # NexusOS **A local-first AI assistant platform.** Runs entirely on your machine — a -Python/FastAPI backend, a bundled Ollama instance for inference, persistent -memory, and a React frontend. No external AI provider is called. +Python/FastAPI backend, an Ollama-compatible endpoint for inference, a +persistent memory service, and a React frontend. Ollama is local by default; +Termux and container installs can explicitly point at a separately managed +endpoint. @@ -15,10 +17,12 @@ memory, and a React frontend. No external AI provider is called. ## What it is NexusOS ("Nexus") is a self-hosted assistant you actually own. All inference -runs through a **locally bundled Ollama** on `localhost`; conversations, facts, -and settings live in local SQLite. It ships with desktop branding (XFCE theme, +runs through **Ollama** on `localhost` by default; conversations, facts, and +settings live in local SQLite. It ships with desktop branding (XFCE theme, icons, boot splash) so it can be run as a full assistant environment on Linux, -not just a web app. +not just a web app. A remote Ollama-compatible URL is an explicit configuration +option for lightweight clients; NexusOS never starts or stops that remote +process. Chat with vision and voice, persistent memory, tool-using playbooks, document RAG scoped to Projects, gated action tools, and full model management — see @@ -88,6 +92,29 @@ NexusOS runs **single-process**: the backend on `:8000` serves the built web UI itself, so there's no separate frontend server at runtime. Ollama is started manually from the app (**Start AI** in the sidebar), not at boot. +### Python package and portable CLI + +The portable package installs `nexus`, `ncp`, and `nexusos` as equivalent +commands. From a checkout today: + +```bash +cd interface/web && npm ci && npm run build && cd ../.. # compile the UI +python -m pip install -e ".[standard]" +nexus init +nexus doctor +nexus serve +``` + +After a package release, the install becomes `python -m pip install +"nexusos-ai[standard]"`. The wheel includes the compiled web UI and default +playbooks; it keeps writable state outside `site-packages`. See +[the CLI reference](docs/CLI.md) for commands, configuration, and dependency +profiles. + +Termux uses the base package with a remote Ollama-compatible provider. Its +bootstrap and the current Android native-wheel gate are documented in +[the Termux guide](docs/TERMUX.md). + ### Linux Nexus was built using an Apple T2 computer running Linux Mint XFCE. The desktop @@ -234,7 +261,8 @@ are the exception (YAML files in `data/playbooks/`). All paths are defined in - `synapse/` — FastAPI backend + memory curator + playbook/ollama managers - `modules/` — auto-discovered feature plugins - `interface/web/` — React + Vite frontend -- `management/` — nexus-cli.sh, ncp API client, control panel, desktop theme +- `nexusos_cli/` — the portable CLI the wheel ships (`nexus`/`ncp`/`nexusos`) +- `management/` — nexus-cli.sh wrapper, control panel, desktop theme - `bin/` — install, backup/restore, panel + provisioning scripts - `assets/` — branding: icons, boot splash, XFCE/GTK theme - `data/playbooks/` — active playbook YAML diff --git a/bin/check.sh b/bin/check.sh index c060d53..8a64390 100644 --- a/bin/check.sh +++ b/bin/check.sh @@ -49,5 +49,39 @@ else echo "-- skipped: pwsh not installed" fi +echo "== shell parse ==" +for f in scripts/install-termux.sh launch_nexus.sh management/nexus-cli.sh; do + [ -f "$f" ] && { bash -n "$f" || fail=1; } +done + +echo "== packaging ==" +# The wheel is the other shippable artifact, so it belongs in the same gate: +# a broken pyproject or a missing web build only shows up at build time. +if Promethean/bin/python -c "import build, twine" 2>/dev/null; then + rm -rf .build-check + if Promethean/bin/python -m build --outdir .build-check >/dev/null 2>&1; then + Promethean/bin/python -m twine check .build-check/* || fail=1 + # The compiled UI has to actually be inside the wheel - a wheel that + # builds but ships no dist/ serves a blank page. + Promethean/bin/python - <<'PY' || fail=1 +import glob, sys, zipfile +wheels = glob.glob(".build-check/*.whl") +if not wheels: + sys.exit("no wheel produced") +names = zipfile.ZipFile(wheels[0]).namelist() +if not any(n.startswith("synapse/_resources/web/") for n in names): + sys.exit("wheel is missing the compiled web UI (cd interface/web && npm run build)") +if not any(n.startswith("synapse/_resources/playbooks/") for n in names): + sys.exit("wheel is missing the seed playbooks") +print(f"wheel OK: {len(names)} files") +PY + else + echo "!! wheel build failed"; fail=1 + fi + rm -rf .build-check +else + echo "-- skipped: build/twine missing (pip install -e '.[dev]')" +fi + [ "$fail" -eq 0 ] && echo "OK" || echo "FAILED" exit "$fail" diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 0000000..440a7a8 --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,118 @@ +# NexusOS CLI + +The Python package installs three equivalent command names: `nexus`, `ncp`, +and `nexusos`. New documentation uses `nexus`; `ncp` remains available for +existing desktop installs and scripts. Legacy spellings such as `ncp web`, +`ncp start -b`, `ncp refresh`, `ncp backup`, and `ncp restore` remain supported; +checkout-specific operations report a clear error when invoked from a wheel. + +## Install + +From a source checkout. Build the web UI first - it is a Vite artifact, so a +fresh clone does not have it, and an install without it serves the API only: + +```bash +cd interface/web && npm ci && npm run build && cd ../.. +python -m pip install -e ".[standard]" +nexus init +nexus doctor +``` + +From the package index after a release is published: + +```bash +python -m pip install "nexusos-ai[standard]" +nexus init +nexus serve +``` + +The base install contains the backend, memory service, compiled web UI, CLI, +and seed playbooks. Extras keep platform-sensitive dependencies optional: + +- `standard`: documents, vector search, web search, and process control +- `documents`: PDF and DOCX ingestion +- `vector`: sqlite-vec semantic indexes +- `voice`: local faster-whisper transcription +- `process`: psutil-backed process and port inspection +- `desktop`: desktop process support and Windows pywebview +- `search`: DuckDuckGo web search for chat +- `mail`: IMAP mail reading +- `all`: every optional capability at once + +## Common commands + +```text +nexus init Create writable state and seed playbooks +nexus doctor [--fix] [--json] Diagnose the install and provider +nexus paths [--json] Show package, state, and asset locations +nexus status [--json] Show services and provider reachability +nexus monitor [--once] [--json] ASCII live dashboard (services, resources, tools) +nexus serve Run backend + memory in the foreground +nexus start|stop|restart Manage background services +nexus open Open the compiled web interface +nexus logs [service] --follow Tail service logs +nexus models list|pull|remove Manage Ollama-compatible models +nexus config list|get|set|unset Manage persistent settings +``` + +API commands are also available directly: + +```bash +nexus chat send "Hello" +nexus history list +nexus memory list +nexus playbook list +``` + +Run `nexus COMMAND --help` for command-specific arguments. + +## Providers + +Local desktop installs can allow NexusOS to start and stop a local Ollama: + +```bash +nexus provider use local +``` + +For Termux, containers, or a separate inference machine, configure a remote +Ollama-compatible endpoint. NexusOS probes it but never manages its process: + +```bash +nexus provider use remote --url http://192.168.1.20:11434 +nexus provider show --json +``` + +## State and configuration + +Installed wheels never write into `site-packages`. Writable files use the +platform data directory, while configuration uses the platform config +directory. Inspect the exact locations with `nexus paths`. + +Environment variables override persisted settings. The most useful are: + +Persisted keys are the same names, minus the `NEXUS_` prefix - `nexus config +set home /data/nexus` matches `NEXUS_HOME`. `nexus config list` shows what is +set; `nexus config set` warns when a change would point NexusOS at a database +that does not exist yet (the file is never moved for you). + +```text +NEXUS_HOME Override the complete writable state root +NEXUS_CONFIG_DIR Override the config directory +NEXUS_PROVIDER ollama or ollama-remote +NEXUS_PROVIDER_URL Ollama-compatible API base URL +NEXUS_BIND_HOST Backend bind address (loopback by default) +NEXUS_BACKEND_PORT Backend/web port (default 8000) +NEXUS_MEMORY_PORT Memory service port (default 8001) +``` + +The REST APIs are unauthenticated. `nexus serve` refuses non-loopback binds +unless `--allow-lan` is given; that flag is an explicit acknowledgement, not +an authentication layer. + +`--allow-lan` widens the accepted `Host` headers and CORS origins to the +addresses the bind actually answers on - it does **not** set them to `*`. +That keeps `TrustedHostMiddleware` enforcing something, which is what stops a +web page you visit from resolving a name it controls to your machine and +driving the API through your browser. Export `NEXUS_ALLOWED_HOSTS` yourself if +you genuinely need a blanket, and understand that anyone who can reach the +port has full admin and data access. diff --git a/docs/TERMUX.md b/docs/TERMUX.md new file mode 100644 index 0000000..3c94f7d --- /dev/null +++ b/docs/TERMUX.md @@ -0,0 +1,49 @@ +# Termux installation path + +NexusOS is packaged so its Python runtime, memory database, playbooks, and +compiled web UI can run without a source checkout or Node.js. Inference is +configured separately through an Ollama-compatible HTTP endpoint; NexusOS does +not attempt to manage that remote process. + +## Bootstrap + +After `nexusos-ai` and a compatible Android `pydantic-core` wheel are published: + +```bash +curl -fsSLO https://git.enderofwings.com/enderofwings/NexusOS/raw/branch/main/scripts/install-termux.sh +chmod +x install-termux.sh +NEXUS_ANDROID_WHEEL_INDEX=https://packages.example.invalid/android/simple \ + ./install-termux.sh +``` + +For a local release artifact, pass the wheel path or URL as the first argument: + +```bash +NEXUS_ANDROID_WHEEL_INDEX=https://packages.example.invalid/android/simple \ + ./scripts/install-termux.sh ./dist/nexusos_ai-1.0.0-py3-none-any.whl +``` + +Then configure inference and serve the UI: + +```bash +nexus provider use remote --url http://192.168.1.20:11434 +nexus serve +termux-open-url http://127.0.0.1:8000 +``` + +## Native wheel gate + +Current Termux Python is 3.14, so Pydantic 1 is not a safe fallback. Pydantic 2 +depends on the Rust-based `pydantic-core`. PyPI publishes Linux, macOS, Windows, +and WebAssembly wheels but no Android wheel, while the current Termux Rust +package cannot build common Rust extensions on-device. + +The bootstrap script therefore requires a binary `pydantic-core` and accepts a +trusted PEP 503 wheel index through `NEXUS_ANDROID_WHEEL_INDEX`. It fails early +with the detected Python ABI and CPU when that artifact is missing. The release +pipeline can publish the pure NexusOS wheel today; an Android wheel job/index is +the remaining prerequisite for a one-line public Termux install. + +Do not work around this by downloading an unverified binary or by exposing the +NexusOS server with `--allow-lan`. Keep the UI on loopback and let the Android +browser connect to `127.0.0.1`. diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..f5e8b1a --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,55 @@ +"""Build hook that ships the compiled web UI without breaking `pip install -e .`. + +interface/web/dist is gitignored - it is a Vite build artifact, not source - so +a fresh clone does not have it. A static force-include of a missing path aborts +the build, which would make the README's first step ("pip install -e .") fail +before the reader ever gets to `npm run build`. + +So the include is decided here instead: + * editable install -> skip a missing dist, the UI just isn't served yet + * wheel / sdist -> hard error naming the exact command to run + +Set NEXUS_ALLOW_UILESS_BUILD=1 to build a deliberately headless distribution +(API and CLI only). +""" +from __future__ import annotations + +import os +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +_UI_SOURCE = Path("interface") / "web" / "dist" +_UI_TARGETS = { + "wheel": "synapse/_resources/web", + "sdist": "interface/web/dist", +} + + +class NexusBuildHook(BuildHookInterface): + PLUGIN_NAME = "custom" + + def initialize(self, version: str, build_data: dict) -> None: + target = _UI_TARGETS.get(self.target_name) + if target is None: + return + + source = Path(self.root) / _UI_SOURCE + if (source / "index.html").is_file(): + build_data.setdefault("force_include", {})[str(source)] = target + return + + if version == "editable" or os.getenv("NEXUS_ALLOW_UILESS_BUILD") == "1": + self.app.display_warning( + f"No compiled web UI at {_UI_SOURCE} - the backend will serve the " + "API only. Build it with: cd interface/web && npm ci && npm run build" + ) + return + + raise RuntimeError( + f"Cannot build a {self.target_name}: the compiled web UI is missing from " + f"{_UI_SOURCE}.\n" + "Build it first:\n" + " cd interface/web && npm ci && npm run build\n" + "Or set NEXUS_ALLOW_UILESS_BUILD=1 to ship an API/CLI-only distribution." + ) diff --git a/management/__init__.py b/management/__init__.py new file mode 100644 index 0000000..e360e23 --- /dev/null +++ b/management/__init__.py @@ -0,0 +1,5 @@ +"""Desktop-only NexusOS management helpers (Tk control panel, XFCE panel). + +The portable CLI lives in the `nexusos_cli` package - that is what the +wheel ships and what `ncp`/`nexus`/`nexusos` dispatch to. +""" diff --git a/management/ncp.cmd b/management/ncp.cmd index 08c3de2..6f447c1 100644 --- a/management/ncp.cmd +++ b/management/ncp.cmd @@ -12,17 +12,20 @@ REM exactly what this file exists to avoid. REM REM ASCII only, same rule as the .ps1 files - a test in tests/test_smoke.py enforces it. setlocal -REM ncp.py pins its stdout to UTF-8 (it prints check marks and em-dashes, and a +REM The CLI pins its stdout to UTF-8 (it prints check marks and em-dashes, and a REM redirected stdout would otherwise raise UnicodeEncodeError). A console still REM on codepage 437 renders those bytes as mojibake, so switch it to UTF-8 here. REM ponytail: chcp changes the calling console's codepage and does not restore REM it on exit. Harmless in practice; if that ever matters, set the console CP -REM from inside ncp.py with ctypes SetConsoleOutputCP instead. +REM from inside the CLI with ctypes SetConsoleOutputCP instead. chcp 65001 >nul 2>&1 set "ROOT=%~dp0.." REM System Python fallback so backup/restore work before the venv exists; those REM delegate to the stdlib-only bin/sync.py. Mirrors the same fallback in ncp.ps1. set "PY=%ROOT%\Promethean\Scripts\python.exe" if not exist "%PY%" set "PY=python" -"%PY%" "%ROOT%\management\ncp.py" %* -exit /b %ERRORLEVEL% +pushd "%ROOT%" +"%PY%" -m nexusos_cli.cli %* +set "NEXUS_EXIT=%ERRORLEVEL%" +popd +exit /b %NEXUS_EXIT% diff --git a/management/nexus-cli.sh b/management/nexus-cli.sh index c8f3b04..4f452a9 100644 --- a/management/nexus-cli.sh +++ b/management/nexus-cli.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# ncp - Linux entry point. The CLI itself is management/ncp.py, which runs +# ncp - Linux entry point. The CLI itself is nexusos_cli/cli.py, which runs # unchanged on Windows too (see management/ncp.cmd); this stays a shell script # because ~/.bashrc, launch_nexus.sh, bin/restore-linux.sh, controlpanel.py, # bin/panel/nexus-popup.py and management/nexus-app.sh all invoke this path. @@ -8,9 +8,10 @@ # directory drives itself instead of reaching into the real install. NEXUS_ROOT="$(cd "$(dirname "$(realpath "$0")")/.." && pwd)" -# ncp.py needs psutil (venv), but its backup/restore path delegates to the -# stdlib-only bin/sync.py and must work before the venv is built. +# Prefer the project venv for the full desktop dependency set. The portable CLI +# falls back to system Python when the package is installed without that venv. PY="$NEXUS_ROOT/Promethean/bin/python3" [ -x "$PY" ] || PY=python3 -exec "$PY" "$NEXUS_ROOT/management/ncp.py" "$@" +cd "$NEXUS_ROOT" +exec "$PY" -m nexusos_cli.cli "$@" diff --git a/management/test_controlpanel_close.py b/management/test_controlpanel_close.py index ceb0268..8a9d4de 100644 --- a/management/test_controlpanel_close.py +++ b/management/test_controlpanel_close.py @@ -11,7 +11,10 @@ import pytest pytest.importorskip("tkinter", reason="controlpanel is a Tk GUI; headless boxes lack python3-tk") -from controlpanel import NexusControlPanel # noqa: E402 +try: + from management.controlpanel import NexusControlPanel # noqa: E402 +except ModuleNotFoundError: # direct: python management/test_controlpanel_close.py + from controlpanel import NexusControlPanel # type: ignore[no-redef] # noqa: E402 def _fake(closing, after_raises): diff --git a/nexusos_cli/__init__.py b/nexusos_cli/__init__.py new file mode 100644 index 0000000..4a16cd7 --- /dev/null +++ b/nexusos_cli/__init__.py @@ -0,0 +1,7 @@ +"""The portable NexusOS command line, shipped in the wheel. + +Kept out of `management/` so the installed distribution does not claim a +top-level `management` package name in site-packages. `management/` stays in +the source checkout for the desktop-only pieces - the Tk control panel, the +shell wrappers, the XFCE panel and .desktop wiring. +""" diff --git a/nexusos_cli/cli.py b/nexusos_cli/cli.py new file mode 100644 index 0000000..66dadde --- /dev/null +++ b/nexusos_cli/cli.py @@ -0,0 +1,815 @@ +"""Portable NexusOS command line used by the ``nexus`` and ``ncp`` scripts.""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +import webbrowser +from pathlib import Path +from urllib.parse import urlparse + +from synapse import nexus_config as config +from synapse.nexus_config import settings + +from . import ncp as services + + +CONFIG_SCHEMA = { + "home": "path", + "api_url": "url", + "bind_host": "text", + "backend_port": "port", + "provider": "provider", + "provider_url": "url", + "provider_timeout": "positive_int", + "data_dir": "path", + "models_dir": "path", + "runtime_dir": "path", + "memory_dir": "path", + "memory_db": "path", +} + +LEGACY_TARGETS = { + "-b": "backend", + "--backend": "backend", + "-f": "frontend", + "--frontend": "frontend", + "-a": "ai", + "--ai": "ai", +} + + +def _emit(payload, json_output: bool = False) -> None: + if json_output: + print(json.dumps(payload, indent=2, sort_keys=True)) + elif isinstance(payload, str): + print(payload) + else: + for key, value in payload.items(): + print(f"{key}: {value}") + + +def _http_ok(url: str, timeout: float = 1.0) -> bool: + try: + urllib.request.urlopen(url, timeout=timeout).read(1) + return True + except urllib.error.HTTPError: + return True + except Exception: + return False + + +def _is_termux() -> bool: + prefix = os.getenv("PREFIX", "") + return "com.termux" in prefix or bool(os.getenv("TERMUX_VERSION")) + + +def _validate_config(key: str, raw: str): + kind = CONFIG_SCHEMA[key] + value = raw.strip() + if kind == "url": + parsed = urlparse(value) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ValueError(f"{key} must be an http(s) URL") + return value.rstrip("/") + if kind == "port": + number = int(value) + if not 1 <= number <= 65535: + raise ValueError(f"{key} must be between 1 and 65535") + return number + if kind == "positive_int": + number = int(value) + if number <= 0: + raise ValueError(f"{key} must be greater than zero") + return number + if kind == "provider": + if value not in ("ollama", "ollama-remote"): + raise ValueError("provider must be ollama or ollama-remote") + return value + if kind == "path": + return str(Path(value).expanduser().resolve()) + if not value: + raise ValueError(f"{key} cannot be empty") + return value + + +def cmd_init(args) -> int: + config.CONFIG_DIR.mkdir(parents=True, exist_ok=True) + copied = list(config.INITIALIZED_FILES) + config.init_state() + payload = { + "status": "initialized", + "state_dir": str(settings.state_dir), + "config_file": str(settings.config_file), + "data_dir": str(settings.data_dir), + "models_dir": str(settings.models_dir), + "runtime_dir": str(settings.runtime_dir), + "seeded_playbooks": len(copied), + } + _emit(payload, args.json) + return 0 + + +def cmd_paths(args) -> int: + payload = { + "install_mode": "checkout" if settings.source_checkout else "wheel", + "project_root": str(settings.project_root), + "resource_root": str(settings.resource_root), + "state_dir": str(settings.state_dir), + "config_file": str(settings.config_file), + "data_dir": str(settings.data_dir), + "memory_db": str(settings.memory_db), + "models_dir": str(settings.models_dir), + "runtime_dir": str(settings.runtime_dir), + "web_dist_dir": str(settings.web_dist_dir), + } + _emit(payload, args.json) + return 0 + + +# Keys whose value decides where the SQLite database is looked up. Changing one +# does not move the file, so the next start would quietly open a fresh, empty +# database and the user's conversations and memories would look deleted. +_DB_LOCATION_KEYS = ("memory_db", "memory_dir", "data_dir", "home") + + +def _relocation_warning(key: str, value) -> str | None: + """Warn when a config change points the database somewhere with no data.""" + if key not in _DB_LOCATION_KEYS: + return None + current = settings.memory_db + if not current.is_file(): + return None + if key == "memory_db": + new_db = Path(str(value)) + elif key == "memory_dir": + new_db = Path(str(value)) / current.name + else: + # data_dir/home only decide the DB location when nothing more specific + # does, and only in the layout where the DB actually sits under them - + # a source checkout keeps it in synapse/memory/ regardless. + existing = config.read_user_config() + if "memory_dir" in existing or "memory_db" in existing: + return None + anchor = settings.data_dir if key == "data_dir" else settings.state_dir + try: + tail = current.resolve().relative_to(anchor.resolve()) + except ValueError: + return None + new_db = Path(str(value)) / tail + if new_db.resolve() == current.resolve() or new_db.is_file(): + return None + return ( + f"{current} holds your existing conversations and memories, but this " + f"change points NexusOS at {new_db}, which does not exist yet - it will " + "start with an empty database. Stop NexusOS and move the .db (plus any " + "-wal/-shm files) to the new path to keep your history." + ) + + +def cmd_config(args) -> int: + values = config.read_user_config() + if args.action == "path": + print(config.CONFIG_FILE) + return 0 + if args.action == "list": + _emit(values, args.json) + return 0 + if args.action == "get": + if args.key not in CONFIG_SCHEMA: + print(f"Unknown configuration key: {args.key}", file=sys.stderr) + return 2 + value = values.get(args.key, getattr(settings, args.key, None)) + _emit({args.key: value}, args.json) + return 0 + if args.action == "set": + if args.key not in CONFIG_SCHEMA: + print(f"Unknown configuration key: {args.key}", file=sys.stderr) + print("Known keys: " + ", ".join(CONFIG_SCHEMA), file=sys.stderr) + return 2 + try: + values[args.key] = _validate_config(args.key, args.value) + except (TypeError, ValueError) as exc: + print(f"Invalid value: {exc}", file=sys.stderr) + return 2 + payload = {"updated": args.key, "value": values[args.key], "restart_required": True} + moved = _relocation_warning(args.key, values[args.key]) + if moved: + payload["warning"] = moved + config.write_user_config(values) + _emit(payload, args.json) + if moved and not args.json: + print(f"\nWARNING: {moved}", file=sys.stderr) + return 0 + if args.action == "unset": + if args.key not in CONFIG_SCHEMA: + print(f"Unknown configuration key: {args.key}", file=sys.stderr) + return 2 + values.pop(args.key, None) + config.write_user_config(values) + _emit({"removed": args.key, "restart_required": True}, args.json) + return 0 + return 2 + + +def _provider_payload() -> dict: + url = settings.ollama_host.rstrip("/") + return { + "provider": settings.provider, + "url": url, + "managed_by_nexus": settings.manage_ollama, + "reachable": _http_ok(url + "/api/tags", timeout=2.0), + } + + +def cmd_provider(args) -> int: + if args.action == "show": + _emit(_provider_payload(), args.json) + return 0 + + values = config.read_user_config() + if args.mode == "local": + values["provider"] = "ollama" + try: + values["provider_url"] = _validate_config( + "provider_url", args.url or "http://127.0.0.1:11434" + ) + except ValueError as exc: + print(f"Invalid value: {exc}", file=sys.stderr) + return 2 + else: + if not args.url: + print("Remote provider setup requires --url", file=sys.stderr) + return 2 + try: + values["provider_url"] = _validate_config("provider_url", args.url) + except ValueError as exc: + print(f"Invalid value: {exc}", file=sys.stderr) + return 2 + values["provider"] = "ollama-remote" + config.write_user_config(values) + _emit({ + "provider": values["provider"], + "url": values["provider_url"], + "restart_required": True, + }, args.json) + return 0 + + +def _check_import(module: str) -> bool: + return importlib.util.find_spec(module) is not None + + +def _writable(path: Path) -> bool: + try: + path.mkdir(parents=True, exist_ok=True) + probe = path / ".nexus-write-test" + probe.write_text("ok", encoding="utf-8") + probe.unlink() + return True + except OSError: + return False + + +def diagnostics() -> dict: + checks: list[dict] = [] + + def add(name: str, ok: bool, detail: str, required: bool = True): + checks.append({ + "name": name, + "status": "pass" if ok else ("fail" if required else "warn"), + "detail": detail, + "required": required, + }) + + add("python", sys.version_info >= (3, 11), sys.version.split()[0]) + add("state", _writable(settings.state_dir), str(settings.state_dir)) + add("database directory", _writable(settings.memory_db.parent), str(settings.memory_db.parent)) + add("web assets", (settings.web_dist_dir / "index.html").is_file(), str(settings.web_dist_dir)) + add( + "provider mode", + settings.provider in ("ollama", "ollama-remote"), + settings.provider, + ) + add( + "service ports", + 1 <= settings.backend_port <= 65535, + f"backend={settings.backend_port}", + ) + for module in ("fastapi", "uvicorn", "httpx", "pydantic", "yaml"): + add(f"import:{module}", _check_import(module), module) + + add("backend", _http_ok(settings.api_url + "/status"), settings.api_url, required=False) + provider = _provider_payload() + add("provider", provider["reachable"], provider["url"], required=False) + if settings.manage_ollama: + add("ollama executable", bool(services.ollama_bin()), services.ollama_bin() or "not on PATH", required=False) + + for label, module in ( + ("process control", "psutil"), + ("vector search", "sqlite_vec"), + ("voice transcription", "faster_whisper"), + ("PDF documents", "pypdf"), + ("Word documents", "docx"), + ("web search", "duckduckgo_search"), + ): + add(label, _check_import(module), module, required=False) + + return { + "ok": not any(c["status"] == "fail" for c in checks), + "version": settings.version, + "install_mode": "checkout" if settings.source_checkout else "wheel", + "platform": sys.platform, + "termux": _is_termux(), + "checks": checks, + } + + +def _doctor_fix() -> None: + config.CONFIG_DIR.mkdir(parents=True, exist_ok=True) + config.init_state() + index = settings.web_dist_dir / "index.html" + npm = shutil.which("npm.cmd" if os.name == "nt" else "npm") + if settings.source_checkout and not index.exists() and npm: + subprocess.run([npm, "run", "build"], cwd=str(settings.frontend_source_dir), check=False) + + +def cmd_doctor(args) -> int: + if args.fix: + _doctor_fix() + result = diagnostics() + if args.json: + _emit(result, True) + else: + print(f"NexusOS {result['version']} diagnostics ({result['install_mode']})\n") + marks = {"pass": "OK", "warn": "WARN", "fail": "FAIL"} + for check in result["checks"]: + print(f" {marks[check['status']]:<4} {check['name']:<20} {check['detail']}") + print("\nCore runtime is ready." if result["ok"] else "\nCore runtime has required failures.") + return 0 if result["ok"] else 1 + + +def service_status() -> dict: + payload = {} + for key in ("backend", "frontend"): + svc = services.SERVICES[key] + pid = services.read_pid(svc) + payload[key] = { + "running": services.alive(pid) or _http_ok(svc.url), + "pid": pid if services.alive(pid) else None, + "url": svc.url, + } + payload["provider"] = _provider_payload() + return payload + + +def cmd_status(args) -> int: + payload = service_status() + if args.json: + _emit(payload, True) + return 0 + print("Nexus Service Status:\n") + for key in ("backend", "frontend"): + info = payload[key] + suffix = f" (PID {info['pid']})" if info["pid"] else "" + print(f" {key:<10} {'RUNNING' if info['running'] else 'STOPPED'}{suffix} {info['url']}") + p = payload["provider"] + print(f" provider {'RUNNING' if p['reachable'] else 'STOPPED'} {p['provider']} @ {p['url']}") + return 0 + + +def cmd_monitor(args) -> int: + from .monitor import run_monitor + return run_monitor( + interval=getattr(args, "interval", 1.5), + once=bool(getattr(args, "once", False) or getattr(args, "json", False)), + json_output=bool(getattr(args, "json", False)), + ) + + +def _target_flag(target: str | None): + return { + "backend": "--backend", + "frontend": "--frontend", + "ai": "--ai", + }.get(target or "all") + + +def cmd_start(args) -> int: + services.cmd_start(_target_flag(args.target)) + return 0 + + +def cmd_stop(args) -> int: + services.cmd_stop(_target_flag(args.target)) + return 0 + + +def cmd_refresh(args) -> int: + services.cmd_stop(None) + services.cmd_start(None) + return 0 + + +def _lan_hostnames(host: str) -> list[str]: + """Every name/address a --allow-lan bind should accept in a Host header. + + A wildcard bind answers on all interfaces, so enumerate them; an explicit + address answers only as itself. The machine hostname comes along because + that is what people actually type.""" + import socket + + names: list[str] = [] + + def add(value: str) -> None: + if value and value not in names: + names.append(value) + + if host in ("0.0.0.0", "::", "*"): + hostname = socket.gethostname() + add(hostname) + add(hostname.split(".")[0] + ".local") + for family in (socket.AF_INET, socket.AF_INET6): + try: + for info in socket.getaddrinfo(hostname, None, family): + add(info[4][0]) + except OSError: + pass + # getaddrinfo(hostname) misses the routable address on hosts that map + # their own name to loopback; a connectionless UDP socket finds it. + for probe, family in (("8.8.8.8", socket.AF_INET), ("2001:4860:4860::8888", socket.AF_INET6)): + sock = socket.socket(family, socket.SOCK_DGRAM) + try: + sock.connect((probe, 80)) + add(sock.getsockname()[0]) + except OSError: + pass + finally: + sock.close() + else: + add(host.strip("[]")) + # A Host header carries an IPv6 literal bracketed; allow both spellings so + # the check matches however the client wrote it. + for value in list(names): + if ":" in value: + add(f"[{value}]") + return names + + +def _origin_host(name: str) -> str: + """Origin-safe spelling: IPv6 literals must be bracketed in a URL.""" + if ":" in name and not name.startswith("["): + return f"[{name}]" + return name + + +def cmd_serve(args) -> int: + host = args.host or settings.bind_host + if host not in ("127.0.0.1", "localhost", "::1") and not args.allow_lan: + print("Refusing an unauthenticated LAN bind. Add --allow-lan to acknowledge the exposure.", file=sys.stderr) + return 2 + if _http_ok(f"http://127.0.0.1:{args.port}/"): + print(f"Port {args.port} is already serving HTTP.", file=sys.stderr) + return 2 + + settings.backend_port = args.port + settings.bind_host = host + settings.api_url = f"http://127.0.0.1:{args.port}" + os.environ["NEXUS_BACKEND_PORT"] = str(args.port) + os.environ["NEXUS_BIND_HOST"] = host + for origin_host in ("localhost", "127.0.0.1"): + origin = f"http://{origin_host}:{args.port}" + if origin not in config.ALLOWED_ORIGINS: + config.ALLOWED_ORIGINS.append(origin) + if args.allow_lan: + # Widen to the addresses this bind actually answers on - NOT "*". + # ALLOWED_HOSTS drives TrustedHostMiddleware, which is the DNS-rebinding + # defense: with "*" any site the user browses could resolve a name it + # controls to this machine and drive the unauthenticated API. Naming the + # real addresses keeps that check doing its job. An explicitly exported + # NEXUS_ALLOWED_HOSTS still wins, for anyone who needs the old blanket. + names = _lan_hostnames(host) + for name in names: + if name not in config.ALLOWED_HOSTS: + config.ALLOWED_HOSTS.append(name) + origin = f"http://{_origin_host(name)}:{args.port}" + if origin not in config.ALLOWED_ORIGINS: + config.ALLOWED_ORIGINS.append(origin) + os.environ.setdefault("NEXUS_ALLOWED_HOSTS", ",".join(config.ALLOWED_HOSTS)) + os.environ.setdefault("NEXUS_ALLOWED_ORIGINS", ",".join(config.ALLOWED_ORIGINS)) + print( + "LAN exposure enabled for: " + ", ".join(names) + + "\nThe REST API is unauthenticated - anyone who can reach this port" + " has full admin and data access." + ) + + print(f"NexusOS serving on http://{host}:{args.port}") + import uvicorn + uvicorn.run( + "synapse.main:sio_app", host=host, port=args.port, + reload=bool(args.reload and settings.source_checkout), + log_level=args.log_level, + ) + return 0 + + +def cmd_open(args) -> int: + if not _http_ok(settings.api_url + "/status") and not args.no_start: + services.cmd_start(None) + url = args.url or settings.api_url + opener = shutil.which("termux-open-url") if _is_termux() else None + opened = False + if opener: + opened = subprocess.run([opener, url], check=False).returncode == 0 + else: + opened = webbrowser.open(url) + print(f"{'Opened' if opened else 'NexusOS is available at'} {url}") + return 0 + + +def cmd_web(args) -> int: + """Preserve ``ncp web`` for desktop checkouts; wheels use the browser UI.""" + if settings.source_checkout: + return services.cmd_web() + return cmd_open(argparse.Namespace(url=None, no_start=False)) + + +def cmd_panel(_args) -> int: + if not settings.source_checkout: + print("The legacy Tk control panel is only available in a desktop source install.", file=sys.stderr) + return 2 + return services.main(["panel"]) + + +def cmd_backup(args) -> int: + if not settings.source_checkout: + print("Backup is a source-checkout command; wheel state should be backed up from nexus paths.", file=sys.stderr) + return 2 + if args.check: + return services.sync_py("backup", "--check") + if args.full: + return services.sync_py("backup", "--full") + return services.sync_py("backup") + + +def cmd_restore(args) -> int: + if not settings.source_checkout: + print("Restore is a source-checkout command; reinstall the wheel and restore its state directory.", file=sys.stderr) + return 2 + return services.cmd_restore("--check" if args.check else None) + + +def cmd_nvidia_reqs(_args) -> int: + if not settings.source_checkout: + print("nvidia-reqs is only available in a desktop source install.", file=sys.stderr) + return 2 + return subprocess.run( + [sys.executable, str(settings.project_root / "bin" / "gen-nvidia-reqs.py")] + ).returncode + + +def cmd_logs(args) -> int: + keys = ("backend", "frontend") if args.target == "all" else (args.target,) + paths = [services.SERVICES[key].log_file for key in keys] + for path in paths: + print(f"=== {path.name} ===") + services._tail(path, args.lines) + if not args.follow: + return 0 + offsets = {path: path.stat().st_size if path.exists() else 0 for path in paths} + try: + while True: + for path in paths: + if not path.exists(): + continue + size = path.stat().st_size + if size < offsets[path]: + offsets[path] = 0 + if size > offsets[path]: + with open(path, encoding="utf-8", errors="replace") as stream: + stream.seek(offsets[path]) + sys.stdout.write(stream.read()) + sys.stdout.flush() + offsets[path] = stream.tell() + time.sleep(0.5) + except KeyboardInterrupt: + print() + return 130 + + +def cmd_clean(args) -> int: + for path in (settings.runtime_dir / "pids", settings.runtime_dir / "logs"): + if path.is_dir(): + for file in path.glob("*"): + if file.is_file(): + file.unlink(missing_ok=True) + for pattern in ("*.log", "*.pid"): + for file in settings.runtime_dir.glob(pattern): + file.unlink(missing_ok=True) + print(f"Cleaned runtime files under {settings.runtime_dir}") + return 0 + + +def cmd_update(args) -> int: + if settings.source_checkout: + services.cmd_update() + return 0 + command = [sys.executable, "-m", "pip", "install", "--upgrade", "nexusos-ai"] + if args.pre: + command.append("--pre") + return subprocess.run(command).returncode + + +def cmd_models(args) -> int: + import httpx + + base = settings.ollama_host.rstrip("/") + try: + if args.action == "list": + response = httpx.get(base + "/api/tags", timeout=5.0) + response.raise_for_status() + models = response.json().get("models", []) + if args.json: + _emit(models, True) + elif not models: + print("No models installed.") + else: + for model in models: + print(f"{model.get('name', ''):<36} {model.get('size', 0) / 1024**3:5.1f} GB") + elif args.action == "available": + services.cmd_models("available", None) + elif args.action in ("pull", "install"): + with httpx.stream("POST", base + "/api/pull", json={"name": args.name}, timeout=None) as response: + response.raise_for_status() + for line in response.iter_lines(): + if line: + try: + item = json.loads(line) + status = item.get("status") or item.get("error") + if status: + print(status) + except ValueError: + print(line) + elif args.action in ("remove", "rm"): + response = httpx.request("DELETE", base + "/api/delete", json={"name": args.name}, timeout=30.0) + response.raise_for_status() + print(f"Removed {args.name}") + return 0 + except httpx.HTTPError as exc: + print(f"Provider request failed at {base}: {exc}", file=sys.stderr) + return 1 + + +def cmd_api(args) -> int: + from . import nexus_api + nexus_api.BASE = args.api_url or settings.api_url + rest = list(args.rest) + if args.command == "chat" and rest[:1] == ["send"]: + rest.pop(0) + if args.command == "history" and rest[:1] == ["list"]: + rest.pop(0) + return nexus_api.main([args.command, *rest], prog=f"nexus {args.command}") + + +def _add_json(parser) -> None: + parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + + +def _port(value: str) -> int: + try: + return _validate_config("backend_port", value) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError(str(exc)) from exc + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="nexus", description="NexusOS local AI runtime and API client") + parser.add_argument("--version", action="version", version=f"NexusOS {settings.version}") + parser.add_argument("--api-url", help="override the NexusOS backend URL for this command") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("init", help="create user state and seed default playbooks"); _add_json(p); p.set_defaults(fn=cmd_init) + p = sub.add_parser("paths", help="show resolved package and writable paths"); _add_json(p); p.set_defaults(fn=cmd_paths) + + p = sub.add_parser("config", help="manage persistent CLI/runtime configuration") + p.add_argument("action", choices=["list", "get", "set", "unset", "path"]) + p.add_argument("key", nargs="?"); p.add_argument("value", nargs="?"); _add_json(p); p.set_defaults(fn=cmd_config) + + p = sub.add_parser("provider", help="configure the Ollama-compatible model provider") + provider_sub = p.add_subparsers(dest="action", required=True) + show = provider_sub.add_parser("show"); _add_json(show); show.set_defaults(fn=cmd_provider) + use = provider_sub.add_parser("use"); use.add_argument("mode", choices=["local", "remote"]) + use.add_argument("--url"); _add_json(use); use.set_defaults(fn=cmd_provider) + + p = sub.add_parser("doctor", help="check the core runtime and optional capabilities") + p.add_argument("--fix", action="store_true"); _add_json(p); p.set_defaults(fn=cmd_doctor) + p = sub.add_parser("status", help="show service and provider status"); _add_json(p); p.set_defaults(fn=cmd_status) + p = sub.add_parser("monitor", help="ASCII dashboard for services, resources, and tool stats") + p.add_argument("--once", action="store_true", help="print one frame and exit") + p.add_argument("--interval", type=float, default=1.5, help="refresh seconds (live mode)") + _add_json(p) + p.set_defaults(fn=cmd_monitor) + + p = sub.add_parser("serve", help="run NexusOS in the foreground") + p.add_argument("--host"); p.add_argument("--port", type=_port, default=settings.backend_port) + p.add_argument("--allow-lan", action="store_true") + p.add_argument("--reload", action="store_true"); p.add_argument("--log-level", default="info") + p.set_defaults(fn=cmd_serve) + + for name, fn, help_text in ( + ("start", cmd_start, "start services in the background"), + ("stop", cmd_stop, "stop background services"), + ): + p = sub.add_parser(name, help=help_text) + p.add_argument("target", nargs="?", choices=["all", "backend", "frontend", "ai"], default="all") + p.set_defaults(fn=fn) + sub.add_parser("restart", aliases=["refresh"], help="restart all services").set_defaults(fn=cmd_refresh) + sub.add_parser("kill", help="force-stop NexusOS-owned processes").set_defaults(fn=lambda _a: services.cmd_kill() or 0) + + p = sub.add_parser("open", help="open the web interface") + p.add_argument("--url"); p.add_argument("--no-start", action="store_true"); p.set_defaults(fn=cmd_open) + sub.add_parser("web", help="legacy desktop alias for open").set_defaults(fn=cmd_web) + sub.add_parser("panel", help="launch the legacy desktop control panel").set_defaults(fn=cmd_panel) + p = sub.add_parser("logs", help="read or follow service logs") + p.add_argument("target", nargs="?", choices=["all", "backend", "frontend"], default="all") + p.add_argument("--lines", type=int, choices=range(1, 10001), default=50, metavar="1..10000") + p.add_argument("--follow", "-f", action="store_true"); p.set_defaults(fn=cmd_logs) + sub.add_parser("clean", help="remove runtime logs and stale PID files").set_defaults(fn=cmd_clean) + p = sub.add_parser("update", help="update dependencies or the installed wheel"); p.add_argument("--pre", action="store_true"); p.set_defaults(fn=cmd_update) + + p = sub.add_parser("backup", help="back up a desktop source checkout") + p.add_argument("--full", "-f", action="store_true"); p.add_argument("--check", "-c", action="store_true") + p.set_defaults(fn=cmd_backup) + p = sub.add_parser("restore", help="restore a desktop source checkout") + p.add_argument("--check", "-c", action="store_true"); p.set_defaults(fn=cmd_restore) + sub.add_parser("nvidia-reqs", help="regenerate source-checkout NVIDIA requirements").set_defaults(fn=cmd_nvidia_reqs) + + p = sub.add_parser("models", help="list, pull, and remove provider models") + model_sub = p.add_subparsers(dest="action", required=True) + item = model_sub.add_parser("list"); _add_json(item); item.set_defaults(fn=cmd_models) + model_sub.add_parser("available", aliases=["search"]).set_defaults(fn=cmd_models, action="available") + item = model_sub.add_parser("pull", aliases=["install"]); item.add_argument("name"); item.set_defaults(fn=cmd_models, action="pull") + item = model_sub.add_parser("remove", aliases=["rm"]); item.add_argument("name"); item.set_defaults(fn=cmd_models, action="remove") + + for command in ("chat", "memory", "playbook", "history"): + p = sub.add_parser(command, add_help=False, help=f"use the {command} API from the terminal") + p.add_argument("rest", nargs=argparse.REMAINDER) + p.set_defaults(fn=cmd_api) + return parser + + +def _normalize_legacy_argv(argv) -> list[str]: + """Translate the old shell CLI spelling before argparse sees it.""" + normalized = list(argv or []) + if normalized == ["help"]: + return ["--help"] + # start/stop only. `logs` registers -f as the short form of --follow, so + # translating it here would silently rewrite `logs -f` to `logs frontend` + # - a tail of the wrong file instead of a follow, with no error. + if normalized and normalized[0] in ("start", "stop"): + normalized[1:] = [LEGACY_TARGETS.get(value, value) for value in normalized[1:]] + elif normalized[:1] == ["logs"]: + normalized[1:] = [ + value if value in ("-f", "--follow") else LEGACY_TARGETS.get(value, value) + for value in normalized[1:] + ] + if normalized[:2] == ["backup", "full"]: + normalized[1] = "--full" + elif len(normalized) > 1 and normalized[0] == "backup" and normalized[1] in ("check", "--claude"): + normalized[1] = "--check" + elif len(normalized) > 1 and normalized[0] == "restore": + if normalized[1] in ("check", "--claude"): + normalized[1] = "--check" + elif normalized[1] in ("full", "-f", "--full"): + normalized.pop(1) + return normalized + + +def main(argv=None) -> int: + parser = build_parser() + args = parser.parse_args(_normalize_legacy_argv(argv)) + if args.command == "config": + if args.action in ("get", "unset") and not args.key: + parser.error(f"config {args.action} requires KEY") + if args.action == "set" and (not args.key or args.value is None): + parser.error("config set requires KEY VALUE") + try: + result = args.fn(args) + return result if isinstance(result, int) else 0 + except KeyboardInterrupt: + print() + return 130 + + +def entrypoint() -> int: + return main(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(entrypoint()) diff --git a/nexusos_cli/monitor.py b/nexusos_cli/monitor.py new file mode 100644 index 0000000..8697a55 --- /dev/null +++ b/nexusos_cli/monitor.py @@ -0,0 +1,456 @@ +"""ASCII dashboard for live NexusOS service / tool / resource stats. + +No curses, no rich — pure box-drawing + optional ANSI color so it works in +Termux, plain SSH, and Windows Terminal alike. The collector is separate from +the renderer so tests can feed fixtures without a running stack. +""" +from __future__ import annotations + +import json +import shutil +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from synapse.nexus_config import settings + +from . import ncp as services + +# Box drawing — ASCII fallbacks when the terminal can't do Unicode. +_BOX = { + "tl": "┌", "tr": "┐", "bl": "└", "br": "┘", + "h": "─", "v": "│", "l": "├", "r": "┤", +} +_BOX_ASCII = { + "tl": "+", "tr": "+", "bl": "+", "br": "+", + "h": "-", "v": "|", "l": "+", "r": "+", +} + +_FILL = "█" +_EMPTY = "░" +_FILL_ASCII = "#" +_EMPTY_ASCII = "-" + + +def _use_unicode() -> bool: + enc = (getattr(__import__("sys").stdout, "encoding", None) or "").lower() + return "utf" in enc or enc in ("cp65001",) + + +def _http_ok(url: str, timeout: float = 1.0) -> bool: + try: + urllib.request.urlopen(url, timeout=timeout).read(1) + return True + except urllib.error.HTTPError: + return True + except Exception: + return False + + +def _provider_payload() -> dict: + url = settings.ollama_host.rstrip("/") + return { + "provider": settings.provider, + "url": url, + "managed_by_nexus": settings.manage_ollama, + "reachable": _http_ok(url + "/api/tags", timeout=2.0), + } + + +def _service_status() -> dict: + payload = {} + for key in ("backend", "frontend"): + svc = services.SERVICES[key] + pid = services.read_pid(svc) + payload[key] = { + "running": services.alive(pid) or _http_ok(svc.url), + "pid": pid if services.alive(pid) else None, + "url": svc.url, + } + payload["provider"] = _provider_payload() + return payload + + +def _get_json(url: str, timeout: float = 1.5) -> Any | None: + try: + with urllib.request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8", errors="replace")) + except Exception: + return None + + +def _bar(ratio: float, width: int = 20, unicode: bool = True) -> str: + ratio = max(0.0, min(1.0, float(ratio))) + filled = int(round(ratio * width)) + fill = _FILL if unicode else _FILL_ASCII + empty = _EMPTY if unicode else _EMPTY_ASCII + return fill * filled + empty * (width - filled) + + +def _fmt_bytes(n: float | int | None) -> str: + if n is None: + return "—" + n = float(n) + for unit in ("B", "K", "M", "G", "T"): + if abs(n) < 1024 or unit == "T": + return f"{n:.0f}{unit}" if unit == "B" else f"{n:.1f}{unit}" + n /= 1024 + return f"{n:.1f}T" + + +def _pid_stats(pids: list[int | None]) -> dict: + """Aggregate CPU%/RSS for known service PIDs. Soft-depends on psutil.""" + live = [int(p) for p in pids if p] + if not live: + return {"cpu_pct": None, "rss": None, "pids": []} + try: + import psutil # type: ignore + except ImportError: + return {"cpu_pct": None, "rss": None, "pids": live} + cpu = 0.0 + rss = 0 + seen: list[int] = [] + for pid in live: + try: + proc = psutil.Process(pid) + cpu += proc.cpu_percent(interval=0.0) + rss += proc.memory_info().rss + seen.append(pid) + except (psutil.Error, ProcessLookupError, ValueError): + continue + return {"cpu_pct": cpu, "rss": rss, "pids": seen} + + +def _host_stats() -> dict: + try: + import psutil # type: ignore + except ImportError: + return {"cpu_pct": None, "mem_used": None, "mem_total": None, "mem_pct": None} + vm = psutil.virtual_memory() + return { + "cpu_pct": psutil.cpu_percent(interval=0.05), + "mem_used": vm.used, + "mem_total": vm.total, + "mem_pct": vm.percent, + } + + +def _api_counts(api_url: str) -> dict: + """Pull cheap inventory counts from the backend when it is up.""" + base = api_url.rstrip("/") + out = { + "online": False, + "version": None, + "ollama": None, + "memories": None, + "conversations": None, + "playbooks": None, + "models": None, + "action_tool_policy": None, + } + status = _get_json(base + "/status") + if not isinstance(status, dict): + return out + out["online"] = True + out["version"] = status.get("version") + out["ollama"] = status.get("ollama") + + mem = _get_json(base + "/memory") + if isinstance(mem, list): + out["memories"] = len(mem) + elif isinstance(mem, dict) and isinstance(mem.get("memories"), list): + out["memories"] = len(mem["memories"]) + + conv = _get_json(base + "/conversations") + if isinstance(conv, list): + out["conversations"] = len(conv) + elif isinstance(conv, dict): + items = conv.get("conversations") or conv.get("items") or [] + if isinstance(items, list): + out["conversations"] = len(items) + + pbs = _get_json(base + "/playbooks") + if isinstance(pbs, list): + out["playbooks"] = len(pbs) + elif isinstance(pbs, dict) and isinstance(pbs.get("playbooks"), list): + out["playbooks"] = len(pbs["playbooks"]) + + models = _get_json(base + "/models") + if isinstance(models, list): + out["models"] = len(models) + elif isinstance(models, dict): + items = models.get("models") or models.get("items") or [] + if isinstance(items, list): + out["models"] = len(items) + + settings_payload = _get_json(base + "/settings") + if isinstance(settings_payload, dict): + out["action_tool_policy"] = settings_payload.get("action_tool_policy") + + return out + + +def _toolchain_stats() -> list[dict]: + """Which run_snippet languages have a host toolchain right now.""" + try: + from synapse import code_run + except Exception: + return [] + rows = [] + for name, spec in code_run.RUN_LANGS.items(): + tool = None + try: + tool = spec["tool"]() + except Exception: + tool = None + rows.append({ + "lang": name, + "ready": bool(tool), + "tool": tool or None, + "summary": spec.get("summary") or name, + }) + return rows + + +def _recent_tools(log_path: Path, limit: int = 8) -> list[str]: + """Best-effort scrape of recent tool names from chat.log.""" + if not log_path.is_file(): + return [] + try: + # Read the tail without pulling a multi-MB log into memory. + data = log_path.read_bytes() + if len(data) > 64_000: + data = data[-64_000:] + text = data.decode("utf-8", errors="replace") + except OSError: + return [] + found: list[str] = [] + for line in reversed(text.splitlines()): + # Chat tool loop yields "__status__"; logs may also name tools + # in JSON payloads. Keep the match narrow. + if "__status__" in line: + name = line.split("__status__", 1)[-1].strip().split()[0].strip(",\"'") + if name and name not in ("tools",) and name not in found: + found.append(name) + elif '"name":' in line and any( + t in line for t in ("render_preview", "run_snippet", "web_search", + "fetch_url", "remember", "get_time") + ): + for t in ("run_snippet", "render_preview", "web_search", "fetch_url", + "remember", "get_time", "search_documents"): + if t in line and t not in found: + found.append(t) + if len(found) >= limit: + break + return found + + +def collect_snapshot() -> dict: + """Gather one monitoring frame. Safe when services are down.""" + services_payload = _service_status() + pids = [ + services_payload.get("backend", {}).get("pid"), + services_payload.get("frontend", {}).get("pid"), + ] + api = _api_counts(settings.api_url) + return { + "ts": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"), + "version": settings.version, + "services": services_payload, + "api": api, + "host": _host_stats(), + "procs": _pid_stats(pids), + "toolchains": _toolchain_stats(), + "recent_tools": _recent_tools(settings.logs_dir / "chat.log"), + "paths": { + "api_url": settings.api_url, + "runtime_dir": str(settings.runtime_dir), + }, + } + + +def _pad(text: str, width: int) -> str: + # Visual width ≈ len for our ASCII/box content (no wide East-Asian chars). + if len(text) > width: + return text[: width - 1] + "…" if width > 1 else text[:width] + return text + " " * (width - len(text)) + + +def _row(box: dict, inner: str, width: int) -> str: + return f"{box['v']} {_pad(inner, width - 4)} {box['v']}" + + +def _rule(box: dict, width: int, kind: str = "mid") -> str: + h = box["h"] * (width - 2) + if kind == "top": + return f"{box['tl']}{h}{box['tr']}" + if kind == "bot": + return f"{box['bl']}{h}{box['br']}" + return f"{box['l']}{h}{box['r']}" + + +def _svc_line(name: str, running: bool, detail: str, unicode: bool) -> str: + mark = (_FILL if unicode else _FILL_ASCII) * 3 if running else (_EMPTY if unicode else _EMPTY_ASCII) * 3 + state = "UP " if running else "DOWN" + return f"{name:<10} {mark} {state} {detail}" + + +def render_frame(snapshot: dict, *, width: int | None = None, unicode: bool | None = None) -> str: + """Turn a snapshot into a single multi-line ASCII panel.""" + if unicode is None: + unicode = _use_unicode() + box = _BOX if unicode else _BOX_ASCII + cols = shutil.get_terminal_size((80, 24)).columns if width is None else width + width = max(56, min(100, cols)) + + lines: list[str] = [] + lines.append(_rule(box, width, "top")) + title = f"NexusOS {snapshot.get('version', '')} monitor" + raw_ts = snapshot.get("ts") or "" + # Prefer local clock HH:MM:SS from an ISO stamp; fall back to wall clock. + stamp = "" + if "T" in raw_ts: + try: + stamp = raw_ts.split("T", 1)[1][:8] + except Exception: + stamp = "" + if not stamp: + stamp = datetime.now().strftime("%H:%M:%S") + gap = max(1, width - 4 - len(title) - len(stamp)) + header = f"{title}{' ' * gap}{stamp}" + lines.append(_row(box, header, width)) + lines.append(_rule(box, width, "mid")) + + lines.append(_row(box, "SERVICES", width)) + svcs = snapshot.get("services") or {} + for key, label in (("backend", "backend"), ("frontend", "frontend")): + info = svcs.get(key) or {} + running = bool(info.get("running")) + pid = info.get("pid") + url = info.get("url") or "" + detail = url + if pid: + detail = f"pid {pid} {url}" + lines.append(_row(box, _svc_line(label, running, detail, unicode), width)) + provider = svcs.get("provider") or _provider_payload() + pref = f"{provider.get('provider', '?')} @ {provider.get('url', '')}" + lines.append(_row(box, _svc_line("provider", bool(provider.get("reachable")), pref, unicode), width)) + + lines.append(_rule(box, width, "mid")) + lines.append(_row(box, "RESOURCES", width)) + host = snapshot.get("host") or {} + procs = snapshot.get("procs") or {} + cpu = host.get("cpu_pct") + if cpu is not None: + lines.append(_row( + box, + f"host CPU [{_bar(cpu / 100.0, 22, unicode)}] {cpu:5.1f}%", + width, + )) + else: + lines.append(_row(box, "host CPU (install psutil for live bars)", width)) + mem_pct = host.get("mem_pct") + if mem_pct is not None: + lines.append(_row( + box, + f"host MEM [{_bar(mem_pct / 100.0, 22, unicode)}] " + f"{_fmt_bytes(host.get('mem_used'))} / {_fmt_bytes(host.get('mem_total'))}", + width, + )) + proc_cpu = procs.get("cpu_pct") + proc_rss = procs.get("rss") + if proc_cpu is not None or proc_rss is not None: + lines.append(_row( + box, + f"nexus cpu={proc_cpu if proc_cpu is not None else '—':>5} " + f"rss={_fmt_bytes(proc_rss)} pids={','.join(str(p) for p in (procs.get('pids') or [])) or '—'}", + width, + )) + + lines.append(_rule(box, width, "mid")) + lines.append(_row(box, "DATA / TOOLS", width)) + api = snapshot.get("api") or {} + if api.get("online"): + policy = api.get("action_tool_policy") or "—" + lines.append(_row( + box, + f"api UP v{api.get('version') or '?'} ollama={api.get('ollama') or '—'} " + f"tools={policy}", + width, + )) + lines.append(_row( + box, + f"memories={_n(api.get('memories'))} " + f"chats={_n(api.get('conversations'))} " + f"playbooks={_n(api.get('playbooks'))} " + f"models={_n(api.get('models'))}", + width, + )) + else: + lines.append(_row(box, "api DOWN — start with: nexus start", width)) + + recent = snapshot.get("recent_tools") or [] + lines.append(_row( + box, + "recent " + (", ".join(recent) if recent else "(none in chat.log)"), + width, + )) + + lines.append(_rule(box, width, "mid")) + lines.append(_row(box, "RUN TOOLCHAINS (run_snippet)", width)) + chains = snapshot.get("toolchains") or [] + if not chains: + lines.append(_row(box, "(code_run unavailable)", width)) + else: + # Pack ready/missing into one or two compact lines. + ready = [c["lang"] for c in chains if c.get("ready")] + missing = [c["lang"] for c in chains if not c.get("ready")] + lines.append(_row( + box, + f"ready {', '.join(ready) if ready else '—'}", + width, + )) + lines.append(_row( + box, + f"missing {', '.join(missing) if missing else '—'}", + width, + )) + + lines.append(_rule(box, width, "bot")) + return "\n".join(lines) + + +def _n(value) -> str: + return "—" if value is None else str(value) + + +def run_monitor(*, interval: float = 1.5, once: bool = False, json_output: bool = False) -> int: + """Print one frame, or refresh in place until interrupted.""" + clear = "\033[H\033[J" + first = True + while True: + snap = collect_snapshot() + if json_output: + print(json.dumps(snap, indent=2, sort_keys=True)) + else: + frame = render_frame(snap) + if once or not first: + # Replacing the screen keeps the panel stable; first frame of a + # live session also clears so leftover shell output doesn't mix. + if not once: + print(clear + frame, end="", flush=True) + else: + print(frame) + else: + print(clear + frame, end="", flush=True) + first = False + if once: + return 0 + try: + time.sleep(max(0.3, float(interval))) + except KeyboardInterrupt: + print() + return 130 diff --git a/management/ncp.py b/nexusos_cli/ncp.py similarity index 89% rename from management/ncp.py rename to nexusos_cli/ncp.py index 6fd4f24..be38932 100644 --- a/management/ncp.py +++ b/nexusos_cli/ncp.py @@ -28,15 +28,21 @@ import urllib.request from dataclasses import dataclass, field from pathlib import Path -ROOT = Path(__file__).resolve().parent.parent -PID_DIR = ROOT / "runtime" / "pids" -LOG_DIR = ROOT / "runtime" -FRONTEND_DIR = ROOT / "interface" / "web" +from synapse import proc_util +from synapse.nexus_config import SOURCE_CHECKOUT, settings + +ROOT = settings.project_root +PID_DIR = settings.runtime_dir / "pids" +LOG_DIR = settings.runtime_dir +FRONTEND_DIR = settings.frontend_source_dir OLLAMA_BIN = ROOT / "ollama" / "bin" / ("ollama.exe" if os.name == "nt" else "ollama") -OLLAMA_MODELS_DIR = ROOT / "models" +OLLAMA_MODELS_DIR = settings.models_dir WINDOWS = os.name == "nt" -PYTHON = ROOT / "Promethean" / ("Scripts/python.exe" if WINDOWS else "bin/python3") +_VENV_PYTHON = ROOT / "Promethean" / ("Scripts/python.exe" if WINDOWS else "bin/python3") +PYTHON = Path(os.getenv("NEXUS_PYTHON", "")) if os.getenv("NEXUS_PYTHON") else ( + _VENV_PYTHON if _VENV_PYTHON.exists() else Path(sys.executable) +) PID_DIR.mkdir(parents=True, exist_ok=True) LOG_DIR.mkdir(parents=True, exist_ok=True) @@ -93,7 +99,7 @@ def _psutil(): import psutil return psutil except ImportError: - sys.exit("psutil is missing - run ./install.sh (or install-windows.ps1) to rebuild the venv.") + return None # -- services ------------------------------------------------------------------ @@ -124,7 +130,7 @@ class Service: # Bind loopback by default: the backend REST API is unauthenticated, so # binding 0.0.0.0 handed the full admin+data plane to any host on the LAN. Set # NEXUS_BIND_HOST=0.0.0.0 to opt into LAN exposure once real auth is in place. -BIND_HOST = os.environ.get("NEXUS_BIND_HOST", "127.0.0.1") +BIND_HOST = settings.bind_host def _uvicorn(app: str, port: int): @@ -140,9 +146,9 @@ def _uvicorn(app: str, port: int): SERVICES = { - "backend": Service("backend", "NEXUS BACKEND SERVICE", 8000, ROOT, + "backend": Service("backend", "NEXUS BACKEND SERVICE", settings.backend_port, settings.state_dir, ["uvicorn synapse.main"], - lambda: _uvicorn("synapse.main:sio_app", 8000)), + lambda: _uvicorn("synapse.main:sio_app", settings.backend_port)), "frontend": Service("frontend", "NEXUS FRONTEND SERVICE", 5173, FRONTEND_DIR, ["vite --host", "npm run dev"], lambda: [npm(), "run", "dev", "--", "--host", "0.0.0.0"]), @@ -158,7 +164,13 @@ def read_pid(svc: Service): def alive(pid) -> bool: ps = _psutil() - return pid is not None and ps.pid_exists(pid) + if pid is None: + return False + if ps is not None: + return ps.pid_exists(pid) + # Not os.kill(pid, 0): that reports False for a live process owned by + # another user, and Windows has no signals to fall back on. + return proc_util.pid_alive(pid) def pid_is_ours(pid, patterns) -> bool: @@ -168,7 +180,10 @@ def pid_is_ours(pid, patterns) -> bool: is not enough. TERMing a recycled PID can log the user out.""" ps = _psutil() try: - cmd = " ".join(ps.Process(pid).cmdline()) + if ps is not None: + cmd = " ".join(ps.Process(pid).cmdline()) + else: + cmd = proc_util.pid_cmdline(pid) except Exception: return False return any(p in cmd for p in patterns) @@ -255,6 +270,13 @@ def kill_matching(patterns, force=False) -> int: ps = _psutil() me = os.getpid() hit = 0 + if ps is None: + for pid, cmd in proc_util.iter_processes(): + if pid == me or not any(pattern in cmd for pattern in patterns): + continue + if proc_util.terminate_pid(pid, force=force): + hit += 1 + return hit for proc in ps.process_iter(["pid", "cmdline"]): if proc.info["pid"] == me: continue @@ -272,6 +294,12 @@ def kill_port(port: int) -> bool: """Whatever holds the port IS the service - this is the backstop that makes stop reliable regardless of process-tree shape or PID-file accuracy.""" ps = _psutil() + if ps is None: + killed = False + for pid in proc_util.pids_listening_on(port): + if pid != os.getpid() and proc_util.terminate_pid(pid, force=True): + killed = True + return killed killed = False try: conns = ps.net_connections(kind="inet") @@ -295,13 +323,18 @@ def stop_service(svc: Service) -> bool: if alive(pid) and pid_is_ours(pid, svc.patterns): ps = _psutil() try: - proc = ps.Process(pid) - for child in proc.children(recursive=True): - try: - child.terminate() - except Exception: - pass - proc.terminate() + if ps is not None: + proc = ps.Process(pid) + for child in proc.children(recursive=True): + try: + child.terminate() + except Exception: + pass + proc.terminate() + else: + # No psutil means no process tree; the kill_matching sweep + # below is what catches reparented children here. + proc_util.terminate_pid(pid) except Exception: pass svc.pid_file.unlink(missing_ok=True) @@ -332,8 +365,15 @@ def start_ollama(background: bool = False) -> None: background=False (`ncp start --ai`): blocks until the model is warmed - weights read off disk into RAM/VRAM, routinely about a minute - so "started" means the AI can actually answer.""" + if not settings.manage_ollama: + running = http_ok(settings.ollama_host.rstrip("/") + "/api/tags", timeout=3) + print( + f"REMOTE OLLAMA {'REACHABLE' if running else 'UNREACHABLE'} " + f"({settings.ollama_host})" + ) + return print("Starting OLLAMA...") - url = "http://localhost:8000/ollama/start" + ("?background=true" if background else "") + url = settings.api_url + "/ollama/start" + ("?background=true" if background else "") req = urllib.request.Request(url, method="POST") try: urllib.request.urlopen(req, timeout=180).read(1) @@ -346,7 +386,10 @@ def stop_ollama() -> None: """Prefer the backend endpoint for a clean OllamaManager shutdown; if the backend is already down, kill `ollama serve` directly so it never lingers holding VRAM/RAM. Must run BEFORE the backend is torn down.""" - req = urllib.request.Request("http://localhost:8000/ollama/stop", method="POST") + if not settings.manage_ollama: + print(f"REMOTE OLLAMA IS EXTERNALLY MANAGED ({settings.ollama_host})") + return + req = urllib.request.Request(settings.api_url + "/ollama/stop", method="POST") try: urllib.request.urlopen(req, timeout=5).read(1) print("NEXUS OLLAMA STOPPED") @@ -367,6 +410,9 @@ def cmd_start(target) -> None: launch(SERVICES["backend"]); wait_for_port(SERVICES["backend"]) start_ollama() elif target in ("--frontend", "-f"): + if not SOURCE_CHECKOUT: + print("Vite source is unavailable in wheel installs; the backend serves the bundled UI.") + return launch(SERVICES["frontend"]); wait_for_port(SERVICES["frontend"]) elif target in ("--ai", "-a"): start_ollama() @@ -389,7 +435,7 @@ def cmd_start(target) -> None: wait_for_port(SERVICES["backend"]) t_services = time.perf_counter() start_ollama(background=True) - if not WINDOWS: + if SOURCE_CHECKOUT and not WINDOWS: launch(SERVICES["frontend"]) t_bg = time.perf_counter() print("\nBoot timing:") @@ -405,6 +451,8 @@ def cmd_stop(target) -> None: stop_ollama(); stop_service(SERVICES["backend"]) elif target in ("--frontend", "-f"): stop_service(SERVICES["frontend"]) + elif target in ("--ai", "-a"): + stop_ollama() elif target in (None, "", "all"): stop_ollama() stop_service(SERVICES["backend"]) @@ -415,14 +463,20 @@ def cmd_stop(target) -> None: def cmd_kill() -> None: print("Force-killing all Nexus processes...") - for port, name in ((8000, "SYNAPSE"), - (5173, "INTERFACE"), (11434, "OLLAMA")): + targets = [ + (settings.backend_port, "SYNAPSE"), + (5173, "INTERFACE"), + ] + patterns = ["uvicorn synapse", "npm run dev", "vite --host"] + if settings.manage_ollama: + targets.append((11434, "OLLAMA")) + patterns.append("ollama serve") + for port, name in targets: if kill_port(port): print(f" KILLED: {name} (:{port})") else: print(f" NOT RUNNING: {name} (:{port})") - kill_matching(["uvicorn synapse", "npm run dev", "vite --host", "ollama serve"], - force=True) + kill_matching(patterns, force=True) for pid_file in PID_DIR.glob("*.pid"): pid_file.unlink(missing_ok=True) print("Done.") @@ -445,8 +499,9 @@ def cmd_status() -> None: print("\nFrontend:") one("Vite ", SERVICES["frontend"]) print("\nModel server:") - running = http_ok("http://localhost:11434/api/tags") - print(f" Ollama : {'RUNNING (:11434)' if running else 'STOPPED'}") + running = http_ok(settings.ollama_host.rstrip("/") + "/api/tags") + mode = "remote" if not settings.manage_ollama else "local" + print(f" Ollama ({mode}) : {'RUNNING' if running else 'STOPPED'} ({settings.ollama_host})") def _tail(path: Path, n: int) -> None: @@ -687,10 +742,10 @@ MODEL_CATALOG = [ def cmd_models(action, name) -> None: if action == "list": - if not http_ok("http://localhost:11434/api/tags"): + if not http_ok(settings.ollama_host.rstrip("/") + "/api/tags"): print("Ollama is not running. Start the backend first with: ncp start -b") return - raw = urllib.request.urlopen("http://localhost:11434/api/tags", timeout=5).read() + raw = urllib.request.urlopen(settings.ollama_host.rstrip("/") + "/api/tags", timeout=5).read() models = json.loads(raw).get("models", []) print("Installed models:\n") if not models: diff --git a/management/nexus_api.py b/nexusos_cli/nexus_api.py similarity index 95% rename from management/nexus_api.py rename to nexusos_cli/nexus_api.py index 21c58f4..4952ddb 100644 --- a/management/nexus_api.py +++ b/nexusos_cli/nexus_api.py @@ -11,7 +11,9 @@ import sys import httpx -BASE = __import__("os").environ.get("NEXUS_API", "http://localhost:8000") +from synapse.nexus_config import settings + +BASE = settings.api_url def _client(): @@ -118,8 +120,8 @@ def cmd_history(args): _die_if_down(e) -def main(): - p = argparse.ArgumentParser(prog="ncp") +def main(argv=None, prog="nexus"): + p = argparse.ArgumentParser(prog=prog) sub = p.add_subparsers(dest="cmd", required=True) c = sub.add_parser("chat"); c.add_argument("message", nargs="+") @@ -135,8 +137,9 @@ def main(): h = sub.add_parser("history"); h.add_argument("query", nargs="?") h.add_argument("--limit", type=int, default=20); h.set_defaults(fn=cmd_history) - args = p.parse_args() - args.fn(args) + args = p.parse_args(argv) + result = args.fn(args) + return result if isinstance(result, int) else 0 if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f4a8cfc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,128 @@ +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[project] +name = "nexusos-ai" +dynamic = ["version"] +description = "Local-first AI assistant runtime, web UI, and portable CLI" +readme = "README.md" +requires-python = ">=3.11" +authors = [{name = "EnderOfWings"}] +keywords = ["ai", "assistant", "ollama", "local-ai", "termux"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Framework :: FastAPI", + "Operating System :: Android", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Communications :: Chat", +] +dependencies = [ + "fastapi>=0.115,<1", + "httpx>=0.27,<1", + "pydantic>=2.7,<3", + "python-dotenv>=1,<2", + "PyYAML>=6,<7", + "uvicorn>=0.30,<1", +] + +[project.optional-dependencies] +documents = ["pypdf>=5,<7", "python-docx>=1.1,<2"] +process = ["psutil>=5.9,<8"] +vector = ["sqlite-vec>=0.1,<1"] +voice = ["faster-whisper>=1.1,<2"] +mail = ["imap-tools>=1.7,<2"] +# synapse/search.py imports this lazily behind a bare except, so without it +# declared the chat web-search path silently returns nothing. +search = ["duckduckgo-search>=6,<9"] +desktop = [ + "psutil>=5.9,<8", + "pywebview>=5,<7; platform_system == 'Windows'", +] +standard = [ + "pypdf>=5,<7", + "python-docx>=1.1,<2", + "psutil>=5.9,<8", + "sqlite-vec>=0.1,<1", + "duckduckgo-search>=6,<9", +] +# Every optional capability at once. Kept in sync with the extras above by +# tests/test_packaging_deps.py, which also checks that nothing synapse imports +# is missing from this file. +all = [ + "pypdf>=5,<7", + "python-docx>=1.1,<2", + "psutil>=5.9,<8", + "sqlite-vec>=0.1,<1", + "faster-whisper>=1.1,<2", + "imap-tools>=1.7,<2", + "duckduckgo-search>=6,<9", + "pywebview>=5,<7; platform_system == 'Windows'", +] +dev = [ + "build>=1.2,<2", + "pytest>=8,<10", + "twine>=7,<8", +] + +[project.scripts] +nexus = "nexusos_cli.cli:entrypoint" +ncp = "nexusos_cli.cli:entrypoint" +nexusos = "nexusos_cli.cli:entrypoint" + +[project.urls] +Homepage = "https://git.enderofwings.com/enderofwings/NexusOS" +Issues = "https://git.enderofwings.com/enderofwings/NexusOS/issues" +Repository = "https://git.enderofwings.com/enderofwings/NexusOS" + +[tool.hatch.version] +path = "VERSION" +pattern = "^(?P[^\\s]+)$" + +[tool.hatch.build] +skip-excluded-dirs = true + +# Ships interface/web/dist when it has been built. It is gitignored, so a +# static force-include would abort `pip install -e .` on a fresh clone. +[tool.hatch.build.hooks.custom] +path = "hatch_build.py" + +[tool.hatch.build.targets.wheel] +packages = ["synapse", "nexusos_cli", "modules"] + +[tool.hatch.build.targets.wheel.force-include] +"VERSION" = "synapse/_resources/VERSION" +"data/playbooks" = "synapse/_resources/playbooks" +"assets/n-small.png" = "synapse/_resources/assets/n-small.png" +"assets/themes/NexusOS-icons-src/nexus-underlay.svg" = "synapse/_resources/assets/themes/NexusOS-icons-src/nexus-underlay.svg" +"assets/themes/NexusOS-icons-src/nexus-underlay-ring.svg" = "synapse/_resources/assets/themes/NexusOS-icons-src/nexus-underlay-ring.svg" + +[tool.hatch.build.targets.sdist] +include = [ + "/assets/n-small.png", + "/assets/themes/NexusOS-icons-src/nexus-underlay.svg", + "/assets/themes/NexusOS-icons-src/nexus-underlay-ring.svg", + "/data/playbooks", + "/docs", + "/management", + "/modules", + "/nexusos_cli", + "/scripts", + "/synapse", + "/tests", + "/README.md", + "/VERSION", + "/pyproject.toml", + "/hatch_build.py", +] + + +[tool.pytest.ini_options] +testpaths = ["tests", "management"] diff --git a/requirements-base.txt b/requirements-base.txt index 523686f..f374594 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -38,6 +38,9 @@ sqlite-vec faster-whisper # Email client: IMAP read (SMTP send is stdlib). Pure-Python, no native deps. imap-tools +# Chat web search (synapse/search.py). Imported lazily behind a bare except, +# so a missing install shows up as search silently returning nothing. +duckduckgo-search # Documentation Support markdown-it-py diff --git a/scripts/install-termux.sh b/scripts/install-termux.sh new file mode 100644 index 0000000..501c75e --- /dev/null +++ b/scripts/install-termux.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Install the portable NexusOS wheel in Termux. +# +# NEXUS_PACKAGE may be a PyPI requirement, wheel URL, or local wheel path. +# NEXUS_ANDROID_WHEEL_INDEX may point at a trusted PEP 503 index containing +# Android pydantic-core wheels for the device's Python/CPU combination. +set -euo pipefail + +if [[ "${PREFIX:-}" != *com.termux* ]]; then + echo "This installer must be run inside Termux." >&2 + exit 2 +fi + +PACKAGE_SPEC="${1:-${NEXUS_PACKAGE:-nexusos-ai}}" + +echo "==> Installing Termux prerequisites" +pkg update -y +pkg install -y python python-pip curl clang make + +PYTHON_TAG="$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")')" +ARCH="$(uname -m)" +echo "==> Python ${PYTHON_TAG}; architecture ${ARCH}" + +PIP_INDEX_ARGS=() +if [[ -n "${NEXUS_ANDROID_WHEEL_INDEX:-}" ]]; then + PIP_INDEX_ARGS+=(--extra-index-url "${NEXUS_ANDROID_WHEEL_INDEX}") +fi + +# Pydantic 2 is required on Python 3.14+, and its Rust extension is not built +# reliably on-device. Require a binary before asking pip to resolve NexusOS so +# failures are immediate and actionable. Older Termux environments also benefit +# from using a wheel instead of compiling the extension on a phone. +echo "==> Checking for an Android pydantic-core wheel" +if ! python -m pip install \ + --only-binary=pydantic-core \ + "${PIP_INDEX_ARGS[@]}" \ + "pydantic>=2.7,<3"; then + cat >&2 < Installing ${PACKAGE_SPEC}" +# Same extra index as the pydantic-core probe: a NexusOS wheel hosted there +# would otherwise be invisible to the resolver. +python -m pip install "${PIP_INDEX_ARGS[@]}" "${PACKAGE_SPEC}" + +echo "==> Initializing NexusOS" +nexus init + +if [[ -n "${NEXUS_PROVIDER_URL:-}" ]]; then + nexus provider use remote --url "${NEXUS_PROVIDER_URL}" +fi + +# Report health without aborting: `set -e` would otherwise skip the guidance +# below whenever doctor finds a failing required check - exactly when the +# reader most needs to see what to do next. The status is re-raised at exit. +DOCTOR_STATUS=0 +nexus doctor || DOCTOR_STATUS=$? + +cat <<'EOF' + +NexusOS is installed. Configure an Ollama-compatible provider, then run: + nexus provider use remote --url http://YOUR-OLLAMA-HOST:11434 + nexus serve + +Open http://127.0.0.1:8000 in the Android browser. +EOF + +exit "${DOCTOR_STATUS}" diff --git a/synapse/frontend_manager.py b/synapse/frontend_manager.py index 6c55299..7c91318 100644 --- a/synapse/frontend_manager.py +++ b/synapse/frontend_manager.py @@ -13,11 +13,15 @@ import shutil import subprocess import urllib.request -import psutil +try: + import psutil +except ImportError: # optional in the portable/Termux core install + psutil = None -from .nexus_config import PROJECT_ROOT, RUNTIME_DIR +from . import proc_util +from .nexus_config import FRONTEND_SOURCE_DIR, RUNTIME_DIR -FRONTEND_DIR = PROJECT_ROOT / "interface" / "web" +FRONTEND_DIR = FRONTEND_SOURCE_DIR PID_FILE = RUNTIME_DIR / "pids" / "frontend.pid" LOG_FILE = RUNTIME_DIR / "frontend.log" PORT = 5173 @@ -41,12 +45,25 @@ def _is_ours(pid: int) -> bool: # the tracked PID's own cmdline. Loosen to "vite" (matches once the tree # gets that far) or "npm"+"dev" both present (matches the wrapper hop too). try: - cmd = " ".join(psutil.Process(pid).cmdline()) + if psutil is not None: + cmd = " ".join(psutil.Process(pid).cmdline()) + else: + cmd = proc_util.pid_cmdline(pid) except Exception: return False return "vite" in cmd or ("npm" in cmd and "dev" in cmd) +def _alive(pid: int | None) -> bool: + if pid is None: + return False + if psutil is not None: + return psutil.pid_exists(pid) + # proc_util rather than os.kill(pid, 0): same probe, but it also works + # when the PID belongs to another user. + return proc_util.pid_alive(pid) + + def _http_up() -> bool: try: with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/", timeout=1.5) as r: @@ -60,7 +77,7 @@ def is_running() -> bool: port actually answers (covers Vite started by another process, or a lost PID file) - not the stricter pattern match `stop()` uses before killing.""" pid = _read_pid() - if pid is not None and psutil.pid_exists(pid): + if _alive(pid): return True return _http_up() @@ -68,6 +85,8 @@ def is_running() -> bool: def start() -> dict: if is_running(): return {"status": "already_running"} + if not FRONTEND_DIR.is_dir(): + return {"status": "unavailable", "detail": "Vite source is not included in wheel installs"} npm = _npm() if not npm: return {"status": "error", "detail": "npm not found - install Node.js"} @@ -92,17 +111,20 @@ def start() -> dict: def stop() -> dict: pid = _read_pid() - if pid is not None and psutil.pid_exists(pid) and _is_ours(pid): + if _alive(pid) and _is_ours(pid): try: - proc = psutil.Process(pid) - # npm.cmd -> node -> vite is a multi-hop tree; kill it depth-first - # so the parent doesn't outlive its children as an orphaned shell. - for child in proc.children(recursive=True): - try: - child.terminate() - except Exception: - pass - proc.terminate() + if psutil is not None: + proc = psutil.Process(pid) + # npm.cmd -> node -> vite is a multi-hop tree; kill it depth-first + # so the parent doesn't outlive its children as an orphaned shell. + for child in proc.children(recursive=True): + try: + child.terminate() + except Exception: + pass + proc.terminate() + else: + proc_util.terminate_pid(pid) except Exception: pass PID_FILE.unlink(missing_ok=True) diff --git a/synapse/icons/compositor.py b/synapse/icons/compositor.py index dbd12bc..50ab3e7 100644 --- a/synapse/icons/compositor.py +++ b/synapse/icons/compositor.py @@ -5,9 +5,11 @@ import tempfile import os import re -_ROOT = Path(__file__).resolve().parents[2] -UNDERLAY_FILE = _ROOT / "assets/themes/NexusOS-icons-src/nexus-underlay.svg" -RING_FILE = _ROOT / "assets/themes/NexusOS-icons-src/nexus-underlay-ring.svg" +from ..nexus_config import settings + +_ROOT = settings.project_root +UNDERLAY_FILE = settings.assets_dir / "themes/NexusOS-icons-src/nexus-underlay.svg" +RING_FILE = settings.assets_dir / "themes/NexusOS-icons-src/nexus-underlay-ring.svg" ICONS_OUT = Path.home() / ".icons" / "NexusOS" SIZES = [16, 22, 24, 32, 48, 64, 128] @@ -18,7 +20,7 @@ _ALLOWED_ROOTS = [ "/opt", str(Path.home() / ".local/share/icons"), str(Path.home() / ".icons"), - str(_ROOT / "assets"), + str(settings.assets_dir), ] _UNDERLAY_FALLBACK = """\ diff --git a/synapse/main.py b/synapse/main.py index 3bd9147..03597e6 100644 --- a/synapse/main.py +++ b/synapse/main.py @@ -198,7 +198,6 @@ from .memory.store import store, MemoryItem from .playbooks.store import playbook_store, PlaybookItem from .search import needs_web_search, web_search - app = FastAPI(title="Synapse Backend", version=VERSION) # Alias for startup scripts @@ -1620,7 +1619,7 @@ async def delete_conversation(conversation_id: str): # ── Icon branding routes ────────────────────────────────────────────────────── -_REPO_ASSETS = str(Path(__file__).resolve().parents[1] / "assets") +_REPO_ASSETS = str(settings.assets_dir) _ALLOWED_ICON_ROOTS = [ "/usr/share/icons", "/usr/share/pixmaps", @@ -1701,7 +1700,7 @@ async def apply_icon_cache_route(): # and uses the Vite dev server as before. from fastapi.staticfiles import StaticFiles # noqa: E402 -_DIST = Path(__file__).resolve().parent.parent / "interface" / "web" / "dist" +_DIST = settings.web_dist_dir if _DIST.is_dir(): app.mount("/", StaticFiles(directory=str(_DIST), html=True), name="ui") diff --git a/synapse/nexus_config.py b/synapse/nexus_config.py index 4615248..bb0ddcf 100644 --- a/synapse/nexus_config.py +++ b/synapse/nexus_config.py @@ -1,7 +1,10 @@ # config.py from __future__ import annotations +import json import os +import shutil +from importlib import metadata from pathlib import Path from typing import Dict, Any @@ -12,14 +15,88 @@ try: except Exception: pass -# --- PROJECT ROOT --- -PROJECT_ROOT = Path(__file__).resolve().parent.parent +# --- INSTALL / RESOURCE LAYOUT --- +PACKAGE_DIR = Path(__file__).resolve().parent +_CHECKOUT_ROOT = PACKAGE_DIR.parent +SOURCE_CHECKOUT = ( + (_CHECKOUT_ROOT / "VERSION").is_file() + and (_CHECKOUT_ROOT / "interface" / "web" / "package.json").is_file() +) -# --- VERSION (single source of truth: the VERSION file at the repo root) --- +# PROJECT_ROOT remains the source checkout for developer installs. In a wheel it +# is the installed package directory; writable state is deliberately elsewhere. +PROJECT_ROOT = Path(os.getenv("NEXUS_PROJECT_ROOT", "")).expanduser() if os.getenv( + "NEXUS_PROJECT_ROOT" +) else (_CHECKOUT_ROOT if SOURCE_CHECKOUT else PACKAGE_DIR) +PROJECT_ROOT = PROJECT_ROOT.resolve() +RESOURCE_ROOT = PROJECT_ROOT if SOURCE_CHECKOUT else PACKAGE_DIR / "_resources" + + +def _user_dir(env_name: str, windows_leaf: str, xdg_name: str, xdg_fallback: str) -> Path: + # os.getenv's default only applies when a variable is *unset*. An exported + # but empty XDG_DATA_HOME / LOCALAPPDATA would otherwise give Path("") == + # ".", scattering state through whatever the cwd happened to be. The XDG + # spec says to treat an empty value as unset, so `or` - not a default arg. + override = os.getenv(env_name, "").strip() + if override: + return Path(override).expanduser().resolve() + if os.name == "nt": + base = Path(os.getenv("LOCALAPPDATA", "").strip() or Path.home() / "AppData" / "Local") + return (base / windows_leaf).expanduser().resolve() + base = Path(os.getenv(xdg_name, "").strip() or Path.home() / xdg_fallback).expanduser() + return (base / "nexusos").resolve() + + +CONFIG_DIR = _user_dir("NEXUS_CONFIG_DIR", "NexusOS", "XDG_CONFIG_HOME", ".config") +CONFIG_FILE = CONFIG_DIR / "config.json" + + +def read_user_config() -> dict[str, Any]: + try: + data = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except (OSError, ValueError, TypeError): + return {} + + +def write_user_config(values: dict[str, Any]) -> None: + """Atomically persist CLI-managed configuration.""" + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + tmp = CONFIG_FILE.with_suffix(".tmp") + tmp.write_text(json.dumps(values, indent=2, sort_keys=True) + "\n", encoding="utf-8") + tmp.replace(CONFIG_FILE) + + +USER_CONFIG = read_user_config() + + +def _value(key: str, env_name: str, default: Any) -> Any: + raw = os.getenv(env_name) + return raw if raw not in (None, "") else USER_CONFIG.get(key, default) + + +def _int_value(key: str, env_name: str, default: int) -> int: + try: + return int(_value(key, env_name, default)) + except (TypeError, ValueError): + return default + + +def _configured_path(key: str, env_name: str, default: Path) -> Path: + return Path(str(_value(key, env_name, default))).expanduser().resolve() + + +# --- VERSION (repo file in a checkout; distribution metadata in a wheel) --- try: - VERSION = (PROJECT_ROOT / "VERSION").read_text(encoding="utf-8").strip() or "0.0.0" + if SOURCE_CHECKOUT: + VERSION = (PROJECT_ROOT / "VERSION").read_text(encoding="utf-8").strip() + else: + VERSION = metadata.version("nexusos-ai") except Exception: - VERSION = "0.0.0" + try: + VERSION = (RESOURCE_ROOT / "VERSION").read_text(encoding="utf-8").strip() + except Exception: + VERSION = "0.0.0" # --- MODEL DEFAULTS --- # Single source of truth for the three models NexusOS ships with. The installers @@ -43,11 +120,24 @@ DEFAULT_MEMORY_MODEL = DEFAULT_CHAT_MODEL DEFAULT_EMBED_MODEL = "nomic-embed-text" # --- CORE DIRECTORIES --- -DATA_DIR = PROJECT_ROOT / "data" -MODELS_DIR = PROJECT_ROOT / "models" -RUNTIME_DIR = PROJECT_ROOT / "runtime" +_DEFAULT_STATE = PROJECT_ROOT if SOURCE_CHECKOUT else _user_dir( + "NEXUS_HOME", "NexusOS", "XDG_DATA_HOME", ".local/share" +) +STATE_DIR = _configured_path("home", "NEXUS_HOME", _DEFAULT_STATE) +_USE_CHECKOUT_STATE = SOURCE_CHECKOUT and not os.getenv("NEXUS_HOME", "").strip() +DATA_DIR = _configured_path("data_dir", "NEXUS_DATA_DIR", ( + PROJECT_ROOT / "data" if _USE_CHECKOUT_STATE else STATE_DIR / "data" +)) +MODELS_DIR = _configured_path("models_dir", "NEXUS_MODELS_DIR", ( + PROJECT_ROOT / "models" if _USE_CHECKOUT_STATE else STATE_DIR / "models" +)) +RUNTIME_DIR = _configured_path("runtime_dir", "NEXUS_RUNTIME_DIR", ( + PROJECT_ROOT / "runtime" if _USE_CHECKOUT_STATE else STATE_DIR / "runtime" +)) -MEMORY_DIR = PROJECT_ROOT / "synapse" / "memory" +MEMORY_DIR = _configured_path("memory_dir", "NEXUS_MEMORY_DIR", ( + PROJECT_ROOT / "synapse" / "memory" if _USE_CHECKOUT_STATE else DATA_DIR +)) LOGS_DIR = RUNTIME_DIR / "logs" CACHE_DIR = RUNTIME_DIR / "cache" @@ -57,9 +147,19 @@ TEMP_DIR = RUNTIME_DIR / "tmp" PLAYBOOK_DIR = DATA_DIR / "playbooks" # YAML playbook files (PlaybookFileStore) UPLOADS_DIR = DATA_DIR / "uploads" EXPORTS_DIR = DATA_DIR / "exports" +WEB_DIST_DIR = ( + PROJECT_ROOT / "interface" / "web" / "dist" + if SOURCE_CHECKOUT else RESOURCE_ROOT / "web" +) +FRONTEND_SOURCE_DIR = PROJECT_ROOT / "interface" / "web" +ASSETS_DIR = PROJECT_ROOT / "assets" if SOURCE_CHECKOUT else RESOURCE_ROOT / "assets" +SEED_PLAYBOOK_DIR = ( + PROJECT_ROOT / "data" / "playbooks" + if SOURCE_CHECKOUT else RESOURCE_ROOT / "playbooks" +) # --- DATABASE / STORAGE FILES (match your repo) --- -MEMORY_DB = MEMORY_DIR / "memory.db" +MEMORY_DB = _configured_path("memory_db", "NEXUS_MEMORY_DB", MEMORY_DIR / "memory.db") # --- LOG FILES --- BACKEND_LOG = RUNTIME_DIR / "backend.log" @@ -67,7 +167,8 @@ OLLAMA_LOG = LOGS_DIR / "ollama.log" CHAT_LOG = LOGS_DIR / "chat.log" # --- ENSURE REQUIRED DIRECTORIES EXIST --- -for d in ( +_REQUIRED_DIRS = ( + STATE_DIR, DATA_DIR, MODELS_DIR, RUNTIME_DIR, @@ -78,8 +179,25 @@ for d in ( PLAYBOOK_DIR, UPLOADS_DIR, EXPORTS_DIR, -): - d.mkdir(parents=True, exist_ok=True) + MEMORY_DB.parent, +) + + +def init_state() -> list[Path]: + """Create writable state and seed playbooks on a first wheel install.""" + for directory in _REQUIRED_DIRS: + directory.mkdir(parents=True, exist_ok=True) + copied: list[Path] = [] + if SEED_PLAYBOOK_DIR.resolve() != PLAYBOOK_DIR.resolve() and SEED_PLAYBOOK_DIR.is_dir(): + for source in SEED_PLAYBOOK_DIR.glob("*.yaml"): + target = PLAYBOOK_DIR / source.name + if not target.exists(): + shutil.copy2(source, target) + copied.append(target) + return copied + + +INITIALIZED_FILES = init_state() # --- PATH ACCESSOR (fail-fast) --- def path(name: str) -> Path: @@ -88,6 +206,9 @@ def path(name: str) -> Path: """ mapping = { "root": PROJECT_ROOT, + "resources": RESOURCE_ROOT, + "state": STATE_DIR, + "config": CONFIG_FILE, "data": DATA_DIR, "models": MODELS_DIR, "runtime": RUNTIME_DIR, @@ -98,6 +219,8 @@ def path(name: str) -> Path: "playbooks": PLAYBOOK_DIR, "uploads": UPLOADS_DIR, "exports": EXPORTS_DIR, + "web": WEB_DIST_DIR, + "assets": ASSETS_DIR, "memory_db": MEMORY_DB, "backend_log": BACKEND_LOG, "ollama_log": OLLAMA_LOG, @@ -146,12 +269,19 @@ class Settings: """ def __init__(self) -> None: self.version: str = VERSION + self.source_checkout: bool = SOURCE_CHECKOUT self.project_root: Path = PROJECT_ROOT + self.resource_root: Path = RESOURCE_ROOT + self.state_dir: Path = STATE_DIR + self.config_file: Path = CONFIG_FILE self.data_dir: Path = DATA_DIR self.models_dir: Path = MODELS_DIR self.runtime_dir: Path = RUNTIME_DIR self.memory_dir: Path = MEMORY_DIR self.logs_dir: Path = LOGS_DIR + self.web_dist_dir: Path = WEB_DIST_DIR + self.frontend_source_dir: Path = FRONTEND_SOURCE_DIR + self.assets_dir: Path = ASSETS_DIR # DB files self.memory_db: Path = MEMORY_DB @@ -166,23 +296,51 @@ class Settings: # should CONNECT. `ollama_bind` keeps the user's literal intent for a # serve we spawn (0.0.0.0 to expose it on the LAN); `ollama_host` is the # connectable form for our own requests. - self.ollama_bind: str = os.getenv("OLLAMA_HOST", "") or "127.0.0.1:11434" - self.ollama_host: str = _normalize_ollama_host( - os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434") + self.provider: str = str(_value("provider", "NEXUS_PROVIDER", "ollama")) + # A Nexus provider setting is more specific than the legacy Ollama bind + # variable. This matters on a desktop that has OLLAMA_HOST globally set + # but configures NexusOS to use a different remote inference machine. + provider_url = str(_value("provider_url", "NEXUS_PROVIDER_URL", "")).strip() + configured_host = ( + provider_url + or os.getenv("OLLAMA_HOST", "").strip() + or "http://127.0.0.1:11434" ) - self.ollama_timeout: int = int(os.getenv("OLLAMA_TIMEOUT", "120")) + self.ollama_bind: str = configured_host or "127.0.0.1:11434" + self.ollama_host: str = _normalize_ollama_host( + configured_host + ) + self.provider_url: str = self.ollama_host + self.manage_ollama: bool = self.provider == "ollama" + self.ollama_timeout: int = _int_value("provider_timeout", "OLLAMA_TIMEOUT", 120) + self.bind_host: str = str(_value( + "bind_host", "NEXUS_BIND_HOST", "127.0.0.1" + )) + self.backend_port: int = _int_value("backend_port", "NEXUS_BACKEND_PORT", 8000) + self.api_url: str = str(_value( + "api_url", "NEXUS_API", f"http://127.0.0.1:{self.backend_port}" + )).rstrip("/") def as_dict(self) -> Dict[str, Any]: return { "version": self.version, + "source_checkout": self.source_checkout, "project_root": str(self.project_root), + "resource_root": str(self.resource_root), + "state_dir": str(self.state_dir), + "config_file": str(self.config_file), "data_dir": str(self.data_dir), "models_dir": str(self.models_dir), "runtime_dir": str(self.runtime_dir), "memory_dir": str(self.memory_dir), "memory_db": str(self.memory_db), + "web_dist_dir": str(self.web_dist_dir), + "provider": self.provider, "ollama_host": self.ollama_host, "ollama_timeout": self.ollama_timeout, + "api_url": self.api_url, + "bind_host": self.bind_host, + "backend_port": self.backend_port, } # --- local-access allowlists (shared by the backend + memory FastAPI apps) --- @@ -203,8 +361,12 @@ _LOCAL_HOSTS = ["localhost", "127.0.0.1", "[::1]", "::1", "testserver"] _LOCAL_ORIGINS = [ f"http://{h}:{p}" for h in ("localhost", "127.0.0.1") - for p in (8000, 5173) + for p in ( + _int_value("backend_port", "NEXUS_BACKEND_PORT", 8000), + 5173, + ) ] +_LOCAL_ORIGINS.extend(["capacitor://localhost", "https://localhost"]) ALLOWED_HOSTS = _csv_env("NEXUS_ALLOWED_HOSTS", _LOCAL_HOSTS) ALLOWED_ORIGINS = _csv_env("NEXUS_ALLOWED_ORIGINS", _LOCAL_ORIGINS) @@ -250,9 +412,13 @@ settings = Settings() # explicit exports for static checkers and IDEs __all__ = ["Settings", "settings", "path", "VERSION", "DEFAULT_CHAT_MODEL", "DEFAULT_MEMORY_MODEL", "DEFAULT_EMBED_MODEL", - "PROJECT_ROOT", "DATA_DIR", "MODELS_DIR", "RUNTIME_DIR", + "PACKAGE_DIR", "PROJECT_ROOT", "RESOURCE_ROOT", "SOURCE_CHECKOUT", + "STATE_DIR", "CONFIG_DIR", "CONFIG_FILE", "USER_CONFIG", + "read_user_config", "write_user_config", "init_state", "INITIALIZED_FILES", + "DATA_DIR", "MODELS_DIR", "RUNTIME_DIR", "MEMORY_DIR", "LOGS_DIR", "PLAYBOOK_DIR", "UPLOADS_DIR", - "EXPORTS_DIR", "MEMORY_DB", + "EXPORTS_DIR", "MEMORY_DB", "WEB_DIST_DIR", "FRONTEND_SOURCE_DIR", + "ASSETS_DIR", "SEED_PLAYBOOK_DIR", "BACKEND_LOG", "OLLAMA_LOG", "CHAT_LOG", "ALLOWED_HOSTS", "ALLOWED_ORIGINS", "MAX_REQUEST_BYTES", "MAX_UPLOAD_BYTES", "MAX_PDF_PAGES", diff --git a/synapse/ollama_manager.py b/synapse/ollama_manager.py index 0281d23..66a56b0 100644 --- a/synapse/ollama_manager.py +++ b/synapse/ollama_manager.py @@ -20,7 +20,9 @@ _log = logging.getLogger("nexus.ollama") _ollama_manager = None # Bundled binary ships alongside the project; fall back to system PATH -_BUNDLED_OLLAMA = Path(__file__).resolve().parent.parent / "ollama" / "bin" / "ollama" +_BUNDLED_OLLAMA = settings.project_root / "ollama" / "bin" / ( + "ollama.exe" if os.name == "nt" else "ollama" +) # POSIX: detach the child into its own session so we can signal the whole group. # Windows has no setsid/killpg — run the child normally and terminate() it. @@ -340,7 +342,7 @@ class OllamaManager: self.running = False self._available = None # see is_available() - self.runtime_dir = Path(runtime_dir) if runtime_dir else Path(__file__).resolve().parent.parent / "runtime" + self.runtime_dir = Path(runtime_dir) if runtime_dir else settings.runtime_dir (self.runtime_dir / "logs").mkdir(parents=True, exist_ok=True) self.log_file = self.runtime_dir / "logs" / "ollama.log" @@ -406,6 +408,10 @@ class OllamaManager: return env def is_available(self): + if not settings.manage_ollama: + # A remote provider has no local executable to discover. Availability + # means it is configured; is_running() performs the live probe. + return True # ponytail: cached for the life of the process. This spawns a subprocess, # and /status calls it on every poll - the frontend polls continuously, # so it was a process spawn per tick to answer a question whose answer @@ -433,6 +439,9 @@ class OllamaManager: return False def start(self): + if not settings.manage_ollama: + _log.info("Remote Ollama is externally managed at %s", self._api_base) + return self.is_running() if not self.is_available(): _log.warning("Ollama not found at %s; skipping startup", _ollama_bin()) return False @@ -474,6 +483,9 @@ class OllamaManager: async def start_async(self): """Async-safe version of start() for use inside async startup handlers.""" + if not settings.manage_ollama: + _log.info("Remote Ollama is externally managed at %s", self._api_base) + return self.is_running() if not self.is_available(): _log.warning("Ollama not found at %s; skipping startup", _ollama_bin()) return False @@ -514,6 +526,9 @@ class OllamaManager: return False def stop(self): + if not settings.manage_ollama: + _log.info("Not stopping externally managed Ollama at %s", self._api_base) + return False # Terminate a server we spawned ourselves. if self.process: try: @@ -902,4 +917,4 @@ def shutdown_ollama() -> None: global _ollama_manager if _ollama_manager is not None: _ollama_manager.stop() - _ollama_manager = None \ No newline at end of file + _ollama_manager = None diff --git a/synapse/proc_util.py b/synapse/proc_util.py new file mode 100644 index 0000000..2e67c1b --- /dev/null +++ b/synapse/proc_util.py @@ -0,0 +1,212 @@ +"""Process inspection and termination that works with or without psutil. + +psutil became an optional extra when the wheel landed, so the base install +(notably `pip install nexusos-ai` on Windows) has to manage PIDs with the +stdlib alone. Callers should keep using psutil when it is importable - it is +faster and more precise - and fall back here when it is not. + +Windows has no signals: os.kill(pid, sig) special-cases CTRL_C_EVENT and +CTRL_BREAK_EVENT, treats sig 0 as an existence check, and calls +TerminateProcess(handle, sig) for *everything else*. So os.kill(pid, 15) is not +a polite request there - it is an immediate, unblockable kill with exit code +15, and there is no equivalent of SIGTERM. Everything below goes through the +Win32 API via ctypes so the intent is explicit at each call site rather than +resting on which signal numbers happen to be special. +""" +from __future__ import annotations + +import os +import signal +import subprocess +from pathlib import Path + +WINDOWS = os.name == "nt" + +# Win32 constants (winnt.h / processthreadsapi.h) +_SYNCHRONIZE = 0x00100000 +_PROCESS_TERMINATE = 0x0001 +_WAIT_TIMEOUT = 0x00000102 + +# Keep console windows from flashing on every helper subprocess. +_NO_WINDOW = subprocess.CREATE_NO_WINDOW if WINDOWS else 0 + + +def _kernel32(): + import ctypes + + return ctypes.WinDLL("kernel32", use_last_error=True) + + +def _run(argv: list[str], timeout: float = 10.0) -> str: + """Run a helper command, returning stdout ('' on any failure).""" + try: + done = subprocess.run( + argv, capture_output=True, text=True, timeout=timeout, + creationflags=_NO_WINDOW, + ) + except (OSError, subprocess.SubprocessError): + return "" + return done.stdout or "" + + +def pid_alive(pid: int | None) -> bool: + """True if the PID names a live process. + + On Windows this opens a handle and polls it; a signalled handle means the + process has exited. That is equivalent to os.kill(pid, 0) - CPython + special-cases signal 0 into an existence check there - but it also reports + True for a process owned by another user, where os.kill raises. + """ + if pid is None: + return False + try: + pid = int(pid) + except (TypeError, ValueError): + return False + if pid <= 0: + return False + if WINDOWS: + import ctypes + + k = _kernel32() + handle = k.OpenProcess(_SYNCHRONIZE, False, pid) + if not handle: + return False + try: + return k.WaitForSingleObject(ctypes.c_void_p(handle), 0) == _WAIT_TIMEOUT + finally: + k.CloseHandle(ctypes.c_void_p(handle)) + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True # exists, owned by someone else + except (OSError, ValueError): + return False + + +def terminate_pid(pid: int | None, force: bool = False) -> bool: + """Ask a process to exit. Returns True if the request was delivered.""" + if not pid_alive(pid): + return False + pid = int(pid) + if WINDOWS: + # No graceful path without a shared console; TerminateProcess is what + # psutil.terminate() resolves to on Windows anyway. + import ctypes + + k = _kernel32() + handle = k.OpenProcess(_PROCESS_TERMINATE, False, pid) + if not handle: + return False + try: + return bool(k.TerminateProcess(ctypes.c_void_p(handle), 1)) + finally: + k.CloseHandle(ctypes.c_void_p(handle)) + try: + os.kill(pid, signal.SIGKILL if force else signal.SIGTERM) + return True + except OSError: + return False + + +def pid_cmdline(pid: int | None) -> str: + """Full command line for a PID, or '' when it cannot be determined.""" + if pid is None: + return "" + try: + pid = int(pid) + except (TypeError, ValueError): + return "" + if WINDOWS: + for entry_pid, cmd in iter_processes(): + if entry_pid == pid: + return cmd + return "" + proc = Path(f"/proc/{pid}/cmdline") + try: + return proc.read_bytes().replace(b"\0", b" ").decode(errors="replace").strip() + except OSError: + pass + # macOS and other POSIX hosts without /proc. + out = _run(["ps", "-o", "command=", "-p", str(pid)]) + return out.strip() + + +def iter_processes() -> list[tuple[int, str]]: + """(pid, command_line) for every visible process. + + Windows needs CIM for command lines - the ctypes snapshot APIs only expose + image names, which is not enough to tell `uvicorn synapse.main` apart from + any other python.exe. This is slow, so it is strictly the no-psutil path. + """ + if WINDOWS: + out = _run([ + "powershell", "-NoProfile", "-NonInteractive", "-Command", + "Get-CimInstance Win32_Process | " + "ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }", + ], timeout=30.0) + entries: list[tuple[int, str]] = [] + for line in out.splitlines(): + head, _, cmd = line.partition("\t") + if head.strip().isdigit(): + entries.append((int(head), cmd.strip())) + return entries + + entries = [] + proc_root = Path("/proc") + if proc_root.is_dir(): + for entry in proc_root.iterdir(): + if not entry.name.isdigit(): + continue + try: + cmd = (entry / "cmdline").read_bytes() + except OSError: + continue + entries.append((int(entry.name), cmd.replace(b"\0", b" ").decode(errors="replace").strip())) + return entries + + for line in _run(["ps", "-A", "-o", "pid=,command="]).splitlines(): + head, _, cmd = line.strip().partition(" ") + if head.isdigit(): + entries.append((int(head), cmd.strip())) + return entries + + +def pids_listening_on(port: int) -> list[int]: + """PIDs holding a listening TCP socket on `port`.""" + pids: list[int] = [] + if WINDOWS: + for line in _run(["netstat", "-ano", "-p", "TCP"]).splitlines(): + parts = line.split() + # Proto Local Foreign State PID + if len(parts) < 5 or parts[3] != "LISTENING": + continue + local = parts[1] + if local.rsplit(":", 1)[-1] == str(port) and parts[4].isdigit(): + pids.append(int(parts[4])) + return sorted(set(pids)) + + # -t TCP, -l listening, -n numeric, -P no port names. + for line in _run(["lsof", "-nP", "-tiTCP:%d" % port, "-sTCP:LISTEN"]).splitlines(): + if line.strip().isdigit(): + pids.append(int(line.strip())) + if pids: + return sorted(set(pids)) + + for line in _run(["ss", "-lptnH", "sport = :%d" % port]).splitlines(): + # ... users:(("uvicorn",pid=1234,fd=3)) + marker = "pid=" + start = line.find(marker) + while start != -1: + digits = "" + for ch in line[start + len(marker):]: + if not ch.isdigit(): + break + digits += ch + if digits: + pids.append(int(digits)) + start = line.find(marker, start + 1) + return sorted(set(pids)) diff --git a/tests/test_cli_packaging.py b/tests/test_cli_packaging.py new file mode 100644 index 0000000..d3b1753 --- /dev/null +++ b/tests/test_cli_packaging.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def run_cli(tmp_path: Path, *args: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + for key in tuple(env): + if key.startswith("NEXUS_"): + env.pop(key) + env["NEXUS_HOME"] = str(tmp_path / "state") + env["NEXUS_CONFIG_DIR"] = str(tmp_path / "config") + return subprocess.run( + [sys.executable, "-m", "nexusos_cli.cli", *args], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + + +def test_help_exposes_portable_command_tree(tmp_path): + result = run_cli(tmp_path, "--help") + assert result.returncode == 0, result.stderr + for command in ("init", "config", "provider", "doctor", "serve", "models", "chat", "monitor"): + assert command in result.stdout + + +def test_legacy_cli_spellings_remain_compatible(): + from nexusos_cli.cli import _normalize_legacy_argv + + assert _normalize_legacy_argv(["start", "-b"]) == ["start", "backend"] + assert _normalize_legacy_argv(["stop", "--ai"]) == ["stop", "ai"] + assert _normalize_legacy_argv(["backup", "full"]) == ["backup", "--full"] + # -f is --follow for `logs`, but --frontend for start/stop. Translating it + # for logs turned `logs -f` into a one-shot tail of the frontend log. + assert _normalize_legacy_argv(["logs", "-f"]) == ["logs", "-f"] + assert _normalize_legacy_argv(["start", "-f"]) == ["start", "frontend"] + assert _normalize_legacy_argv(["restore", "-f"]) == ["restore"] + assert _normalize_legacy_argv(["help"]) == ["--help"] + + +def test_init_uses_external_state_and_seeds_playbooks(tmp_path): + result = run_cli(tmp_path, "init", "--json") + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert Path(payload["state_dir"]) == (tmp_path / "state").resolve() + assert payload["seeded_playbooks"] == len(list((ROOT / "data" / "playbooks").glob("*.yaml"))) + assert len(list((tmp_path / "state" / "data" / "playbooks").glob("*.yaml"))) > 0 + + +def test_config_persists_validated_values(tmp_path): + set_result = run_cli(tmp_path, "config", "set", "backend_port", "8123", "--json") + assert set_result.returncode == 0, set_result.stderr + + get_result = run_cli(tmp_path, "config", "get", "backend_port", "--json") + assert get_result.returncode == 0, get_result.stderr + assert json.loads(get_result.stdout)["backend_port"] == 8123 + + invalid = run_cli(tmp_path, "config", "set", "backend_port", "70000") + assert invalid.returncode == 2 + assert "between 1 and 65535" in invalid.stderr + + +def test_remote_provider_configuration_is_explicit(tmp_path): + result = run_cli( + tmp_path, "provider", "use", "remote", "--url", "http://phone-lan:11434", "--json" + ) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["provider"] == "ollama-remote" + assert payload["url"] == "http://phone-lan:11434" + + values = json.loads((tmp_path / "config" / "config.json").read_text(encoding="utf-8")) + assert values["provider"] == "ollama-remote" + + show = run_cli(tmp_path, "provider", "show", "--json") + assert show.returncode == 0, show.stderr + assert json.loads(show.stdout)["url"] == "http://phone-lan:11434" + + +def test_serve_rejects_invalid_ports_before_startup(tmp_path): + result = run_cli(tmp_path, "serve", "--port", "0") + assert result.returncode == 2 + assert "between 1 and 65535" in result.stderr + + +def test_remote_provider_is_never_stopped_or_force_killed(monkeypatch): + from nexusos_cli import ncp + + killed_ports = [] + killed_patterns = [] + monkeypatch.setattr(ncp.settings, "manage_ollama", False) + monkeypatch.setattr(ncp.settings, "ollama_host", "http://remote.test:11434") + monkeypatch.setattr(ncp, "kill_port", lambda port: killed_ports.append(port) or False) + monkeypatch.setattr( + ncp, "kill_matching", lambda patterns, force=False: killed_patterns.extend(patterns) or 0 + ) + + ncp.stop_ollama() + ncp.cmd_kill() + + assert 11434 not in killed_ports + assert "ollama serve" not in killed_patterns + + +def test_allow_lan_names_addresses_instead_of_disabling_the_host_check(): + """ALLOWED_HOSTS=* would switch TrustedHostMiddleware off entirely, and that + middleware is the DNS-rebinding defense for an unauthenticated API.""" + from nexusos_cli.cli import _lan_hostnames + + assert _lan_hostnames("192.168.1.20") == ["192.168.1.20"] + wildcard = _lan_hostnames("0.0.0.0") + assert wildcard, "a wildcard bind must resolve to concrete host names" + assert "*" not in wildcard + # IPv6 literals need to match a Host header written either way. + for name in wildcard: + if ":" in name and not name.startswith("["): + assert f"[{name}]" in wildcard + + +def test_empty_platform_dir_variables_do_not_put_state_in_the_cwd(tmp_path, monkeypatch): + """os.getenv's default only fires when a variable is *unset*; an exported + but empty XDG_DATA_HOME / LOCALAPPDATA made Path("") == "." the base, so + state landed in whatever directory the command happened to run from. + + Patching os.name to exercise the other platform's branch is not an option - + pathlib dispatches on it - so this checks the branch this host actually + takes.""" + from synapse import nexus_config + + if os.name == "nt": + monkeypatch.setenv("LOCALAPPDATA", "") + expected = Path.home() / "AppData" / "Local" / "NexusOS" + else: + monkeypatch.setenv("XDG_DATA_HOME", "") + expected = Path.home() / ".local/share" / "nexusos" + monkeypatch.delenv("NEXUS_HOME", raising=False) + monkeypatch.chdir(tmp_path) + + resolved = nexus_config._user_dir("NEXUS_HOME", "NexusOS", "XDG_DATA_HOME", ".local/share") + assert resolved.is_absolute() + assert tmp_path.resolve() != resolved.parent + assert resolved == expected.resolve() + + +def test_config_set_warns_before_orphaning_the_database(tmp_path): + """Repointing the DB does not move it - say so, or history looks deleted.""" + assert run_cli(tmp_path, "init").returncode == 0 + db = tmp_path / "state" / "data" / "memory.db" + db.parent.mkdir(parents=True, exist_ok=True) + db.write_bytes(b"SQLite format 3" + bytes(1)) + + result = run_cli(tmp_path, "config", "set", "memory_db", str(tmp_path / "elsewhere.db"), "--json") + assert result.returncode == 0, result.stderr + assert "warning" in json.loads(result.stdout) + + same = run_cli(tmp_path, "config", "set", "memory_db", str(db), "--json") + assert "warning" not in json.loads(same.stdout) diff --git a/tests/test_monitor.py b/tests/test_monitor.py new file mode 100644 index 0000000..b9eac43 --- /dev/null +++ b/tests/test_monitor.py @@ -0,0 +1,77 @@ +"""Tests for the ASCII monitor — renderer + collectors, no live stack required.""" +from __future__ import annotations + +from nexusos_cli.monitor import _bar, _fmt_bytes, _recent_tools, render_frame + + +def test_bar_bounds(): + assert _bar(0, 10, unicode=False) == "-" * 10 + assert _bar(1, 10, unicode=False) == "#" * 10 + assert _bar(0.5, 10, unicode=False).count("#") == 5 + + +def test_fmt_bytes(): + assert _fmt_bytes(None) == "—" + assert _fmt_bytes(512) == "512B" + assert _fmt_bytes(2048).endswith("K") + + +def test_render_frame_contains_sections(): + snap = { + "ts": "2026-08-20T12:00:00-05:00", + "version": "0.0.0", + "services": { + "backend": {"running": True, "pid": 11, "url": "http://127.0.0.1:8000"}, + "frontend": {"running": False, "pid": None, "url": "http://127.0.0.1:5173"}, + "provider": { + "provider": "ollama", + "url": "http://127.0.0.1:11434", + "reachable": True, + }, + }, + "api": { + "online": True, + "version": "0.0.0", + "ollama": "running", + "memories": 3, + "conversations": 2, + "playbooks": 1, + "models": 4, + "action_tool_policy": "ask", + }, + "host": {"cpu_pct": 12.5, "mem_used": 1_000_000_000, "mem_total": 8_000_000_000, "mem_pct": 12.5}, + "procs": {"cpu_pct": 1.0, "rss": 50_000_000, "pids": [11]}, + "toolchains": [ + {"lang": "python", "ready": True, "tool": "/usr/bin/python", "summary": "python"}, + {"lang": "rust", "ready": False, "tool": None, "summary": "rust"}, + ], + "recent_tools": ["run_snippet", "render_preview"], + "paths": {}, + } + frame = render_frame(snap, width=72, unicode=False) + assert "SERVICES" in frame + assert "RESOURCES" in frame + assert "DATA / TOOLS" in frame + assert "RUN TOOLCHAINS" in frame + assert "backend" in frame and "UP" in frame + assert "frontend" in frame and "DOWN" in frame + assert "run_snippet" in frame + assert "ready python" in frame + assert "missing rust" in frame + # Fixed-width box: every content line same length. + lengths = {len(line) for line in frame.splitlines()} + assert len(lengths) == 1 + + +def test_recent_tools_parses_status_sentinels(tmp_path): + log = tmp_path / "chat.log" + log.write_text( + "noise\n__status__tools\n__status__run_snippet\n" + 'payload {"name": "render_preview"}\n__status__remember\n', + encoding="utf-8", + ) + found = _recent_tools(log, limit=5) + assert "run_snippet" in found + assert "remember" in found + assert "render_preview" in found + assert "tools" not in found diff --git a/management/test_nexus_api.py b/tests/test_nexus_api.py similarity index 76% rename from management/test_nexus_api.py rename to tests/test_nexus_api.py index 2f0e06c..8eda300 100644 --- a/management/test_nexus_api.py +++ b/tests/test_nexus_api.py @@ -1,7 +1,5 @@ -"""Pin the SSE parser in nexus_api. Run: python management/test_nexus_api.py""" -import sys, os -sys.path.insert(0, os.path.dirname(__file__)) -from nexus_api import iter_chunks +"""Pin the SSE parser in nexus_api. Run: python tests/test_nexus_api.py""" +from nexusos_cli.nexus_api import iter_chunks # A realistic /chat/stream frame: two token chunks, a meta block, then done. lines = [ diff --git a/tests/test_packaging_deps.py b/tests/test_packaging_deps.py new file mode 100644 index 0000000..ec3e4de --- /dev/null +++ b/tests/test_packaging_deps.py @@ -0,0 +1,150 @@ +"""Guard the wheel's dependency list against drift. + +There are now two dependency declarations: requirements-base.txt (what the +desktop installers pip -r) and pyproject.toml (what the wheel ships). They will +drift. What actually breaks a user is narrower than "they differ", though: it +is an import that no declared distribution provides, so that is what this pins. +""" +from __future__ import annotations + +import ast +import sys +import tomllib +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SHIPPED_PACKAGES = ("synapse", "nexusos_cli", "modules") + +# Import name -> distribution name, where PyPI disagrees with the module. +DISTRIBUTION_OF = { + "docx": "python-docx", + "dotenv": "python-dotenv", + "faster_whisper": "faster-whisper", + "imap_tools": "imap-tools", + "sqlite_vec": "sqlite-vec", + "yaml": "pyyaml", + "PIL": "pillow", +} + +# Provided by another declared distribution rather than named directly. +TRANSITIVE = {"starlette", "socketio", "engineio"} + +# Modules that ship inside this repo. +FIRST_PARTY = {"synapse", "nexusos_cli", "modules", "management", "bin", "tests"} + + +def _pyproject() -> dict: + return tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + + +def _requirement_name(spec: str) -> str: + """'pypdf>=5,<7' -> 'pypdf'; strips extras and environment markers.""" + head = spec.split(";", 1)[0].strip() + for sep in ("[", "=", ">", "<", "!", "~", " "): + head = head.split(sep, 1)[0] + return head.strip().lower().replace("_", "-") + + +def _declared() -> set[str]: + project = _pyproject()["project"] + specs = list(project.get("dependencies", [])) + for extra in project.get("optional-dependencies", {}).values(): + specs.extend(extra) + return {_requirement_name(s) for s in specs} + + +def test_wheel_includes_every_shipped_package(): + wheel = _pyproject()["tool"]["hatch"]["build"]["targets"]["wheel"] + configured = set(wheel["packages"]) + assert configured == set(SHIPPED_PACKAGES) + + +def _imported_modules() -> set[str]: + """Top-level module names imported anywhere in the shipped packages.""" + found: set[str] = set() + for package in SHIPPED_PACKAGES: + for path in (ROOT / package).rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + # level > 0 is a relative (first-party) import. + if node.level == 0 and node.module: + found.add(node.module.split(".")[0]) + return found + + +def _third_party() -> set[str]: + return { + module for module in _imported_modules() + if module not in sys.stdlib_module_names + and module not in FIRST_PARTY + and module not in TRANSITIVE + and not module.startswith("_") + } + + +def test_every_third_party_import_is_a_declared_dependency(): + declared = _declared() + missing = sorted( + module for module in _third_party() + if DISTRIBUTION_OF.get(module, module).lower().replace("_", "-") not in declared + ) + assert not missing, ( + "a shipped package imports these, but pyproject.toml declares no " + f"distribution for them: {missing}. Add them to [project] dependencies " + "or an extra (and to DISTRIBUTION_OF here if the names differ)." + ) + + +def test_all_extra_is_the_union_of_the_capability_extras(): + extras = _pyproject()["project"]["optional-dependencies"] + combined: set[str] = set() + for name, specs in extras.items(): + if name in ("all", "dev", "standard"): + continue + combined.update(_requirement_name(s) for s in specs) + everything = {_requirement_name(s) for s in extras["all"]} + assert combined == everything, ( + "the 'all' extra drifted from the capability extras; " + f"missing={sorted(combined - everything)} extra={sorted(everything - combined)}" + ) + + +@pytest.mark.parametrize("name", ["fastapi", "uvicorn", "httpx", "pydantic", "pyyaml"]) +def test_core_runtime_is_a_hard_dependency_not_an_extra(name): + """These are imported at module scope, so the base install must carry them.""" + base = {_requirement_name(s) for s in _pyproject()["project"]["dependencies"]} + assert name in base + + +def test_optional_imports_are_lazy(): + """Anything only in an extra must not be imported at module scope. + + A base `pip install nexusos-ai` has none of the extras, so a top-level + `import psutil` in synapse would make the backend unimportable. + """ + base = {_requirement_name(s) for s in _pyproject()["project"]["dependencies"]} + offenders: list[str] = [] + for package in SHIPPED_PACKAGES: + for path in (ROOT / package).rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in tree.body: # module scope only + names: list[str] = [] + if isinstance(node, ast.Import): + names = [a.name.split(".")[0] for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names = [node.module.split(".")[0]] + for module in names: + if module in sys.stdlib_module_names or module in FIRST_PARTY: + continue + dist = DISTRIBUTION_OF.get(module, module).lower().replace("_", "-") + if dist not in base and module not in TRANSITIVE: + offenders.append(f"{path.relative_to(ROOT)}: {module}") + assert not offenders, ( + "optional dependencies imported at module scope (wrap in try/ImportError " + f"or import inside the function): {offenders}" + ) diff --git a/tests/test_proc_util.py b/tests/test_proc_util.py new file mode 100644 index 0000000..83a0f92 --- /dev/null +++ b/tests/test_proc_util.py @@ -0,0 +1,94 @@ +"""Pin the psutil-free process helpers. + +psutil is an optional extra, so on a base install these are the only way +`ncp stop` / `nexus status` can see or stop a service. Windows previously had +no fallback at all: pid_is_ours returned False, kill_port returned False, and +the terminate branch was POSIX-only, so stop was a no-op there. + +Probing must also stay side-effect free. That is easy to get wrong on Windows, +where os.kill(pid, sig) is TerminateProcess for every sig except 0 and the two +console-control events - os.kill(pid, 15) kills instead of asking. +""" +from __future__ import annotations + +import subprocess +import sys +import time + +import pytest + +from synapse import proc_util + + +def _spawn(): + return subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + + +def _wait_gone(proc, timeout=10.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if proc.poll() is not None: + return True + time.sleep(0.05) + return False + + +@pytest.fixture +def victim(): + proc = _spawn() + try: + yield proc + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) + + +def test_pid_alive_does_not_kill_the_process(victim): + """Probing must be side-effect free, however it is implemented.""" + for _ in range(5): + assert proc_util.pid_alive(victim.pid) is True + time.sleep(0.3) + assert victim.poll() is None, "pid_alive() terminated the process it probed" + + +def test_pid_alive_is_false_for_a_dead_pid(victim): + victim.kill() + victim.wait(timeout=10) + assert proc_util.pid_alive(victim.pid) is False + + +@pytest.mark.parametrize("pid", [None, 0, -1, "not-a-pid"]) +def test_pid_alive_rejects_junk(pid): + assert proc_util.pid_alive(pid) is False + + +def test_terminate_pid_actually_stops_it(victim): + assert proc_util.terminate_pid(victim.pid) is True + assert _wait_gone(victim), "terminate_pid() did not stop the process" + assert proc_util.pid_alive(victim.pid) is False + + +def test_terminate_pid_is_false_when_already_gone(victim): + victim.kill() + victim.wait(timeout=10) + assert proc_util.terminate_pid(victim.pid) is False + + +def test_pid_cmdline_identifies_the_process(victim): + cmd = proc_util.pid_cmdline(victim.pid) + if not cmd: + pytest.skip("no command-line source on this host") + assert "time.sleep" in cmd or "python" in cmd.lower() + + +def test_iter_processes_includes_this_interpreter(): + import os + + entries = proc_util.iter_processes() + if not entries: + pytest.skip("no process enumeration available on this host") + assert os.getpid() in {pid for pid, _ in entries}