diff --git a/install-windows.ps1 b/install-windows.ps1 index ee8b927..89e4b5f 100644 --- a/install-windows.ps1 +++ b/install-windows.ps1 @@ -305,33 +305,16 @@ Pop-Location if ($seedOk) { Write-OK "Default model set to $ChatModel" } else { Write-Warn "Could not persist default model - pick it at the top of the chat instead." } -Write-Step "Pulling models ($ChatModel for chat, $MemModel for memory, $EmbedModel for recall)" -# A prompt, not "press Ctrl+C to skip": Ctrl+C in PowerShell 5.1 terminates the -# whole script, so the escape hatch the installer advertised was also the one -# thing that stopped it finishing - no Ollama cleanup, no summary, no window -# close. Answering "n" declines the download and the installer carries on. -Write-Host " These are several GB. You can skip and pull them later from the Models tab." -ForegroundColor DarkGray -$pullAnswer = Read-Host " Download them now? [Y/n]" -if ($pullAnswer -match '^\s*(n|no)\s*$') { - Write-Warn "Model download skipped - get them from the Models tab when you are ready." -} else { -# No pipe: 'ollama pull' draws a progress bar with cursor control, and piping it -# (to Out-Host or anything else) buffers the redraws - the download then shows no -# output for minutes and reads as a hang. Let it own the console. -# No try/catch either: a native command that exits non-zero does not throw, so -# the catch never fired and a failed pull was reported as success. - ollama pull $ChatModel - if ($LASTEXITCODE -eq 0) { Write-OK "$ChatModel ready (default chat model)" } - else { Write-Warn "$ChatModel pull skipped/failed - pull it from the Models tab later." } - - ollama pull $MemModel - if ($LASTEXITCODE -eq 0) { Write-OK "$MemModel ready (memory curator)" } - else { Write-Warn "$MemModel pull skipped/failed - the memory service will fall back to the chat model." } - - ollama pull $EmbedModel - if ($LASTEXITCODE -eq 0) { Write-OK "$EmbedModel ready (conversation recall)" } - else { Write-Warn "$EmbedModel pull skipped/failed - recall will fall back to lexical search." } -} +# No auto-download: the right models depend on the machine (a 4GB GPU can't fit +# an 8B model). The Models tab detects VRAM/RAM and marks which models fit, so +# the user pulls the right ones there instead of us guessing several GB. +Write-Step "Skipping model download (pick hardware-appropriate models in the app)" +Write-Host " No models were downloaded. Open NexusOS -> Models: it detects your" -ForegroundColor DarkGray +Write-Host " VRAM/RAM and flags which models fit (green = GPU, yellow = CPU/RAM)." -ForegroundColor DarkGray +Write-Host " Pull at least:" -ForegroundColor DarkGray +Write-Host " - $EmbedModel (required for recall / document search)" -ForegroundColor Gray +Write-Host " - a chat model the Models tab marks as fitting your GPU (or $ChatModel on a big one)" -ForegroundColor Gray +Write-Host " - $MemModel for the memory curator (optional)" -ForegroundColor Gray # -- Make Ollama manual-start (NexusOS owns the lifecycle) ---------------------- Write-Step "Setting Ollama to manual start" diff --git a/interface/web/src/Models.jsx b/interface/web/src/Models.jsx index b66ef7e..7e4e9d1 100644 --- a/interface/web/src/Models.jsx +++ b/interface/web/src/Models.jsx @@ -8,6 +8,7 @@ export function Models({ onPullStateChange }) { const [pulling, setPulling] = useState(false); const [modelName, setModelName] = useState(""); const [pullProgress, setPullProgress] = useState(""); + const [recommended, setRecommended] = useState(null); // {hardware, models} const checkOllamaStatus = useCallback(async () => { try { @@ -42,8 +43,9 @@ export function Models({ onPullStateChange }) { } }, [checkOllamaStatus]); - const pullModel = async () => { - if (!modelName.trim()) { + const pullModel = async (nameArg) => { + const name = (typeof nameArg === "string" ? nameArg : modelName).trim(); + if (!name) { setError("Please enter a model name"); return; } @@ -51,13 +53,13 @@ export function Models({ onPullStateChange }) { setPulling(true); onPullStateChange?.(true); setError(""); - setPullProgress(""); + setPullProgress(`Pulling ${name}…`); try { const response = await fetch(`${API_BASE}/models/pull`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ name: modelName.trim() }), + body: JSON.stringify({ name }), }); if (!response.ok) throw new Error("Failed to pull model"); @@ -122,11 +124,19 @@ export function Models({ onPullStateChange }) { useEffect(() => { loadModels(); + fetch(`${API_BASE}/models/recommended`).then(r => r.ok ? r.json() : null).then(setRecommended).catch(() => {}); const interval = setInterval(loadModels, 30000); return () => clearInterval(interval); }, [loadModels]); + const installedNames = new Set(models.map(m => m.name.toLowerCase())); + const FIT = { + gpu: { label: "🟢 fits GPU", color: "#8aff8a" }, + ram: { label: "🟡 runs on RAM (CPU)", color: "#e8c65a" }, + no: { label: "🔴 too big", color: "#ff8a80" }, + }; + return (
{error && ( @@ -215,6 +225,40 @@ export function Models({ onPullStateChange }) { )}
+ {/* Recommended for your hardware */} + {recommended && ( +
+

Recommended for your hardware

+
+ {recommended.hardware.ram_gb ? `${recommended.hardware.ram_gb} GB RAM` : "RAM unknown"} + {" · "} + {recommended.hardware.vram_gb ? `${recommended.hardware.vram_gb} GB GPU` : "GPU VRAM unknown — sized by RAM"} +
+
+ {recommended.models.map((m) => { + const installed = installedNames.has(m.name.toLowerCase()); + const fit = FIT[m.fit] || FIT.no; + return ( +
+
+ {m.name} + · {m.params} · {m.size_gb} GB · {m.role} +
{m.note}
+
+ {fit.label} + {installed + ? ✓ installed + : } +
+ ); + })} +
+
+ )} + {/* Local Models List */}

diff --git a/synapse/hardware.py b/synapse/hardware.py new file mode 100644 index 0000000..a2aa83c --- /dev/null +++ b/synapse/hardware.py @@ -0,0 +1,81 @@ +"""Hardware detection + a curated model catalog for the Models page. + +Detects system RAM (reliable, psutil) and GPU VRAM (best-effort: nvidia-smi, +then Linux AMD sysfs). VRAM is None when it can't be read (e.g. AMD/Vulkan on +Windows) — the UI then recommends by RAM + model size alone. +""" +from __future__ import annotations + +import glob +import subprocess +from typing import Any, Dict, List, Optional + +# Curated local-friendly models with their Q4 on-disk sizes (GB) and role. +# Sizes are approximate default-quant download sizes. +CATALOG: List[Dict[str, Any]] = [ + {"name": "gemma2:2b", "size_gb": 1.6, "params": "2B", "role": "chat", "note": "Smallest; fast on any GPU"}, + {"name": "llama3.2:3b", "size_gb": 2.0, "params": "3B", "role": "chat", "note": "Small Llama, fits 4GB GPU"}, + {"name": "qwen3:4b", "size_gb": 2.5, "params": "4B", "role": "chat", "note": "Reasoning (Think toggle); great on a 4GB card"}, + {"name": "phi3:mini", "size_gb": 2.2, "params": "3.8B", "role": "chat", "note": "Strong for its size"}, + {"name": "mistral:latest", "size_gb": 4.1, "params": "7B", "role": "chat/memory","note": "Good curator; runs on CPU/8GB+ RAM"}, + {"name": "qwen2.5:7b", "size_gb": 4.7, "params": "7B", "role": "chat", "note": "Capable 7B"}, + {"name": "llama3.1:8b", "size_gb": 4.7, "params": "8B", "role": "chat", "note": "Higher quality; CPU or 6GB+ GPU"}, + {"name": "nomic-embed-text","size_gb": 0.27,"params": "—", "role": "embeddings", "note": "Required for recall / document RAG"}, +] + + +def _ram_gb() -> Optional[float]: + try: + import psutil + return round(psutil.virtual_memory().total / 1024 ** 3, 1) + except Exception: + return None + + +def _vram_gb() -> Optional[float]: + # NVIDIA + try: + r = subprocess.run( + ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=3, + ) + if r.returncode == 0 and r.stdout.strip(): + mb = max(int(x) for x in r.stdout.split()) + return round(mb / 1024, 1) + except Exception: + pass + # Linux AMD/Intel via DRM sysfs + try: + best = 0 + for p in glob.glob("/sys/class/drm/card*/device/mem_info_vram_total"): + try: + best = max(best, int(open(p).read().strip())) + except Exception: + pass + if best: + return round(best / 1024 ** 3, 1) + except Exception: + pass + return None # unknown (e.g. AMD/Vulkan on Windows) -> recommend by RAM + + +def detect() -> Dict[str, Any]: + return {"ram_gb": _ram_gb(), "vram_gb": _vram_gb()} + + +def _fit(size_gb: float, vram: Optional[float], ram: Optional[float]) -> str: + """Where a model can run: 'gpu' (fully resident), 'ram' (CPU), or 'no'.""" + if vram and size_gb + 1.0 <= vram: # ~1GB headroom for KV cache/context + return "gpu" + if ram and size_gb + 2.0 <= ram: # ~2GB headroom for the OS/app + return "ram" + return "no" + + +def recommend() -> Dict[str, Any]: + hw = detect() + models = [ + {**m, "fit": _fit(m["size_gb"], hw["vram_gb"], hw["ram_gb"])} + for m in CATALOG + ] + return {"hardware": hw, "models": models} diff --git a/synapse/main.py b/synapse/main.py index 5f6bb22..e761848 100644 --- a/synapse/main.py +++ b/synapse/main.py @@ -621,6 +621,13 @@ async def reorder_memory(payload: Dict[str, Any] = Body(...)): # ------------------------- # Models # ------------------------- +@app.get("/models/recommended") +async def models_recommended(): + """Curated models annotated with whether they fit this machine's VRAM/RAM.""" + from . import hardware + return hardware.recommend() + + @app.get("/models") async def get_models(): try: diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 894d717..3e70e03 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -43,6 +43,16 @@ def test_keep_alive_pins_the_model(): assert "keep_alive" not in mgr._apply_keep_alive({"model": "x"}) +def test_hardware_fit_logic(): + from synapse import hardware + assert hardware._fit(2.5, 4.0, 16.0) == "gpu" # 2.5+1 <= 4 -> fits GPU + assert hardware._fit(4.7, 4.0, 16.0) == "ram" # too big for 4GB GPU, fits RAM + assert hardware._fit(4.7, None, 16.0) == "ram" # VRAM unknown -> RAM + assert hardware._fit(40.0, 4.0, 16.0) == "no" # too big everywhere + rec = hardware.recommend() + assert "hardware" in rec and all("fit" in m for m in rec["models"]) + + def test_stt_status_endpoint(): # Reports whether local Whisper is installed; wiring must respond either way. resp = TestClient(app).get("/stt/status")