refactor(synapse): backend updates, add icons module, relocate playbooks to data/playbooks

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

synapse/ollama_manager.py

@@ -1,15 +1,16 @@
 import asyncio
 import json
 import logging
-import re
 import subprocess
 import time
 import httpx
 import os
 import signal
 from pathlib import Path
+from typing import Any
 
 from .nexus_config import settings
+from .memory.store import store
 
 OLLAMA_PORT = 11434
 
@@ -131,10 +132,9 @@ def _detect_gpu_backend() -> tuple[str, dict]:
 
     if vulkan_ok:
         idx, name = _best_vulkan_device()
-        env_overrides: dict = {"OLLAMA_GPU": "vulkan"}
-        if idx > 0:
-            # Explicitly route Ollama to the discrete GPU when it isn't device 0
-            env_overrides["GGML_VK_VISIBLE_DEVICES"] = str(idx)
+        # Always pin to the selected device — without this, Ollama may use the Intel
+        # iGPU's shared system RAM as "VRAM" for models that don't fit on discrete VRAM.
+        env_overrides: dict = {"OLLAMA_VULKAN": "1", "GGML_VK_VISIBLE_DEVICES": str(idx)}
         _log.info("GPU backend: Vulkan device %d (%s)", idx, name)
         return f"vulkan ({name})", env_overrides
 
@@ -153,18 +153,36 @@ def _detect_gpu_backend() -> tuple[str, dict]:
     return "cpu", {}
 
 
-def _model_score(name: str) -> tuple:
-    """Score a model name for auto-selection. Higher tuple = better."""
-    lower = name.lower()
-    match = re.search(r'(\d+(?:\.\d+)?)[bB]', lower)
-    params = float(match.group(1)) if match else 7.0
-    # Tier-break: known quality models get a small bonus
-    quality = next(
-        (i for i, prefix in enumerate(("llama3", "llama2", "mistral", "gemma", "phi", "qwen"), 1)
-         if prefix in lower),
-        0,
-    )
-    return (params, quality)
+# Single source of model auto-selection preference. Ordered so small, GPU-fitting
+# models come first (qwen2.5:3b fits a 4GB card and is a strong all-rounder); the
+# tail differs by task. Prefix-matched against installed model names.
+_MODEL_PREFERENCE = {
+    "chat": ("qwen2.5:3b", "qwen2.5", "gemma3:1b", "gemma3", "phi3", "phi-3", "gemma2", "gemma"),
+    "code": ("qwen2.5:3b", "qwen2.5", "gemma3:1b", "gemma3", "phi3", "phi-3", "codellama", "deepseek-coder", "codegemma"),
+}
+
+
+def _preferred_model(models: list, preference) -> str | None:
+    """First installed model whose name starts with a preference prefix."""
+    for prefix in preference:
+        for m in models:
+            if m.lower().startswith(prefix.lower()):
+                return m
+    return None
+
+
+def _chat_options(temperature: float | None, num_gpu: int | None) -> dict:
+    """Assemble the Ollama `options` block from the knobs we expose.
+
+    Returns an empty dict when nothing is set so callers can omit `options`
+    entirely (preserving Ollama's defaults / auto behaviour).
+    """
+    opts: dict = {}
+    if temperature is not None:
+        opts["temperature"] = temperature
+    if num_gpu is not None:
+        opts["num_gpu"] = num_gpu
+    return opts
 
 
 class OllamaManager:
@@ -181,8 +199,43 @@ class OllamaManager:
         self._api_base = settings.ollama_host.rstrip("/")
 
         # Model selection cache
-        self._model_cache: str | None = None
-        self._model_cache_ts: float = 0.0
+        self._model_cache: dict = {}  # intent -> (model, monotonic_ts)
+
+        # Per-model offloadable layer count cache (never changes for a model)
+        self._layer_cache: dict[str, int] = {}
+
+        # How long Ollama keeps the model resident between requests. Applied to
+        # every chat/generate body so the model isn't reloaded on each message.
+        # Overridden from persisted settings at startup. Falsy → omit (Ollama's
+        # 5-minute default).
+        self.keep_alive: str | None = "30m"
+
+    def _apply_keep_alive(self, body: dict) -> dict:
+        """Add `keep_alive` (a top-level Ollama field) to a request body when set."""
+        if self.keep_alive:
+            body["keep_alive"] = self.keep_alive
+        return body
+
+    async def warm(self, model: str | None = None, num_gpu: int | None = None) -> None:
+        """Preload a model so the first request doesn't pay a cold load.
+        An empty-prompt /api/generate is Ollama's documented preload. Pass the
+        same `num_gpu` the real requests use, or the preloaded copy is placed
+        differently and gets reloaded on first use. Best-effort: never raises,
+        so a missing model or down server can't break startup."""
+        try:
+            model = model or await self.select_best_model()
+            if not model:
+                return
+            body = {"model": model, "prompt": "", "stream": False}
+            if num_gpu is not None:
+                body["options"] = {"num_gpu": num_gpu}
+            async with httpx.AsyncClient(timeout=120.0) as client:
+                await client.post(
+                    f"{self._api_base}/api/generate", json=self._apply_keep_alive(body),
+                )
+            _log.info("warm: preloaded model=%s num_gpu=%s keep_alive=%s", model, num_gpu, self.keep_alive)
+        except Exception as e:
+            _log.warning("warm: preload failed: %s", e)
 
     def is_available(self):
         bin_path = _ollama_bin()
@@ -328,12 +381,12 @@ class OllamaManager:
                 async with httpx.AsyncClient(timeout=300.0) as client:
                     r = await client.post(
                         f"{self._api_base}/api/generate",
-                        json={
+                        json=self._apply_keep_alive({
                             "model": model,
                             "prompt": prompt,
                             "system": system,
                             "stream": False,
-                        },
+                        }),
                     )
 
                 elapsed = time.perf_counter() - start
@@ -356,12 +409,12 @@ class OllamaManager:
                 async with client.stream(
                     "POST",
                     f"{self._api_base}/api/generate",
-                    json={
+                    json=self._apply_keep_alive({
                         "model": model,
                         "prompt": prompt,
                         "system": system,
                         "stream": True,
-                    },
+                    }),
                 ) as response:
                     response.raise_for_status()
                     async for line in response.aiter_lines():
@@ -391,17 +444,23 @@ class OllamaManager:
         model: str = "mistral",
         stream: bool = False,
         temperature: float | None = None,
+        num_gpu: int | None = None,
         **kwargs,
     ):
         """Multi-turn chat via /api/chat (accepts a messages array with roles)."""
         start = time.perf_counter()
         try:
             if stream:
-                return self._chat_stream(messages=messages, model=model, temperature=temperature, start=start)
+                return self._chat_stream(
+                    messages=messages, model=model, temperature=temperature,
+                    num_gpu=num_gpu, start=start,
+                )
             else:
                 body: dict = {"model": model, "messages": messages, "stream": False}
-                if temperature is not None:
-                    body["options"] = {"temperature": temperature}
+                opts = _chat_options(temperature, num_gpu)
+                if opts:
+                    body["options"] = opts
+                self._apply_keep_alive(body)
                 async with httpx.AsyncClient(timeout=300.0) as client:
                     r = await client.post(f"{self._api_base}/api/chat", json=body)
                 elapsed = time.perf_counter() - start
@@ -412,6 +471,33 @@ class OllamaManager:
             _log.exception("chat error after %.3fs: %s", elapsed, e)
             return None
 
+    async def embed(self, text: str, model: str = "nomic-embed-text") -> list[float] | None:
+        """Return an embedding vector for `text` via /api/embeddings.
+
+        Returns None on any failure so callers can fall back to lexical search —
+        a missing embedding model should never break chat or recall.
+        """
+        text = (text or "").strip()
+        if not text:
+            return None
+        try:
+            gpu_offload = store.get_settings().get("memory_gpu_offload", 0)
+            num_gpu = await self.resolve_num_gpu(gpu_offload, model)
+            body: dict[str, Any] = {"model": model, "prompt": text}
+            if num_gpu is not None:
+                body["options"] = {"num_gpu": num_gpu}
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                r = await client.post(
+                    f"{self._api_base}/api/embeddings",
+                    json=body,
+                )
+                r.raise_for_status()
+                vec = r.json().get("embedding")
+                return vec if vec else None
+        except Exception as e:
+            _log.debug("embed failed (model=%s): %s", model, e)
+            return None
+
     async def list_models(self) -> list[str]:
         """Return names of all locally installed Ollama models."""
         try:
@@ -422,34 +508,84 @@ class OllamaManager:
         except Exception:
             return []
 
-    async def select_best_model(self) -> str:
-        """Pick the highest-scoring available model, falling back to 'mistral'.
-
-        Result is cached for 60 seconds so rapid chat requests don't each
-        hit the Ollama API to build the model list.
+    async def select_best_model(self, intent: str = "chat") -> str:
+        """Pick the preferred installed model for `intent` ('chat' or 'code'),
+        falling back to any installed model, then 'mistral'. Cached ~60s per
+        intent so rapid requests don't rebuild the model list each time.
         """
         now = time.monotonic()
-        if self._model_cache and (now - self._model_cache_ts) < 60:
-            return self._model_cache
+        cached = self._model_cache.get(intent)
+        if cached and (now - cached[1]) < 60:
+            return cached[0]
 
         models = await self.list_models()
-        best = max(models, key=_model_score) if models else "mistral"
+        pref = _MODEL_PREFERENCE.get(intent, _MODEL_PREFERENCE["chat"])
+        best = _preferred_model(models, pref) or (models[0] if models else "mistral")
 
-        self._model_cache = best
-        self._model_cache_ts = now
+        self._model_cache[intent] = (best, now)
         return best
 
     def invalidate_model_cache(self):
         """Force next select_best_model() to re-query (e.g. after pull/delete)."""
-        self._model_cache = None
-        self._model_cache_ts = 0.0
+        self._model_cache = {}
 
-    async def _chat_stream(self, messages: list, model: str, start: float, temperature: float | None = None):
+    async def get_model_layers(self, model: str) -> int | None:
+        """Total offloadable layer count for `model` (repeating blocks + output layer).
+
+        Used to turn a CPU/GPU offload percentage into an Ollama `num_gpu`
+        value. Reads `<arch>.block_count` from /api/show and adds 1 for the
+        non-repeating output layer (Ollama reports e.g. 33 layers for a model
+        with block_count=32). Cached per-model since it never changes.
+        Returns None if the count can't be determined, so callers fall back
+        to Auto (no num_gpu override).
+        """
+        if model in self._layer_cache:
+            return self._layer_cache[model]
+        try:
+            async with httpx.AsyncClient(timeout=10.0) as client:
+                r = await client.post(f"{self._api_base}/api/show", json={"model": model})
+            r.raise_for_status()
+            info = r.json().get("model_info", {}) or {}
+            block_count = next(
+                (v for k, v in info.items() if k.endswith(".block_count")), None
+            )
+            layers = int(block_count) + 1 if block_count is not None else None
+        except Exception as e:
+            _log.warning("get_model_layers(%s) failed: %s", model, e)
+            layers = None
+        if layers:
+            self._layer_cache[model] = layers
+        return layers
+
+    async def resolve_num_gpu(self, gpu_offload, model: str) -> int | None:
+        """Convert a stored gpu_offload setting into an Ollama `num_gpu` value.
+
+        `gpu_offload` is -1 for Auto (returns None → no override, Ollama auto-fits)
+        or 0–100 for the percent of the model's layers to force onto the GPU
+        (0 = all CPU/RAM). Layer count is model-specific, resolved from the live
+        model. Returns None on anything unexpected so callers fall back to Auto.
+        """
+        try:
+            pct = int(gpu_offload)
+        except (TypeError, ValueError):
+            return None
+        if pct < 0:
+            return None
+        pct = min(pct, 100)
+        layers = await self.get_model_layers(model)
+        if not layers:
+            return None
+        return max(0, round(pct / 100 * layers))
+
+    async def _chat_stream(self, messages: list, model: str, start: float,
+                           temperature: float | None = None, num_gpu: int | None = None):
         """Async generator streaming tokens, then a final __meta__ stats sentinel."""
         try:
             body: dict = {"model": model, "messages": messages, "stream": True}
-            if temperature is not None:
-                body["options"] = {"temperature": temperature}
+            opts = _chat_options(temperature, num_gpu)
+            if opts:
+                body["options"] = opts
+            self._apply_keep_alive(body)
             async with httpx.AsyncClient(timeout=300.0) as client:
                 async with client.stream(
                     "POST",
@@ -484,7 +620,7 @@ class OllamaManager:
         except Exception as e:
             elapsed = time.perf_counter() - start
             _log.exception("chat stream error after %.3fs: %s", elapsed, e)
-            return
+            raise
 
 
 def initialize_ollama() -> OllamaManager: