837 lines
34 KiB
Python
837 lines
34 KiB
Python
"""Tool-using playbook loop — the read-only MVP.
|
|
|
|
Run from nexus-core/ with the Promethean venv active: pytest -q
|
|
|
|
Hermetic: a fake manager stands in for Ollama, so no network/model is needed.
|
|
Guards the two pieces that would silently break the feature: the allowlist
|
|
filter and the tool-call loop's terminate-on-content behaviour.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
|
|
from synapse import tools
|
|
from synapse.chat import _run_tool_loop
|
|
|
|
|
|
def test_schemas_for_drops_unknown_names():
|
|
schemas = tools.schemas_for(["search_memory", "not_a_tool"])
|
|
names = [s["function"]["name"] for s in schemas]
|
|
assert names == ["search_memory"]
|
|
assert tools.schemas_for([]) == []
|
|
|
|
|
|
async def _drain(gen):
|
|
return [s async for s in gen]
|
|
|
|
|
|
def test_remember_writes_a_memory_fact(tmp_path, monkeypatch):
|
|
# The `remember` action tool persists a fact through the store.
|
|
import synapse.memory.store as store_mod
|
|
from synapse.memory.store import PersistentMemoryStore
|
|
fresh = PersistentMemoryStore(tmp_path / "m.db")
|
|
monkeypatch.setattr(tools, "store", fresh)
|
|
out = asyncio.run(tools.dispatch("remember", {"text": "user likes tea", "section": "Prefs"}))
|
|
assert "user likes tea" in out
|
|
assert any(it.text == "user likes tea" for it in fresh.all())
|
|
|
|
|
|
def test_action_tools_registered():
|
|
for name in ("web_search", "fetch_url", "remember"):
|
|
assert name in tools.REGISTRY
|
|
names = [s["function"]["name"] for s in tools.schemas_for(["web_search", "remember", "nope"])]
|
|
assert names == ["web_search", "remember"]
|
|
|
|
|
|
class _ActionManager:
|
|
"""Returns a `remember` (action) tool_call once, then plain content."""
|
|
def __init__(self):
|
|
self.n = 0
|
|
|
|
async def chat(self, **_):
|
|
self.n += 1
|
|
if self.n == 1:
|
|
return {"role": "assistant",
|
|
"tool_calls": [{"function": {"name": "remember", "arguments": {"text": "x"}}}]}
|
|
return {"role": "assistant", "content": "done"}
|
|
|
|
|
|
def _drive_with_decision(decision, monkeypatch):
|
|
from synapse import chat as chatmod
|
|
|
|
async def fake_dispatch(name, args):
|
|
return "saved-ok"
|
|
monkeypatch.setattr(tools, "dispatch", fake_dispatch)
|
|
|
|
async def run():
|
|
messages = [{"role": "user", "content": "remember x"}]
|
|
schemas = tools.schemas_for(["remember"])
|
|
gen = chatmod._run_tool_loop(_ActionManager(), messages, "m", schemas, None, None,
|
|
conversation_id="conv", policy="ask")
|
|
statuses = []
|
|
async for s in gen:
|
|
statuses.append(s)
|
|
if s.startswith("__approve__"):
|
|
w = chatmod.pending_approvals["conv"]
|
|
w["decisions"] = {"remember": decision}
|
|
w["event"].set()
|
|
return statuses, messages
|
|
|
|
return asyncio.run(run())
|
|
|
|
|
|
def test_ask_policy_pauses_then_runs_on_approve(monkeypatch):
|
|
statuses, messages = _drive_with_decision(True, monkeypatch)
|
|
assert any(s.startswith("__approve__") for s in statuses) # paused for approval
|
|
assert "__status__remember" in statuses # approved -> ran
|
|
assert any(m["role"] == "tool" and "saved-ok" in m["content"] for m in messages)
|
|
|
|
|
|
def test_ask_policy_skips_on_deny(monkeypatch):
|
|
statuses, messages = _drive_with_decision(False, monkeypatch)
|
|
assert any(s.startswith("__approve__") for s in statuses)
|
|
assert "__status__remember" not in statuses # denied -> never ran
|
|
assert any(m["role"] == "tool" and "declined" in m["content"] for m in messages)
|
|
|
|
|
|
def test_action_tools_gated_by_consent():
|
|
allow = ["search_memory", "web_search", "remember", "fetch_url"]
|
|
on = [s["function"]["name"] for s in tools.schemas_for(allow, allow_actions=True)]
|
|
off = [s["function"]["name"] for s in tools.schemas_for(allow, allow_actions=False)]
|
|
assert set(on) == set(allow) # all pass when actions allowed
|
|
assert off == ["search_memory"] # action tools withheld when not
|
|
assert tools.is_action("remember") and not tools.is_action("search_memory")
|
|
|
|
|
|
class _FakeManager:
|
|
"""Returns a tool_call on the first chat() call, plain content after."""
|
|
def __init__(self):
|
|
self.calls = 0
|
|
|
|
async def chat(self, **_):
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return {
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{"function": {"name": "search_memory", "arguments": {"query": "gpu"}}}
|
|
],
|
|
}
|
|
return {"role": "assistant", "content": "here is the answer"}
|
|
|
|
|
|
def test_tool_loop_runs_tool_then_stops(monkeypatch):
|
|
async def fake_dispatch(name, args):
|
|
assert name == "search_memory"
|
|
assert args == {"query": "gpu"}
|
|
return '[{"section": "GPU", "text": "Vega 20 4GB"}]'
|
|
|
|
monkeypatch.setattr(tools, "dispatch", fake_dispatch)
|
|
|
|
messages = [{"role": "user", "content": "what gpu do i have?"}]
|
|
schemas = tools.schemas_for(["search_memory"])
|
|
statuses = asyncio.run(_drain(
|
|
_run_tool_loop(_FakeManager(), messages, "m", schemas, None, None)
|
|
))
|
|
|
|
# heartbeat + one status sentinel per tool run
|
|
assert statuses == ["__status__tools", "__status__search_memory"]
|
|
# messages mutated in place: user -> assistant(tool_calls) -> tool(result);
|
|
# the final content turn is NOT appended (the streaming turn regenerates it).
|
|
assert [m["role"] for m in messages] == ["user", "assistant", "tool"]
|
|
assert "Vega 20" in messages[-1]["content"]
|
|
|
|
|
|
def test_tool_loop_degrades_when_model_returns_no_dict():
|
|
class _NoToolManager:
|
|
async def chat(self, **_):
|
|
return None # model can't do tools / errored
|
|
|
|
messages = [{"role": "user", "content": "hi"}]
|
|
before = list(messages)
|
|
statuses = asyncio.run(_drain(_run_tool_loop(_NoToolManager(), messages, "m", [{}], None, None)))
|
|
assert statuses == ["__status__tools"] # heartbeat only; no tool ran
|
|
assert messages == before # untouched -> falls back to a plain stream
|
|
|
|
|
|
def test_read_file_stays_inside_the_repo():
|
|
"""The repo-file tools are the fix for the model inventing paths like
|
|
`nexus/nlp.py`; the deny-list is what keeps them from reading secrets."""
|
|
import json
|
|
|
|
def read(p):
|
|
return asyncio.run(tools._read_file(p))
|
|
|
|
assert "escapes" in read("../../etc/passwd")
|
|
# a leading slash is treated as repo-relative, so it lands nowhere real
|
|
assert "root:" not in read("/etc/passwd")
|
|
assert "required" in read("")
|
|
# private data and heavy trees are refused even though they're in-repo
|
|
for denied in ("synapse/memory/memory.db", ".git/config", "Promethean/pyvenv.cfg"):
|
|
assert "not readable" in read(denied), denied
|
|
assert "does not exist" in read("nexus/nlp.py")
|
|
assert "PROJECT_ROOT" in json.loads(read("synapse/nexus_config.py"))["content"]
|
|
|
|
|
|
def test_list_files_globs_the_repo_without_leaking_denied_paths():
|
|
import json
|
|
|
|
hits = json.loads(asyncio.run(tools._list_files("synapse/**/*")))
|
|
assert "synapse/main.py" in hits
|
|
assert not [h for h in hits if h.endswith(".db") or "__pycache__" in h], hits
|
|
|
|
|
|
def test_routed_reference_playbook_contributes_its_tools(tmp_path, monkeypatch):
|
|
"""A reference playbook routed into the prompt must bring its tools with it.
|
|
Without this the model reads instructions like "you can read the codebase"
|
|
while being advertised zero tools — and narrates tool calls it never made."""
|
|
from synapse.main import _route_playbooks
|
|
from synapse.playbooks.store import PlaybookFileStore, PlaybookItem
|
|
|
|
# Own store, not data/playbooks: the live set is the operator's, and a
|
|
# published clone ships different playbooks - this asserted on data that
|
|
# travels with one machine.
|
|
store = PlaybookFileStore(tmp_path)
|
|
store.add_playbook(PlaybookItem(id="main", title="Main", goal="g",
|
|
instructions="i", order=0))
|
|
store.add_playbook(PlaybookItem(id="dev", title="NexusOS Developer", goal="g",
|
|
instructions="You can read the codebase.", order=1,
|
|
tags=["synapse", "backend"],
|
|
tools=["read_file", "list_files"]))
|
|
monkeypatch.setattr("synapse.playbook_manager.playbook_store", store)
|
|
import synapse.playbook_manager as pm
|
|
|
|
routed = _route_playbooks("why is the memory endpoint in synapse returning 500", pm.get_context_playbooks())
|
|
names = {pb.title for pb in routed}
|
|
assert "NexusOS Developer" in names, names
|
|
|
|
granted = {t for pb in routed for t in (pb.tools or [])}
|
|
assert {"read_file", "list_files"} <= granted, granted
|
|
# none of them are action tools, so they survive the default policy (off)
|
|
assert tools.schemas_for(sorted(granted), allow_actions=False)
|
|
def test_standing_schemas_include_render_preview():
|
|
names = [s["function"]["name"] for s in tools.standing_schemas()]
|
|
assert names == ["render_preview"]
|
|
assert "render_preview" in tools.STANDING_TOOLS
|
|
assert not tools.is_action("render_preview")
|
|
assert tools.wants_render_preview("visualize Collatz with a chart")
|
|
assert not tools.wants_render_preview("what's the weather vibe today")
|
|
|
|
|
|
def test_render_preview_rejects_canvas_that_never_draws():
|
|
bad = """<!DOCTYPE html><html><body>
|
|
<canvas id="c" width="480" height="240"></canvas>
|
|
<script>
|
|
const canvas = document.getElementById('c');
|
|
function spin(num) {
|
|
while (num !== 1) {
|
|
num = num % 2 === 0 ? num / 2 : 3 * num + 1;
|
|
canvas.width = canvas.width;
|
|
}
|
|
}
|
|
spin(40);
|
|
</script></body></html>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html", "title": "Demo", "markup": bad,
|
|
})))
|
|
assert out["ok"] is False
|
|
joined = " ".join(out.get("issues", []))
|
|
assert "draw" in joined.lower() or "getcontext" in joined.lower()
|
|
assert "scaffold" not in out # withheld on a first rejection
|
|
|
|
|
|
def test_render_preview_rejects_forty_by_forty_stub_with_scaffold():
|
|
# Tiny stubs fail critique; tool returns fix hints + generic scaffold — not a
|
|
# canned Collatz/Recamán demo.
|
|
bad = """<!DOCTYPE html><html><head><style>
|
|
.colla { width: 40px; height: 40px; background-color: #2563eb; }
|
|
</style></head><body>
|
|
<canvas id="c" width="40" height="40"></canvas>
|
|
<script>
|
|
const canvas = document.getElementById('c');
|
|
const ctx = canvas.getContext('2d');
|
|
ctx.fillRect(0, 0, 40, 40);
|
|
</script></body></html>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html", "markup": bad, "purpose": "interactive plot demo",
|
|
})))
|
|
assert out["ok"] is False
|
|
assert out.get("repaired") is not True
|
|
assert "fence" not in out or not out.get("fence")
|
|
assert "scaffold" not in out # withheld on a first rejection
|
|
assert "issues" in out
|
|
|
|
|
|
def test_render_preview_accepts_canvas_that_plots():
|
|
good = """<!DOCTYPE html><html><body>
|
|
<canvas id="c" width="480" height="240"></canvas>
|
|
<input id="n" type="number" value="27">
|
|
<button onclick="go()">Go</button>
|
|
<script>
|
|
const c = document.getElementById('c');
|
|
const ctx = c.getContext('2d');
|
|
function go() {
|
|
let n = +document.getElementById('n').value, seq = [];
|
|
while (n !== 1 && seq.length < 500) { seq.push(n); n = n % 2 === 0 ? n/2 : 3*n+1; }
|
|
seq.push(1);
|
|
const max = Math.max(...seq);
|
|
ctx.clearRect(0,0,c.width,c.height);
|
|
ctx.beginPath();
|
|
seq.forEach((v,i) => {
|
|
const x = i * (c.width / Math.max(1, seq.length-1));
|
|
const y = c.height - (v / max) * (c.height - 8);
|
|
if (i === 0) ctx.moveTo(x,y); else ctx.lineTo(x,y);
|
|
});
|
|
ctx.stroke();
|
|
}
|
|
go();
|
|
</script></body></html>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html", "markup": good, "purpose": "line plot of an iterative sequence",
|
|
})))
|
|
assert out["ok"] is True
|
|
assert out["fence"].startswith("```html\n")
|
|
assert "getContext" in out["fence"]
|
|
assert out.get("repaired") is not True
|
|
|
|
|
|
def test_code_that_throws_is_left_to_the_previews_own_error_channel():
|
|
"""This markup is broken twice over: getContext() is assigned to `canvas`
|
|
but drawn with `ctx`, and collatz() is called as coll(). Both used to be
|
|
rejected here by regex. Both now reach the browser, which reports them
|
|
precisely — verified against the real preview:
|
|
|
|
"Uncaught ReferenceError: ctx is not defined (line 4)"
|
|
"Uncaught ReferenceError: coll is not defined (line 5)"
|
|
|
|
Static guessing at runtime failures only ever caught the spellings someone
|
|
anticipated; the error channel catches every one of them and carries a line
|
|
number. What stays in the critique is the opposite case — markup that runs
|
|
perfectly and still isn't a visualization."""
|
|
broken_at_runtime = """<!DOCTYPE html><html><body>
|
|
<canvas id="c" width="480" height="280"></canvas>
|
|
<script>
|
|
const canvas = document.getElementById('c').getContext('2d');
|
|
function collatz(n) {
|
|
const s = [];
|
|
while (n !== 1 && s.length < 500) {
|
|
s.push(n);
|
|
n = n % 2 === 0 ? n / 2 : n * 3 + 1;
|
|
}
|
|
s.push(1);
|
|
return s;
|
|
}
|
|
function plot() {
|
|
const seq = coll(document.getElementById('n').value);
|
|
const max = Math.max(...seq), w = canvas.width, h = canvas.height;
|
|
ctx.clearRect(0, 0, w, h);
|
|
ctx.beginPath();
|
|
seq.forEach((v, i) => {
|
|
const x = i * (w / Math.max(1, seq.length - 1));
|
|
const y = h - (v / max) * h;
|
|
if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y);
|
|
});
|
|
ctx.stroke();
|
|
}
|
|
</script></body></html>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html",
|
|
"purpose": "interactive sequence plot",
|
|
"markup": broken_at_runtime,
|
|
})))
|
|
assert out["ok"] is True, out.get("issues")
|
|
|
|
|
|
def test_render_preview_rejects_decorative_svg_without_guessing_algorithm():
|
|
bad = """<!DOCTYPE html><html><body>
|
|
<div class="wrap">
|
|
<input id="n" type="number" value="27"><button id="go">Plot</button>
|
|
</div>
|
|
<svg width="480" height="200" viewBox="0 0 480 200">
|
|
<defs><linearGradient id="g"><stop offset="0%" stop-color="#1a1a1a"/></linearGradient></defs>
|
|
<rect x="0" y="0" width="480" height="200" fill="url(#g)"/>
|
|
<line x1="0" y1="100" x2="480" y2="100" stroke="#fff"/>
|
|
</svg>
|
|
<script>
|
|
document.getElementById('go').onclick = function() {
|
|
var line = document.createElementNS('line');
|
|
document.querySelector('.wrap').appendChild(line);
|
|
};
|
|
</script></body></html>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html",
|
|
"title": "Unknown sequence",
|
|
"purpose": "interactive sequence demo",
|
|
"markup": bad,
|
|
})))
|
|
assert out["ok"] is False
|
|
assert out.get("repaired") is not True
|
|
blob = json.dumps(out).lower()
|
|
assert "recaman" not in blob and "collatz" not in blob
|
|
assert "scaffold" not in out # withheld on a first rejection
|
|
|
|
|
|
def test_no_sequence_render_seed_helper():
|
|
assert not hasattr(tools, "sequence_render_seed")
|
|
|
|
|
|
_FRONTEND_REGISTRY = ("interface", "web", "src", "preview", "languages.js")
|
|
|
|
|
|
def _frontend_preview_langs() -> list[str]:
|
|
"""Top-level keys of PREVIEW_LANGS in the frontend's preview registry."""
|
|
import re
|
|
from pathlib import Path
|
|
src = Path(__file__).resolve().parents[1].joinpath(*_FRONTEND_REGISTRY)
|
|
text = src.read_text(encoding="utf-8")
|
|
body = re.search(r"^export const PREVIEW_LANGS = \{\n(.*?)^\};", text, re.S | re.M)
|
|
assert body, f"could not find a PREVIEW_LANGS object literal in {src}"
|
|
return re.findall(r"^ (\w+):", body.group(1), re.M)
|
|
|
|
|
|
def test_preview_langs_match_the_frontend_registry():
|
|
"""The render window is two registries — synapse/tools.py validates a
|
|
language, interface/web/src/Markdown.jsx renders it — and a language present
|
|
in only one degrades silently: the model emits a fence the UI shows as a
|
|
plain code block, or the UI offers a preview the tool refuses to produce.
|
|
Nothing at runtime couples them, so this is what keeps them in step."""
|
|
# Plain ASCII in the message: this is read off a Windows console, where
|
|
# pytest's output encoding mangles non-ASCII into replacement characters.
|
|
assert _frontend_preview_langs() == list(tools.PREVIEW_LANGS), (
|
|
"PREVIEW_LANGS differs between synapse/tools.py and "
|
|
"interface/web/src/Markdown.jsx - add the language to both."
|
|
)
|
|
|
|
|
|
def test_preview_lang_enum_is_derived_not_repeated():
|
|
schema, _ = tools.REGISTRY["render_preview"]
|
|
enum = schema["function"]["parameters"]["properties"]["lang"]["enum"]
|
|
assert enum == list(tools.PREVIEW_LANGS)
|
|
|
|
|
|
def test_render_preview_rejects_unknown_lang():
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "python", "markup": "print('hi')" * 5,
|
|
})))
|
|
assert out["ok"] is False
|
|
assert "lang must be" in out["error"]
|
|
|
|
|
|
def test_render_preview_accepts_a_jsx_component():
|
|
good = """export default function Counter() {
|
|
const [n, setN] = useState(0);
|
|
return (
|
|
<div>
|
|
<button onClick={() => setN(n + 1)}>count {n}</button>
|
|
</div>
|
|
);
|
|
}"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": good, "purpose": "interactive counter",
|
|
})))
|
|
assert out["ok"] is True, out.get("issues")
|
|
assert out["fence"].startswith("```jsx\n")
|
|
|
|
|
|
def test_interactive_ui_needs_no_canvas_but_a_chart_does():
|
|
"""Forms and calculators are interactive through DOM elements in either
|
|
HTML or JSX; only a request claiming to be a chart needs a drawing surface."""
|
|
component = """export default function Form() {
|
|
const [name, setName] = useState("");
|
|
return <label>Name <input value={name} onInput={(e) => setName(e.target.value)} /></label>;
|
|
}"""
|
|
ok = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": component, "purpose": "an interactive demo",
|
|
})))
|
|
assert ok["ok"] is True, ok.get("issues")
|
|
|
|
bad = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": component, "purpose": "a chart of the results",
|
|
})))
|
|
assert bad["ok"] is False
|
|
assert any("canvas" in i for i in bad["issues"])
|
|
|
|
html = """<!doctype html><html><body>
|
|
<label>Value <input id="value" type="number" value="2"></label>
|
|
<button onclick="result.textContent = +value.value * 2">Double</button>
|
|
<output id="result">4</output>
|
|
</body></html>"""
|
|
html_ok = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html", "markup": html, "purpose": "an interactive calculator demo",
|
|
})))
|
|
assert html_ok["ok"] is True, html_ok.get("issues")
|
|
|
|
|
|
def test_scaffold_is_withheld_until_the_model_has_failed_twice():
|
|
"""A complete, styled, runnable document handed to a struggling model gets
|
|
pasted rather than adapted — and then persists in the conversation and comes
|
|
back as retrieved context for later requests, carrying its example domain
|
|
with it. A transcript showed this scaffold's CSS reappearing verbatim in an
|
|
answer to an unrelated prompt, in a conversation where the tool was never
|
|
called. So the first rejection says only what is wrong."""
|
|
for args in ({"lang": "html", "markup": "<div>too short</div>"},
|
|
{"lang": "jsx", "markup": ""}):
|
|
first = json.loads(asyncio.run(tools.dispatch("render_preview", args)))
|
|
assert first["ok"] is False
|
|
assert "scaffold" not in first, args
|
|
assert "issues" in first or "error" in first
|
|
|
|
again = json.loads(asyncio.run(tools.dispatch(
|
|
"render_preview", {**args, "_attempt": 1})))
|
|
assert again["ok"] is False
|
|
assert "scaffold" in again, args
|
|
|
|
|
|
def test_repeat_reject_hands_back_the_language_that_was_asked_for():
|
|
"""Answering a rejected component with a full HTML document tells the model
|
|
to write the wrong thing entirely."""
|
|
jsx = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": "<div>too short</div>", "_attempt": 1,
|
|
})))
|
|
assert "export default function App" in jsx["scaffold"]
|
|
assert "<!DOCTYPE html>" not in jsx["scaffold"]
|
|
|
|
html = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html", "markup": "<div>too short</div>", "_attempt": 1,
|
|
})))
|
|
assert "<!DOCTYPE html>" in html["scaffold"]
|
|
|
|
empty = json.loads(asyncio.run(tools.dispatch(
|
|
"render_preview", {"lang": "tsx", "markup": "", "_attempt": 1})))
|
|
assert "export default function App" in empty["scaffold"]
|
|
|
|
|
|
def test_asking_for_a_preview_language_or_pointer_interaction_offers_the_tool():
|
|
"""Each of these is a real prompt from a transcript where the render window
|
|
should have been reachable. The first one was not: no hint matched
|
|
'mouse-over sensitive ... jsx', so the tool was never advertised and the
|
|
model answered about Euler's formula instead."""
|
|
for prompt in (
|
|
"Create a mouse-over sensitive Euler fluid field as a jsx or tsx",
|
|
"write me a small tsx component",
|
|
"make the particles react to hover",
|
|
"a real-time simulation I can drag",
|
|
):
|
|
assert tools.wants_render_preview(prompt), prompt
|
|
|
|
# Still narrow: ordinary chat must not pay for a tool turn.
|
|
for prompt in (
|
|
"what's the weather vibe today",
|
|
"summarise this email thread",
|
|
"write a concise paragraph about caching",
|
|
):
|
|
assert not tools.wants_render_preview(prompt), prompt
|
|
|
|
assert tools.wants_render_preview("compare these graphs")
|
|
|
|
|
|
def test_external_preview_resources_are_rejected_in_attributes_and_css():
|
|
for markup in (
|
|
'<img src=https://example.com/chart.png alt="chart">',
|
|
'<style>.chart { background: url("https://example.com/chart.png"); }</style>',
|
|
'<style>@import "https://example.com/chart.css";</style>',
|
|
):
|
|
issues = tools._critique_shared(markup)
|
|
assert any("external http(s)" in issue for issue in issues), markup
|
|
|
|
|
|
def test_every_preview_language_hints_for_itself():
|
|
for lang in tools.PREVIEW_LANGS:
|
|
assert lang in tools._RENDER_HINTS, lang
|
|
|
|
|
|
def test_size_guidance_never_quotes_the_minimum():
|
|
"""Weak models copy the first dimensions they read. Three transcripts
|
|
produced exactly 320x200 — the old minimum — including one that had a
|
|
480x280 example in front of it. Only the wanted size may be spoken."""
|
|
schema, _ = tools.REGISTRY["render_preview"]
|
|
surfaces = [json.dumps(schema)]
|
|
for lang, markup in (("html", '<canvas width="40" height="40"></canvas>' + "x" * 60),
|
|
("svg", '<svg width="40" height="40"><rect/></svg>' + "x" * 60)):
|
|
surfaces.append(json.dumps(asyncio.run(
|
|
tools._render_preview(lang=lang, markup=markup, purpose="a chart"))))
|
|
blob = " ".join(surfaces)
|
|
assert str(tools._MIN_CANVAS_W) not in blob, "the minimum leaked into guidance"
|
|
assert str(tools._STAGE_W) in blob
|
|
|
|
|
|
def test_prose_only_component_is_rejected_like_a_prose_page():
|
|
"""The JSX that started the Euler misunderstanding: a component returning
|
|
three paragraphs. It was accepted because the prose check lived only on the
|
|
html side."""
|
|
prose = """export default function App() {
|
|
return (
|
|
<div>
|
|
<h1>Euler's Formula</h1>
|
|
<p>The sum of the first n natural numbers is:</p>
|
|
<p>{`f(x) = ${sumOfCubes(10)}`}</p>
|
|
<p>For example, the sum of the cubes of the first 10 is: {sumOfCubes(10)}</p>
|
|
</div>
|
|
);
|
|
}"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": prose, "purpose": "Euler fluid field",
|
|
})))
|
|
assert out["ok"] is False
|
|
assert any("not a visualization" in i for i in out["issues"])
|
|
|
|
|
|
def test_a_path_parked_in_defs_is_not_a_plot():
|
|
"""Straight from a transcript: a long <path> inside <defs> — never drawn —
|
|
passed as proof of a real chart while the preview rendered an empty box."""
|
|
undrawn = (
|
|
'<svg width="480" height="280" xmlns="http://www.w3.org/2000/svg"><defs>'
|
|
'<path d="M10,20L 10,190L 20,180L 30,170L 40,160L 50,150L 60,140L 70,130L 200L 0L" />'
|
|
'</defs><rect x="0" y="0" width="480" height="280" fill="none" stroke="#000" /></svg>'
|
|
)
|
|
assert tools._decorative_svg_not_plot(undrawn)
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "svg", "markup": undrawn, "purpose": "a plot of the field",
|
|
})))
|
|
assert out["ok"] is False
|
|
|
|
# The same path where it actually renders is still a plot.
|
|
drawn = undrawn.replace("<defs>", "").replace("</defs>", "")
|
|
assert not tools._decorative_svg_not_plot(drawn)
|
|
|
|
|
|
def test_render_nudge_never_reaches_the_streaming_turn():
|
|
"""The nudge is a synthetic user turn. Left in place it becomes the last
|
|
thing the user appears to have said, and the model answers it — which is
|
|
exactly what shipped: "please provide the user's request for the rendering",
|
|
twice, in place of a bouncing particle system."""
|
|
from synapse.chat import _strip_internal_turns, _render_nudge_text
|
|
real = {"role": "user", "content": "draw me a bouncing particle system"}
|
|
kept = _strip_internal_turns([
|
|
real,
|
|
{"role": "assistant", "content": "", "tool_calls": [{"function": {"name": "x"}}]},
|
|
{"role": "tool", "content": "{}"},
|
|
{"role": "user", "content": _render_nudge_text()},
|
|
])
|
|
assert kept[-1] == real
|
|
assert len(kept) == 2
|
|
assert kept[0]["role"] == "user"
|
|
assert "Tool results" in kept[0]["content"]
|
|
assert "{}" in kept[0]["content"]
|
|
|
|
|
|
def test_normal_tool_results_reach_streaming_turn():
|
|
"""Flatten Ollama's tool roles without discarding the retrieved data."""
|
|
from synapse.chat import _strip_internal_turns
|
|
request = {"role": "user", "content": "what GPU do I have?"}
|
|
kept = _strip_internal_turns([
|
|
request,
|
|
{"role": "assistant", "content": "", "tool_calls": [{
|
|
"function": {"name": "search_memory", "arguments": {"query": "GPU"}},
|
|
}]},
|
|
{"role": "tool", "content": '[{"text":"Vega 20 4GB"}]'},
|
|
])
|
|
assert kept[-1] == request
|
|
assert "Vega 20 4GB" in kept[-2]["content"]
|
|
assert all(m.get("role") != "tool" and not m.get("tool_calls") for m in kept)
|
|
|
|
|
|
def test_render_nudge_says_only_what_to_do_next():
|
|
"""It cannot refer to something the model can't see, offer a way out, or
|
|
name a size or language that isn't the one we want — it gets answered
|
|
literally."""
|
|
from synapse.chat import _render_nudge_text
|
|
nudge = _render_nudge_text().lower()
|
|
assert "this user request" not in nudge # dangling reference -> "please provide it"
|
|
assert "clarif" not in nudge # escape hatch -> it gets taken
|
|
assert str(tools._MIN_CANVAS_W) not in nudge
|
|
assert f"{tools._STAGE_W}x{tools._STAGE_H}" in nudge
|
|
for lang in tools.PREVIEW_LANGS: # not a hardcoded "html or svg"
|
|
assert lang in nudge, lang
|
|
|
|
|
|
def test_every_language_offers_a_scaffold():
|
|
for lang in tools.PREVIEW_LANGS:
|
|
assert tools._scaffold_for(lang), f"{lang} has no scaffold"
|
|
|
|
|
|
def test_render_preview_rejects_jsx_with_no_component():
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": "const x = 1;\nconsole.log(x);\n// nothing to mount",
|
|
})))
|
|
assert out["ok"] is False
|
|
assert any("No component to mount" in i for i in out["issues"])
|
|
|
|
|
|
def test_render_preview_rejects_jsx_importing_a_third_party_module():
|
|
src = """import { motion } from "framer-motion";
|
|
export default function App() {
|
|
return <motion.div>hello there friend</motion.div>;
|
|
}"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": src,
|
|
})))
|
|
assert out["ok"] is False
|
|
assert any("framer-motion" in i for i in out["issues"])
|
|
|
|
|
|
def test_render_preview_allows_react_imports_in_jsx():
|
|
src = """import { useState } from "react";
|
|
export default function App() {
|
|
const [n] = useState(0);
|
|
return <p>count is {n} right now</p>;
|
|
}"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "jsx", "markup": src,
|
|
})))
|
|
assert out["ok"] is True, out.get("issues")
|
|
|
|
|
|
def test_render_preview_rejects_tiny_decorative_tile():
|
|
stub = """<div style="width:40px;height:40px;background:#2563eb;border:1px solid #000"></div>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html", "markup": stub,
|
|
})))
|
|
assert out["ok"] is False
|
|
|
|
|
|
def test_render_preview_rejects_prose_page():
|
|
bad = """<!DOCTYPE html><html><body>
|
|
<div><h1>Some Topic</h1>
|
|
<p>This explains an idea in several paragraphs without drawing anything.</p>
|
|
<ul><li>one</li><li>two</li><li>three</li><li>four</li></ul>
|
|
<p>To view a live Preview/Code toggle, use:</p>
|
|
<pre><html><body><p>more prose</p></body></html></pre>
|
|
</div></body></html>"""
|
|
out = json.loads(asyncio.run(tools.dispatch("render_preview", {
|
|
"lang": "html",
|
|
"title": "Topic",
|
|
"purpose": "interactive demo",
|
|
"markup": bad,
|
|
})))
|
|
assert out["ok"] is False
|
|
assert "scaffold" not in out # withheld on a first rejection
|
|
assert out.get("repaired") is not True
|
|
|
|
|
|
def test_last_ok_render_fence_prefers_tool_result():
|
|
from synapse.chat import _last_ok_render_fence
|
|
fence, meta = _last_ok_render_fence([
|
|
{"role": "tool", "content": json.dumps({
|
|
"ok": True,
|
|
"fence": "```html\n<canvas width=\"480\" height=\"280\"></canvas>\n```",
|
|
})},
|
|
])
|
|
assert fence.startswith("```html")
|
|
assert meta.get("ok") is True
|
|
|
|
|
|
def test_coerce_tool_calls_from_content_json():
|
|
from synapse.chat import _coerce_tool_calls
|
|
# Structured field wins.
|
|
structured = {"role": "assistant", "tool_calls": [
|
|
{"function": {"name": "get_time", "arguments": {}}}
|
|
]}
|
|
assert _coerce_tool_calls(structured)[0]["function"]["name"] == "get_time"
|
|
# Small models dump a complete call into content.
|
|
content_call = {
|
|
"role": "assistant",
|
|
"content": '{"name":"render_preview","arguments":{"lang":"svg","markup":"<svg/>"}}',
|
|
}
|
|
calls = _coerce_tool_calls(content_call, {"render_preview"})
|
|
assert len(calls) == 1
|
|
assert calls[0]["function"]["name"] == "render_preview"
|
|
assert calls[0]["function"]["arguments"]["lang"] == "svg"
|
|
|
|
# JSON quoted as part of an explanation is output, not an instruction to
|
|
# execute a tool (especially important for action tools such as remember).
|
|
embedded = {
|
|
"role": "assistant",
|
|
"content": (
|
|
'For example: {"name":"remember","arguments":{"text":"do not save"}} '
|
|
"is the tool-call shape."
|
|
),
|
|
}
|
|
assert _coerce_tool_calls(embedded, {"remember"}) == []
|
|
|
|
# Even a whole JSON object cannot call a tool that was not advertised.
|
|
assert _coerce_tool_calls(content_call, {"search_memory"}) == []
|
|
|
|
|
|
def test_tool_loop_runs_content_json_tool_call(monkeypatch):
|
|
"""qwen-style: first turn returns content-JSON tool call, second returns text."""
|
|
from synapse import chat as chatmod
|
|
|
|
class _ContentJsonManager:
|
|
def __init__(self):
|
|
self.n = 0
|
|
|
|
async def chat(self, **_):
|
|
self.n += 1
|
|
if self.n == 1:
|
|
return {
|
|
"role": "assistant",
|
|
"content": json.dumps({
|
|
"name": "render_preview",
|
|
"arguments": {
|
|
"lang": "svg",
|
|
"markup": (
|
|
'<svg xmlns="http://www.w3.org/2000/svg" width="320" height="200">'
|
|
'<circle cx="160" cy="100" r="60" fill="red"/></svg>'
|
|
),
|
|
},
|
|
}),
|
|
}
|
|
return {"role": "assistant", "content": "done"}
|
|
|
|
statuses, messages = asyncio.run(_drain_with_messages(
|
|
_ContentJsonManager(), "m", tools.standing_schemas(),
|
|
user="draw a circle",
|
|
))
|
|
assert any(s == "__status__render_preview" for s in statuses)
|
|
tool_msgs = [m for m in messages if m.get("role") == "tool"]
|
|
assert tool_msgs
|
|
assert json.loads(tool_msgs[0]["content"])["ok"] is True
|
|
|
|
|
|
def test_tool_loop_nudges_render_preview_on_visual_ask():
|
|
"""First turn skips tools; nudge forces a second turn that calls render_preview."""
|
|
class _SkipThenCall:
|
|
def __init__(self):
|
|
self.n = 0
|
|
|
|
async def chat(self, **_):
|
|
self.n += 1
|
|
if self.n == 1:
|
|
return {"role": "assistant", "content": "Sure, here is a chart in prose."}
|
|
if self.n == 2:
|
|
return {
|
|
"role": "assistant",
|
|
"tool_calls": [{
|
|
"function": {
|
|
"name": "render_preview",
|
|
"arguments": {
|
|
"lang": "svg",
|
|
"markup": (
|
|
'<svg xmlns="http://www.w3.org/2000/svg" width="480" height="280">'
|
|
'<rect width="480" height="280" fill="#111"/>'
|
|
'<text x="24" y="150" fill="#eee" font-size="24">hi</text></svg>'
|
|
),
|
|
},
|
|
}
|
|
}],
|
|
}
|
|
return {"role": "assistant", "content": "done"}
|
|
|
|
statuses, messages = asyncio.run(_drain_with_messages(
|
|
_SkipThenCall(), "m", tools.standing_schemas(),
|
|
user="Visualize the Collatz conjecture with an interactive chart",
|
|
))
|
|
assert any(s == "__status__render_preview" for s in statuses)
|
|
assert any(
|
|
m.get("role") == "user" and "render_preview tool now" in (m.get("content") or "")
|
|
for m in messages
|
|
)
|
|
|
|
|
|
async def _drain_with_messages(manager, model, schemas, user="draw a circle"):
|
|
messages = [{"role": "user", "content": user}]
|
|
statuses = await _drain(
|
|
_run_tool_loop(manager, messages, model, schemas, None, None)
|
|
)
|
|
return statuses, messages
|