forked from enderofwings/NexusOS
feat(chat): add run_snippet, an execution track beside the render track
render_preview validates markup and hands it to the browser, which renders it in an opaque-origin sandboxed iframe. Nothing executes server-side. That model fits HTML/SVG/JSX and cannot fit C, Rust or Erlang, which need a real toolchain - so those get a second tool instead of a widened first one. The split is the feature: the model picks a track by picking a tool, rather than picking a `lang` value from an enum where half the entries run server-side and half do not. synapse/code_run.py compiles and runs one file in a throwaway directory and returns a ```nexus-run fence carrying the source and its captured output together, so a model cannot paste output without the code that produced it. Backticks in the source are re-encoded as ` - still valid JSON, and it cannot close the fence early. It is not a sandbox, and the module docstring says so up front. What it gives is containment by layers: consent (an action tool, gated by action_tool_policy, per-call Approve/Deny on "ask"), static screening, a scrubbed environment in a temp dir, wall-clock and POSIX rlimits, and a network namespace on Linux where unprivileged userns are available. Screening is a tripwire against a model reaching for `requests` out of habit, not a boundary against an adversary; layers 1 and 3-5 are the load-bearing ones. Backend RUN_LANGS and frontend run-langs.js are separate registries because the two sides need different things - one executes, one labels - and neither should depend on the other at runtime. tests/test_tools.py asserts the key sets and the fence tag stay equal, so drift fails the gate instead of rendering a run result under the wrong language. tests/snippet_probes/ is a data catalog rather than inlined cases, so adding a language is a data change and the meta-tests can assert every RUN_LANGS key has both a smoke probe and a screening probe. Probes skip cleanly on hosts without the toolchain. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Athena Kaminsky
co-authored by
Claude Opus 5
Claude Opus 5
parent
425184a30b
commit
affba1805c
@@ -0,0 +1 @@
|
||||
"""Data-driven snippet probes exercised by tests/test_snippet_probes.py."""
|
||||
@@ -0,0 +1,274 @@
|
||||
"""Catalog of code-snippet probes for the execution track.
|
||||
|
||||
Each probe is a small, self-contained program (or deliberate reject) that the
|
||||
suite in tests/test_snippet_probes.py runs automatically. Keeping the corpus
|
||||
here — not inlined in the test file — means adding a language or a case is a
|
||||
data change, and the meta-tests can assert every RUN_LANGS key is covered.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Probe:
|
||||
"""One automatic snippet probe.
|
||||
|
||||
kind:
|
||||
run — critique must be clean, then code_run.run (skip if no toolchain)
|
||||
screen — critique must report at least one issue matching needle;
|
||||
never executed
|
||||
tool — same as run, but also through tools.dispatch("run_snippet")
|
||||
"""
|
||||
|
||||
id: str
|
||||
lang: str
|
||||
source: str
|
||||
kind: str = "run"
|
||||
expect_stdout: str | None = None # exact strip() match when set
|
||||
expect_stdout_contains: tuple[str, ...] = ()
|
||||
expect_stderr_contains: tuple[str, ...] = ()
|
||||
expect_exit: int | None = 0 # None = don't care; run-ok can be nonzero
|
||||
expect_ok: bool = True # False => stage failure (compile/timeout/…)
|
||||
expect_stage: str | None = None
|
||||
screen_needle: str | None = None # required substring in critique issues
|
||||
stdin: str = ""
|
||||
tags: tuple[str, ...] = field(default_factory=tuple)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Corpus — keep each source short; the suite runs every probe on every check.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PROBES: tuple[Probe, ...] = (
|
||||
# --- python (always available) -----------------------------------------
|
||||
Probe(
|
||||
id="py-hello",
|
||||
lang="python",
|
||||
source="print('probe-ok', 6 * 7)",
|
||||
expect_stdout="probe-ok 42",
|
||||
tags=("smoke", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-stdin",
|
||||
lang="python",
|
||||
source="import sys\nprint(sys.stdin.read().strip().upper())",
|
||||
stdin="nexus\n",
|
||||
expect_stdout="NEXUS",
|
||||
tags=("stdin", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-nonzero-exit",
|
||||
lang="python",
|
||||
source="import sys\nprint('before')\nsys.exit(3)",
|
||||
expect_stdout="before",
|
||||
expect_exit=3,
|
||||
tags=("exit", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-traceback",
|
||||
lang="python",
|
||||
source="print(1 / 0)",
|
||||
expect_exit=None, # nonzero, exact code is interpreter-dependent enough
|
||||
expect_stderr_contains=("ZeroDivisionError",),
|
||||
tags=("stderr", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-alias-py",
|
||||
lang="py",
|
||||
source="print('alias')",
|
||||
expect_stdout="alias",
|
||||
tags=("alias", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-tool-envelope",
|
||||
lang="python",
|
||||
source="print('via-tool', 2 + 2)",
|
||||
kind="tool",
|
||||
expect_stdout="via-tool 4",
|
||||
tags=("tool", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-screen-network",
|
||||
lang="python",
|
||||
source="import socket\nprint(socket.gethostname())",
|
||||
kind="screen",
|
||||
screen_needle="network",
|
||||
tags=("screen", "python"),
|
||||
),
|
||||
Probe(
|
||||
id="py-screen-subprocess",
|
||||
lang="python",
|
||||
source="import subprocess\nsubprocess.run(['true'])",
|
||||
kind="screen",
|
||||
screen_needle="processes",
|
||||
tags=("screen", "python"),
|
||||
),
|
||||
|
||||
# --- c -----------------------------------------------------------------
|
||||
Probe(
|
||||
id="c-hello",
|
||||
lang="c",
|
||||
source=(
|
||||
"#include <stdio.h>\n"
|
||||
"int main(void) {\n"
|
||||
' printf("c-probe %d\\n", 6 * 7);\n'
|
||||
" return 0;\n"
|
||||
"}\n"
|
||||
),
|
||||
expect_stdout="c-probe 42",
|
||||
tags=("smoke", "c"),
|
||||
),
|
||||
Probe(
|
||||
id="c-math",
|
||||
lang="c",
|
||||
source=(
|
||||
"#include <stdio.h>\n"
|
||||
"#include <math.h>\n"
|
||||
"int main(void) {\n"
|
||||
' printf("%.0f\\n", sqrt(144.0));\n'
|
||||
" return 0;\n"
|
||||
"}\n"
|
||||
),
|
||||
expect_stdout="12",
|
||||
tags=("libm", "c"),
|
||||
),
|
||||
Probe(
|
||||
id="c-compile-error",
|
||||
lang="c",
|
||||
source="int main(void) { return nope; }\n",
|
||||
expect_ok=False,
|
||||
expect_stage="compile",
|
||||
expect_exit=None,
|
||||
expect_stderr_contains=("nope",),
|
||||
tags=("compile", "c"),
|
||||
),
|
||||
Probe(
|
||||
id="c-screen-system",
|
||||
lang="c",
|
||||
source='#include <stdlib.h>\nint main(void){ system("true"); return 0; }\n',
|
||||
kind="screen",
|
||||
screen_needle="processes",
|
||||
tags=("screen", "c"),
|
||||
),
|
||||
Probe(
|
||||
id="c-screen-no-main",
|
||||
lang="c",
|
||||
source="int add(int a){ return a + 1; }\n",
|
||||
kind="screen",
|
||||
screen_needle="main()",
|
||||
tags=("screen", "c"),
|
||||
),
|
||||
|
||||
# --- cpp ---------------------------------------------------------------
|
||||
Probe(
|
||||
id="cpp-hello",
|
||||
lang="cpp",
|
||||
source=(
|
||||
"#include <iostream>\n"
|
||||
"int main() {\n"
|
||||
' std::cout << "cpp-probe " << (6 * 7) << "\\n";\n'
|
||||
" return 0;\n"
|
||||
"}\n"
|
||||
),
|
||||
expect_stdout="cpp-probe 42",
|
||||
tags=("smoke", "cpp"),
|
||||
),
|
||||
Probe(
|
||||
id="cpp-alias",
|
||||
lang="c++",
|
||||
source=(
|
||||
"#include <iostream>\n"
|
||||
"int main(){ std::cout << 9 << \"\\n\"; }\n"
|
||||
),
|
||||
expect_stdout="9",
|
||||
tags=("alias", "cpp"),
|
||||
),
|
||||
Probe(
|
||||
id="cpp-screen-socket",
|
||||
lang="cpp",
|
||||
source="#include <sys/socket.h>\nint main(){ return 0; }\n",
|
||||
kind="screen",
|
||||
screen_needle="network",
|
||||
tags=("screen", "cpp"),
|
||||
),
|
||||
|
||||
# --- rust --------------------------------------------------------------
|
||||
Probe(
|
||||
id="rust-hello",
|
||||
lang="rust",
|
||||
source='fn main() { println!("rust-probe {}", (1..=10).sum::<i32>()); }\n',
|
||||
expect_stdout="rust-probe 55",
|
||||
tags=("smoke", "rust"),
|
||||
),
|
||||
Probe(
|
||||
id="rust-alias-rs",
|
||||
lang="rs",
|
||||
source='fn main() { println!("rs"); }\n',
|
||||
expect_stdout="rs",
|
||||
tags=("alias", "rust"),
|
||||
),
|
||||
Probe(
|
||||
id="rust-compile-error",
|
||||
lang="rust",
|
||||
source="fn main() { let x: i32 = \"nope\"; }\n",
|
||||
expect_ok=False,
|
||||
expect_stage="compile",
|
||||
expect_exit=None,
|
||||
tags=("compile", "rust"),
|
||||
),
|
||||
Probe(
|
||||
id="rust-screen-command",
|
||||
lang="rust",
|
||||
source='fn main() { let _ = std::process::Command::new("true"); }\n',
|
||||
kind="screen",
|
||||
screen_needle="processes",
|
||||
tags=("screen", "rust"),
|
||||
),
|
||||
|
||||
# --- erlang ------------------------------------------------------------
|
||||
Probe(
|
||||
id="erl-escript",
|
||||
lang="erlang",
|
||||
source='main(_) -> io:format("erl-probe ~p~n", [lists:sum(lists:seq(1, 10))]).\n',
|
||||
expect_stdout="erl-probe 55",
|
||||
tags=("smoke", "erlang"),
|
||||
),
|
||||
Probe(
|
||||
id="erl-module",
|
||||
lang="erlang",
|
||||
source=(
|
||||
"-module(main).\n"
|
||||
"-export([main/1]).\n"
|
||||
'main(_) -> io:format("~p~n", [7 * 6]).\n'
|
||||
),
|
||||
expect_stdout="42",
|
||||
tags=("module", "erlang"),
|
||||
),
|
||||
Probe(
|
||||
id="erl-alias",
|
||||
lang="erl",
|
||||
source='main(_) -> io:format("alias~n").\n',
|
||||
expect_stdout="alias",
|
||||
tags=("alias", "erlang"),
|
||||
),
|
||||
Probe(
|
||||
id="erl-screen-os-cmd",
|
||||
lang="erlang",
|
||||
source='main(_) -> os:cmd("true").\n',
|
||||
kind="screen",
|
||||
screen_needle="processes",
|
||||
tags=("screen", "erlang"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def probes_by_tag(*tags: str) -> tuple[Probe, ...]:
|
||||
wanted = set(tags)
|
||||
return tuple(p for p in PROBES if wanted.intersection(p.tags))
|
||||
|
||||
|
||||
def covered_langs() -> set[str]:
|
||||
"""Canonical RUN_LANGS keys touched by at least one probe (aliases resolved)."""
|
||||
from synapse import code_run
|
||||
return {code_run.resolve_lang(p.lang) for p in PROBES}
|
||||
@@ -0,0 +1,308 @@
|
||||
"""The execution track: synapse/code_run.py.
|
||||
|
||||
Python is the only toolchain guaranteed present (it is the interpreter running
|
||||
these tests), so it carries the behavioural coverage — limits, streams, exit
|
||||
codes, isolation. The compiled languages are covered for the parts that hold
|
||||
without their toolchain installed (screening, argv shape, the missing-toolchain
|
||||
message) and skipped where they need it, so this file passes on a machine with
|
||||
no clang and no rustc as well as on one with both.
|
||||
"""
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from synapse import code_run
|
||||
|
||||
|
||||
def _requires(lang):
|
||||
entry = code_run.RUN_LANGS[lang]
|
||||
return pytest.mark.skipif(
|
||||
not entry["tool"](), reason=f"no {lang} toolchain on this machine"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_every_lang_has_a_complete_driver():
|
||||
for name, entry in code_run.RUN_LANGS.items():
|
||||
assert entry["summary"], name
|
||||
assert callable(entry["tool"]), name
|
||||
assert callable(entry["source_name"]), name
|
||||
assert callable(entry["run"]), name
|
||||
assert callable(entry["critique"]), name
|
||||
assert entry["install"], name
|
||||
|
||||
|
||||
def test_aliases_resolve_to_real_langs():
|
||||
for alias, target in code_run.ALIASES.items():
|
||||
assert target in code_run.RUN_LANGS, alias
|
||||
assert code_run.resolve_lang("C++") == "cpp"
|
||||
assert code_run.resolve_lang(" Py ") == "python"
|
||||
assert code_run.resolve_lang("javascript") == "javascript" # unknown passes through
|
||||
|
||||
|
||||
def test_erlang_source_name_follows_the_dialect():
|
||||
"""escript reads the same bytes two ways depending on the extension. Writing
|
||||
a bare main/1 script to main.erl makes it a module with no exports, which
|
||||
fails at load with an error that says nothing about the real problem."""
|
||||
assert code_run._erlang_source_name("main(_) -> ok.") == "main.escript"
|
||||
assert code_run._erlang_source_name("-module(main).\nmain(_) -> ok.") == "main.erl"
|
||||
|
||||
|
||||
def test_erlang_escript_gets_a_shebang_when_missing():
|
||||
"""OTP 28+ rejects shebang-less .escript with 'Premature end of file'."""
|
||||
bare = "main(_) -> ok.\n"
|
||||
out = code_run._erlang_write_source(bare)
|
||||
assert out.startswith("#!/usr/bin/env escript\n")
|
||||
assert out.endswith(bare)
|
||||
already = "#!/usr/bin/env escript\nmain(_) -> ok.\n"
|
||||
assert code_run._erlang_write_source(already) == already
|
||||
module = "-module(main).\n-export([main/1]).\nmain(_) -> ok.\n"
|
||||
assert code_run._erlang_write_source(module) == module
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Screening
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_clean_snippets_pass_screening():
|
||||
assert code_run.critique("python", "print(sum(range(100)))") == []
|
||||
assert code_run.critique("c", '#include <stdio.h>\nint main(void){puts("x");}') == []
|
||||
assert code_run.critique("rust", 'fn main(){ println!("x"); }') == []
|
||||
assert code_run.critique("erlang", 'main(_) -> io:format("x~n").') == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lang,source,needle", [
|
||||
("python", "import socket\nprint(socket)", "network"),
|
||||
("python", "import requests\nprint(requests)", "network"),
|
||||
("python", "import subprocess\nsubprocess.run(['ls'])", "processes"),
|
||||
("c", "#include <sys/socket.h>\nint main(void){return 0;}", "network"),
|
||||
("c", '#include <stdlib.h>\nint main(void){system("ls");}', "processes"),
|
||||
("rust", "fn main(){ std::process::Command::new(\"ls\"); }", "processes"),
|
||||
("erlang", 'main(_) -> os:cmd("ls").', "processes"),
|
||||
])
|
||||
def test_screening_catches_the_obvious_reaches(lang, source, needle):
|
||||
issues = code_run.critique(lang, source)
|
||||
assert issues, f"{lang} snippet passed screening"
|
||||
assert any(needle in i for i in issues), issues
|
||||
|
||||
|
||||
def test_screening_catches_absolute_paths_and_urls():
|
||||
assert any("absolute" in i for i in code_run.critique(
|
||||
"python", 'data = open("/etc/passwd").read()\nprint(data)'))
|
||||
# Escaped and raw Windows paths both count.
|
||||
assert any("absolute" in i for i in code_run.critique(
|
||||
"python", 'data = open("C:\\\\Users\\\\me").read()'))
|
||||
assert any("absolute" in i for i in code_run.critique(
|
||||
"python", r'data = open(r"C:\Users\me").read()'))
|
||||
assert any("http" in i for i in code_run.critique(
|
||||
"python", 'print("see https://example.com/data.csv")'))
|
||||
|
||||
|
||||
def test_url_in_a_comment_is_not_a_network_reach():
|
||||
"""A citation in a comment is not fetch(); bouncing it costs a useless retry."""
|
||||
c = (
|
||||
"/* spec: https://en.cppreference.com/w/c/string */\n"
|
||||
"#include <stdio.h>\n"
|
||||
"int main(void){ puts(\"ok\"); return 0; }\n"
|
||||
)
|
||||
assert code_run.critique("c", c) == []
|
||||
|
||||
|
||||
def test_erlang_nproc_ceiling_is_relative_to_the_user(monkeypatch):
|
||||
"""RLIMIT_NPROC is UID-scoped; an absolute 512 is not 512 of headroom."""
|
||||
monkeypatch.setattr(code_run, "_user_process_count", lambda: 400)
|
||||
assert code_run._max_procs_for("erlang") == 400 + code_run._ERLANG_PROC_HEADROOM
|
||||
assert code_run._max_procs_for("python") == code_run._MAX_PROCS
|
||||
monkeypatch.setattr(code_run, "_user_process_count", lambda: None)
|
||||
assert code_run._max_procs_for("erlang") is None
|
||||
|
||||
|
||||
def test_missing_entry_point_is_named_up_front():
|
||||
"""Without this the user sees a linker error about _main, or an escript load
|
||||
failure — neither of which says 'you forgot the entry point'."""
|
||||
assert any("main()" in i for i in code_run.critique("c", "int add(int a){return a+1;}"))
|
||||
assert any("main()" in i for i in code_run.critique("rust", "fn add(a: i32) -> i32 { a }"))
|
||||
assert any("main/1" in i for i in code_run.critique("erlang", "add(A) -> A + 1."))
|
||||
|
||||
|
||||
def test_empty_source_says_so_and_stops():
|
||||
issues = code_run.critique("python", " ")
|
||||
assert issues == ["source is empty or too short to run."]
|
||||
|
||||
|
||||
def test_unknown_lang_is_refused_by_both_entry_points():
|
||||
assert code_run.critique("brainfuck", "+++.")[0].startswith("cannot run")
|
||||
assert code_run.run("brainfuck", "+++.")["ok"] is False
|
||||
|
||||
|
||||
def test_oversized_source_is_refused_without_running():
|
||||
issues = code_run.critique("python", "x = 1\n" * 40_000)
|
||||
assert issues and "characters" in issues[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Execution (python: always available)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_stdout_and_exit_code_come_back():
|
||||
out = code_run.run("python", "print('hello', 6 * 7)")
|
||||
assert out["ok"] is True
|
||||
assert out["stdout"].strip() == "hello 42"
|
||||
assert out["exit_code"] == 0
|
||||
|
||||
|
||||
def test_streams_are_kept_separate():
|
||||
out = code_run.run("python", "import sys\nprint('o')\nprint('e', file=sys.stderr)")
|
||||
assert out["stdout"].strip() == "o"
|
||||
assert out["stderr"].strip() == "e"
|
||||
|
||||
|
||||
def test_nonzero_exit_is_still_a_successful_run():
|
||||
out = code_run.run("python", "raise SystemExit(3)")
|
||||
assert out["ok"] is True
|
||||
assert out["exit_code"] == 3
|
||||
|
||||
|
||||
def test_a_traceback_is_returned_not_raised():
|
||||
out = code_run.run("python", "print(1/0)")
|
||||
assert out["ok"] is True
|
||||
assert out["exit_code"] != 0
|
||||
assert "ZeroDivisionError" in out["stderr"]
|
||||
|
||||
|
||||
def test_stdin_is_piped_in():
|
||||
out = code_run.run("python", "import sys\nprint(sys.stdin.read().upper())", stdin="abc")
|
||||
assert out["stdout"].strip() == "ABC"
|
||||
|
||||
|
||||
def test_a_program_reading_stdin_with_none_given_does_not_hang():
|
||||
"""stdin is always closed rather than inherited: a snippet calling input()
|
||||
with nothing piped in would otherwise block on the server's own stdin until
|
||||
the wall clock killed it, five seconds of 'thinking' for nothing."""
|
||||
out = code_run.run("python", "print(input('prompt: '))")
|
||||
assert out["ok"] is True
|
||||
assert "EOFError" in out["stderr"]
|
||||
|
||||
|
||||
def test_an_infinite_loop_is_killed_and_explained():
|
||||
out = code_run.run("python", "while True:\n pass")
|
||||
assert out["ok"] is False
|
||||
assert out["stage"] == "run"
|
||||
assert "did not finish" in out["error"]
|
||||
|
||||
|
||||
def test_output_is_truncated_not_streamed_whole():
|
||||
out = code_run.run("python", f"print('x' * {code_run.MAX_OUTPUT * 3})")
|
||||
assert len(out["stdout"]) < code_run.MAX_OUTPUT + 200
|
||||
assert "truncated" in out["stdout"]
|
||||
|
||||
|
||||
def test_the_working_directory_is_ephemeral():
|
||||
"""Two runs must not see each other's files. The temp dir is the only thing
|
||||
standing between a snippet and the user's cwd, so its lifetime is worth a
|
||||
test rather than an assumption."""
|
||||
first = code_run.run("python", "open('note.txt', 'w').write('hi')\nprint('wrote')")
|
||||
assert first["ok"] is True, first
|
||||
second = code_run.run("python", "import os\nprint(os.path.exists('note.txt'))")
|
||||
assert second["stdout"].strip() == "False"
|
||||
|
||||
|
||||
def test_the_child_does_not_inherit_the_servers_environment():
|
||||
"""A snippet has no business reading the process environment it happens to
|
||||
be spawned from — that is where an API key would be."""
|
||||
import os
|
||||
os.environ["NEXUS_RUN_LEAK_PROBE"] = "secret"
|
||||
try:
|
||||
out = code_run.run(
|
||||
"python", "import os\nprint(os.environ.get('NEXUS_RUN_LEAK_PROBE'))"
|
||||
)
|
||||
finally:
|
||||
os.environ.pop("NEXUS_RUN_LEAK_PROBE", None)
|
||||
assert out["stdout"].strip() == "None"
|
||||
|
||||
|
||||
def test_home_points_at_the_scratch_dir():
|
||||
"""HOME is rewritten so a snippet writing a dotfile writes it somewhere that
|
||||
gets deleted, rather than into the user's real home."""
|
||||
out = code_run.run("python", "import os\nprint(os.path.expanduser('~'))")
|
||||
assert "nexus-run-" in out["stdout"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Toolchains
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_a_missing_toolchain_explains_itself(monkeypatch):
|
||||
"""The model has to be able to tell 'you cannot run this here' from 'your
|
||||
code is wrong' — otherwise it rewrites a correct program repeatedly."""
|
||||
monkeypatch.setitem(
|
||||
code_run.RUN_LANGS["rust"], "tool", lambda: None
|
||||
)
|
||||
out = code_run.run("rust", 'fn main(){ println!("x"); }')
|
||||
assert out["ok"] is False
|
||||
assert out["stage"] == "toolchain"
|
||||
assert "rustup.rs" in out["error"]
|
||||
assert "Show the code instead" in out["error"]
|
||||
|
||||
|
||||
@_requires("c")
|
||||
def test_c_compiles_and_runs():
|
||||
out = code_run.run("c", '#include <stdio.h>\nint main(void){printf("%d\\n", 6*7);return 0;}')
|
||||
assert out["ok"] is True, out
|
||||
assert out["stdout"].strip() == "42"
|
||||
|
||||
|
||||
@_requires("c")
|
||||
def test_a_compile_error_comes_back_as_a_compile_error():
|
||||
"""Stage matters: the compiler's diagnostics are the useful payload, and
|
||||
labelling this a run failure would hide that nothing ever executed."""
|
||||
out = code_run.run("c", "int main(void){ return oops; }")
|
||||
assert out["ok"] is False
|
||||
assert out["stage"] == "compile"
|
||||
assert "oops" in out["stderr"]
|
||||
|
||||
|
||||
@_requires("cpp")
|
||||
def test_cpp_compiles_and_runs():
|
||||
out = code_run.run("cpp", '#include <iostream>\nint main(){std::cout << 6*7 << "\\n";}')
|
||||
assert out["ok"] is True, out
|
||||
assert out["stdout"].strip() == "42"
|
||||
|
||||
|
||||
@_requires("rust")
|
||||
def test_rust_compiles_and_runs():
|
||||
out = code_run.run("rust", 'fn main(){ println!("{}", (1..=10).sum::<i32>()); }')
|
||||
assert out["ok"] is True, out
|
||||
assert out["stdout"].strip() == "55"
|
||||
|
||||
|
||||
@_requires("erlang")
|
||||
def test_erlang_runs_a_bare_escript():
|
||||
out = code_run.run("erlang", 'main(_) -> io:format("~p~n", [lists:sum(lists:seq(1,10))]).')
|
||||
assert out["ok"] is True, out
|
||||
assert out["stdout"].strip() == "55"
|
||||
|
||||
|
||||
@_requires("erlang")
|
||||
def test_erlang_runs_a_module_form_script():
|
||||
out = code_run.run(
|
||||
"erlang",
|
||||
"-module(main).\n-export([main/1]).\nmain(_) -> io:format(\"~p~n\", [7*6]).",
|
||||
)
|
||||
assert out["ok"] is True, out
|
||||
assert out["stdout"].strip() == "42"
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="unshare is Linux-only")
|
||||
def test_network_isolation_probe_is_honest():
|
||||
"""Either we got a namespace or we did not — but the probe must never claim
|
||||
one it did not verify, because the tool description promises 'no network' on
|
||||
the strength of it."""
|
||||
prefix = code_run._net_isolation()
|
||||
assert prefix in ([], ["unshare", "-rn"])
|
||||
if prefix:
|
||||
assert shutil.which("unshare")
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Automatic code-snippet probe suite.
|
||||
|
||||
Walks the catalog in tests/snippet_probes/catalog.py and, for every probe:
|
||||
|
||||
* screen — asserts critique rejects it (never executed)
|
||||
* run — asserts critique is clean, then runs via synapse.code_run
|
||||
(skipped cleanly when the host has no toolchain)
|
||||
* tool — same as run, plus tools.dispatch("run_snippet") envelope checks
|
||||
|
||||
Adding a language to RUN_LANGS without a smoke probe fails
|
||||
test_every_run_lang_has_a_smoke_probe. That is the point of the catalog:
|
||||
coverage is automatic and visible.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from synapse import code_run, tools
|
||||
from tests.snippet_probes.catalog import PROBES, Probe
|
||||
|
||||
|
||||
def _has_toolchain(lang: str) -> bool:
|
||||
key = code_run.resolve_lang(lang)
|
||||
entry = code_run.RUN_LANGS.get(key)
|
||||
return bool(entry and entry["tool"]())
|
||||
|
||||
|
||||
def _ids(probes=PROBES):
|
||||
return [p.id for p in probes]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog integrity (runs even when every compiled toolchain is missing)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_probe_ids_are_unique():
|
||||
ids = [p.id for p in PROBES]
|
||||
assert len(ids) == len(set(ids)), "duplicate probe ids in catalog"
|
||||
|
||||
|
||||
def test_every_run_lang_has_a_smoke_probe():
|
||||
"""New RUN_LANGS entry without a smoke probe = silent blind spot."""
|
||||
smoke = {
|
||||
code_run.resolve_lang(p.lang)
|
||||
for p in PROBES
|
||||
if "smoke" in p.tags and p.kind in ("run", "tool")
|
||||
}
|
||||
missing = set(code_run.RUN_LANGS) - smoke
|
||||
assert not missing, (
|
||||
f"RUN_LANGS without a smoke probe: {sorted(missing)}. "
|
||||
"Add a Probe(..., tags=('smoke', ...)) to tests/snippet_probes/catalog.py."
|
||||
)
|
||||
|
||||
|
||||
def test_every_run_lang_has_a_screen_probe():
|
||||
screened = {
|
||||
code_run.resolve_lang(p.lang)
|
||||
for p in PROBES
|
||||
if p.kind == "screen"
|
||||
}
|
||||
missing = set(code_run.RUN_LANGS) - screened
|
||||
assert not missing, (
|
||||
f"RUN_LANGS without a screening probe: {sorted(missing)}."
|
||||
)
|
||||
|
||||
|
||||
def test_catalog_langs_resolve_into_run_langs_or_aliases():
|
||||
for probe in PROBES:
|
||||
key = code_run.resolve_lang(probe.lang)
|
||||
assert key in code_run.RUN_LANGS, (
|
||||
f"probe {probe.id!r} lang={probe.lang!r} resolves to unknown {key!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-probe execution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("probe", PROBES, ids=_ids())
|
||||
def test_snippet_probe(probe: Probe):
|
||||
if probe.kind == "screen":
|
||||
_assert_screen(probe)
|
||||
return
|
||||
|
||||
issues = code_run.critique(probe.lang, probe.source)
|
||||
assert issues == [], f"{probe.id}: unexpected critique issues: {issues}"
|
||||
|
||||
if not _has_toolchain(probe.lang):
|
||||
pytest.skip(f"no {code_run.resolve_lang(probe.lang)} toolchain on this machine")
|
||||
|
||||
result = code_run.run(probe.lang, probe.source, stdin=probe.stdin)
|
||||
_assert_run_result(probe, result)
|
||||
|
||||
if probe.kind == "tool":
|
||||
_assert_tool_envelope(probe, result)
|
||||
|
||||
|
||||
def _assert_screen(probe: Probe) -> None:
|
||||
assert probe.screen_needle, f"{probe.id}: screen probe needs screen_needle"
|
||||
issues = code_run.critique(probe.lang, probe.source)
|
||||
assert issues, f"{probe.id}: expected screening to reject the snippet"
|
||||
assert any(probe.screen_needle in i for i in issues), (
|
||||
f"{probe.id}: needle {probe.screen_needle!r} not in {issues}"
|
||||
)
|
||||
# Screening is the whole point — do not execute a rejected snippet.
|
||||
# (A future change that runs despite issues would be a security regression.)
|
||||
|
||||
|
||||
def _assert_run_result(probe: Probe, result: dict) -> None:
|
||||
assert result.get("ok") is probe.expect_ok, (
|
||||
f"{probe.id}: ok={result.get('ok')} expected {probe.expect_ok}; full={result}"
|
||||
)
|
||||
if probe.expect_stage is not None:
|
||||
assert result.get("stage") == probe.expect_stage, result
|
||||
|
||||
if not probe.expect_ok:
|
||||
# Failure path: still check optional stderr/stdout breadcrumbs.
|
||||
for needle in probe.expect_stderr_contains:
|
||||
assert needle in (result.get("stderr") or ""), result
|
||||
for needle in probe.expect_stdout_contains:
|
||||
assert needle in (result.get("stdout") or ""), result
|
||||
return
|
||||
|
||||
stdout = (result.get("stdout") or "").strip()
|
||||
stderr = (result.get("stderr") or "")
|
||||
if probe.expect_stdout is not None:
|
||||
assert stdout == probe.expect_stdout, (
|
||||
f"{probe.id}: stdout {stdout!r} != {probe.expect_stdout!r}"
|
||||
)
|
||||
for needle in probe.expect_stdout_contains:
|
||||
assert needle in stdout, result
|
||||
for needle in probe.expect_stderr_contains:
|
||||
assert needle in stderr, result
|
||||
if probe.expect_exit is not None:
|
||||
assert result.get("exit_code") == probe.expect_exit, result
|
||||
else:
|
||||
# Explicit "don't care" still requires that a run happened.
|
||||
assert "exit_code" in result, result
|
||||
|
||||
|
||||
def _assert_tool_envelope(probe: Probe, direct: dict) -> None:
|
||||
out = json.loads(asyncio.run(tools.dispatch("run_snippet", {
|
||||
"lang": probe.lang,
|
||||
"source": probe.source,
|
||||
"stdin": probe.stdin,
|
||||
})))
|
||||
assert out.get("ok") is probe.expect_ok, out
|
||||
assert "fence" in out and out["fence"].startswith("```nexus-run\n"), out
|
||||
body = out["fence"].split("\n", 1)[1].rsplit("\n", 1)[0]
|
||||
envelope = json.loads(body)
|
||||
assert envelope.get("lang") == code_run.resolve_lang(probe.lang)
|
||||
if probe.expect_stdout is not None:
|
||||
assert (envelope.get("stdout") or "").strip() == probe.expect_stdout
|
||||
# Direct driver and tool path must agree on exit for successful runs.
|
||||
if probe.expect_ok and probe.expect_exit is not None:
|
||||
assert out.get("exit_code") == direct.get("exit_code") == probe.expect_exit
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# One-shot inventory (useful when running the file directly)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_probe_inventory_lists_toolchain_readiness():
|
||||
"""Not an assertion about readiness — just fails if the inventory shape
|
||||
breaks, so `pytest -k inventory -s` is a quick host capability dump."""
|
||||
rows = []
|
||||
for name, entry in code_run.RUN_LANGS.items():
|
||||
tool = entry["tool"]()
|
||||
n = sum(1 for p in PROBES if code_run.resolve_lang(p.lang) == name)
|
||||
rows.append({"lang": name, "ready": bool(tool), "probes": n, "tool": tool})
|
||||
assert rows and all(r["probes"] >= 1 for r in rows)
|
||||
# Printed only under -s; kept as a structured object for debuggability.
|
||||
print("snippet-probe inventory:", json.dumps(rows, indent=2))
|
||||
@@ -10,6 +10,7 @@ import asyncio
|
||||
import json
|
||||
|
||||
from synapse import tools
|
||||
from synapse import code_run
|
||||
from synapse.chat import _run_tool_loop
|
||||
|
||||
|
||||
@@ -347,6 +348,118 @@ def test_preview_langs_match_the_frontend_registry():
|
||||
)
|
||||
|
||||
|
||||
_FRONTEND_RUN_REGISTRY = ("interface", "web", "src", "preview", "run-langs.js")
|
||||
|
||||
|
||||
def _frontend_registry_keys(parts: tuple, name: str) -> list:
|
||||
"""Top-level keys of a `export const <name> = {...}` object literal."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
src = Path(__file__).resolve().parents[1].joinpath(*parts)
|
||||
text = src.read_text(encoding="utf-8")
|
||||
body = re.search(rf"^export const {name} = \{{\n(.*?)^\}};", text, re.S | re.M)
|
||||
assert body, f"could not find a {name} object literal in {src}"
|
||||
return re.findall(r"^ (\w+):", body.group(1), re.M)
|
||||
|
||||
|
||||
def test_run_langs_match_the_frontend_registry():
|
||||
"""Same failure mode as the preview registries, one track over: a language
|
||||
the backend can run but the frontend does not know about renders as raw JSON
|
||||
in the chat, and one the frontend labels but the backend refuses produces a
|
||||
tool error the user never asked for. Nothing at runtime couples them."""
|
||||
assert _frontend_registry_keys(_FRONTEND_RUN_REGISTRY, "RUN_LANGS") == list(
|
||||
code_run.RUN_LANGS
|
||||
), (
|
||||
"RUN_LANGS differs between synapse/code_run.py and "
|
||||
"interface/web/src/preview/run-langs.js - add the language to both."
|
||||
)
|
||||
|
||||
|
||||
def test_run_fence_tag_matches_the_frontend():
|
||||
"""The tag is the handshake: run_snippet emits it, Markdown.jsx dispatches on
|
||||
it. A mismatch shows the JSON envelope to the user as a code block."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
src = Path(__file__).resolve().parents[1].joinpath(*_FRONTEND_RUN_REGISTRY)
|
||||
m = re.search(r'export const RUN_FENCE_LANG = "([^"]+)"', src.read_text(encoding="utf-8"))
|
||||
assert m and m.group(1) == tools._RUN_FENCE_LANG
|
||||
|
||||
|
||||
def test_run_lang_enum_is_derived_not_repeated():
|
||||
schema, _ = tools.REGISTRY["run_snippet"]
|
||||
enum = schema["function"]["parameters"]["properties"]["lang"]["enum"]
|
||||
assert enum == list(code_run.RUN_LANGS)
|
||||
|
||||
|
||||
def test_run_snippet_is_an_action_tool():
|
||||
"""It executes code on the host, so action_tool_policy has to gate it.
|
||||
Slipping into STANDING_TOOLS (where render_preview lives, ungated) would make
|
||||
every 'run this' a subprocess with no consent step anywhere."""
|
||||
assert tools.is_action("run_snippet")
|
||||
assert "run_snippet" not in tools.STANDING_TOOLS
|
||||
assert "run_snippet" in tools.CUED_ACTION_TOOLS
|
||||
# ...and withholding actions has to actually withhold it.
|
||||
assert tools.schemas_for(["run_snippet"], allow_actions=False) == []
|
||||
|
||||
|
||||
def test_wants_code_run_needs_a_verb_not_a_language():
|
||||
"""A language name must not arm the run track. `python` in _RUN_HINTS would
|
||||
drag every mention of the language into a non-stream tool round - the exact
|
||||
'stuck thinking' problem that kept render_preview off by default."""
|
||||
assert tools.wants_code_run("run this and show me the output")
|
||||
assert tools.wants_code_run("does this compile?")
|
||||
assert not tools.wants_code_run("write me a python function that sorts a list")
|
||||
assert not tools.wants_code_run("explain how rust ownership works")
|
||||
|
||||
|
||||
def test_run_snippet_rejects_a_preview_language():
|
||||
out = json.loads(asyncio.run(tools.dispatch("run_snippet", {
|
||||
"lang": "html", "source": "<p>hello there</p>",
|
||||
})))
|
||||
assert out["ok"] is False
|
||||
assert "render_preview" in out["error"]
|
||||
|
||||
|
||||
def test_run_snippet_fence_survives_backticks_in_the_source():
|
||||
"""A backtick in the source would close the ```nexus-run fence early, and the
|
||||
rest of the envelope would spill into the chat as prose."""
|
||||
out = json.loads(asyncio.run(tools.dispatch("run_snippet", {
|
||||
"lang": "python", "source": "s = '``` still inside'\nprint(len(s))",
|
||||
})))
|
||||
assert out["ok"] is True, out
|
||||
body = out["fence"].split("\n", 1)[1].rsplit("\n", 1)[0]
|
||||
assert "```" not in body
|
||||
assert json.loads(body)["source"].startswith("s = '```")
|
||||
|
||||
|
||||
def test_run_snippet_reports_a_program_that_fails():
|
||||
"""A non-zero exit is a successful run, not a tool failure: its stderr is the
|
||||
answer. Reporting ok=False here would send the model into a retry loop over
|
||||
a program that did exactly what it was asked to demonstrate."""
|
||||
out = json.loads(asyncio.run(tools.dispatch("run_snippet", {
|
||||
"lang": "python",
|
||||
"source": "import sys\nprint('before')\nsys.exit(2)",
|
||||
})))
|
||||
assert out["ok"] is True
|
||||
assert out["exit_code"] == 2
|
||||
assert "before" in out["stdout"]
|
||||
|
||||
|
||||
def test_run_snippet_escalates_a_scaffold_on_retry():
|
||||
"""Same discipline as render_preview: first reject is issues-only; second
|
||||
gets a pattern. _attempt is supplied by the tool loop."""
|
||||
first = json.loads(asyncio.run(tools.dispatch("run_snippet", {
|
||||
"lang": "python", "source": "import socket\nprint(1)", "_attempt": 0,
|
||||
})))
|
||||
assert first["ok"] is False
|
||||
assert "scaffold" not in first
|
||||
second = json.loads(asyncio.run(tools.dispatch("run_snippet", {
|
||||
"lang": "python", "source": "import socket\nprint(1)", "_attempt": 1,
|
||||
})))
|
||||
assert second["ok"] is False
|
||||
assert "scaffold" in second and "print" in second["scaffold"]
|
||||
|
||||
|
||||
def test_preview_lang_enum_is_derived_not_repeated():
|
||||
schema, _ = tools.REGISTRY["render_preview"]
|
||||
enum = schema["function"]["parameters"]["properties"]["lang"]["enum"]
|
||||
|
||||
Reference in New Issue
Block a user
GitNexus · part of the Nexus stack