"""Catalog of code-snippet probes for the execution track. Each probe is a small, self-contained program (or deliberate reject) that the suite in tests/test_snippet_probes.py runs automatically. Keeping the corpus here — not inlined in the test file — means adding a language or a case is a data change, and the meta-tests can assert every RUN_LANGS key is covered. """ from __future__ import annotations from dataclasses import dataclass, field @dataclass(frozen=True) class Probe: """One automatic snippet probe. kind: run — critique must be clean, then code_run.run (skip if no toolchain) screen — critique must report at least one issue matching needle; never executed tool — same as run, but also through tools.dispatch("run_snippet") """ id: str lang: str source: str kind: str = "run" expect_stdout: str | None = None # exact strip() match when set expect_stdout_contains: tuple[str, ...] = () expect_stderr_contains: tuple[str, ...] = () expect_exit: int | None = 0 # None = don't care; run-ok can be nonzero expect_ok: bool = True # False => stage failure (compile/timeout/…) expect_stage: str | None = None screen_needle: str | None = None # required substring in critique issues stdin: str = "" tags: tuple[str, ...] = field(default_factory=tuple) # --------------------------------------------------------------------------- # Corpus — keep each source short; the suite runs every probe on every check. # --------------------------------------------------------------------------- PROBES: tuple[Probe, ...] = ( # --- python (always available) ----------------------------------------- Probe( id="py-hello", lang="python", source="print('probe-ok', 6 * 7)", expect_stdout="probe-ok 42", tags=("smoke", "python"), ), Probe( id="py-stdin", lang="python", source="import sys\nprint(sys.stdin.read().strip().upper())", stdin="nexus\n", expect_stdout="NEXUS", tags=("stdin", "python"), ), Probe( id="py-nonzero-exit", lang="python", source="import sys\nprint('before')\nsys.exit(3)", expect_stdout="before", expect_exit=3, tags=("exit", "python"), ), Probe( id="py-traceback", lang="python", source="print(1 / 0)", expect_exit=None, # nonzero, exact code is interpreter-dependent enough expect_stderr_contains=("ZeroDivisionError",), tags=("stderr", "python"), ), Probe( id="py-alias-py", lang="py", source="print('alias')", expect_stdout="alias", tags=("alias", "python"), ), Probe( id="py-tool-envelope", lang="python", source="print('via-tool', 2 + 2)", kind="tool", expect_stdout="via-tool 4", tags=("tool", "python"), ), Probe( id="py-screen-network", lang="python", source="import socket\nprint(socket.gethostname())", kind="screen", screen_needle="network", tags=("screen", "python"), ), Probe( id="py-screen-subprocess", lang="python", source="import subprocess\nsubprocess.run(['true'])", kind="screen", screen_needle="processes", tags=("screen", "python"), ), # --- c ----------------------------------------------------------------- Probe( id="c-hello", lang="c", source=( "#include \n" "int main(void) {\n" ' printf("c-probe %d\\n", 6 * 7);\n' " return 0;\n" "}\n" ), expect_stdout="c-probe 42", tags=("smoke", "c"), ), Probe( id="c-math", lang="c", source=( "#include \n" "#include \n" "int main(void) {\n" ' printf("%.0f\\n", sqrt(144.0));\n' " return 0;\n" "}\n" ), expect_stdout="12", tags=("libm", "c"), ), Probe( id="c-compile-error", lang="c", source="int main(void) { return nope; }\n", expect_ok=False, expect_stage="compile", expect_exit=None, expect_stderr_contains=("nope",), tags=("compile", "c"), ), Probe( id="c-screen-system", lang="c", source='#include \nint main(void){ system("true"); return 0; }\n', kind="screen", screen_needle="processes", tags=("screen", "c"), ), Probe( id="c-screen-no-main", lang="c", source="int add(int a){ return a + 1; }\n", kind="screen", screen_needle="main()", tags=("screen", "c"), ), # --- cpp --------------------------------------------------------------- Probe( id="cpp-hello", lang="cpp", source=( "#include \n" "int main() {\n" ' std::cout << "cpp-probe " << (6 * 7) << "\\n";\n' " return 0;\n" "}\n" ), expect_stdout="cpp-probe 42", tags=("smoke", "cpp"), ), Probe( id="cpp-alias", lang="c++", source=( "#include \n" "int main(){ std::cout << 9 << \"\\n\"; }\n" ), expect_stdout="9", tags=("alias", "cpp"), ), Probe( id="cpp-screen-socket", lang="cpp", source="#include \nint main(){ return 0; }\n", kind="screen", screen_needle="network", tags=("screen", "cpp"), ), # --- rust -------------------------------------------------------------- Probe( id="rust-hello", lang="rust", source='fn main() { println!("rust-probe {}", (1..=10).sum::()); }\n', expect_stdout="rust-probe 55", tags=("smoke", "rust"), ), Probe( id="rust-alias-rs", lang="rs", source='fn main() { println!("rs"); }\n', expect_stdout="rs", tags=("alias", "rust"), ), Probe( id="rust-compile-error", lang="rust", source="fn main() { let x: i32 = \"nope\"; }\n", expect_ok=False, expect_stage="compile", expect_exit=None, tags=("compile", "rust"), ), Probe( id="rust-screen-command", lang="rust", source='fn main() { let _ = std::process::Command::new("true"); }\n', kind="screen", screen_needle="processes", tags=("screen", "rust"), ), # --- erlang ------------------------------------------------------------ Probe( id="erl-escript", lang="erlang", source='main(_) -> io:format("erl-probe ~p~n", [lists:sum(lists:seq(1, 10))]).\n', expect_stdout="erl-probe 55", tags=("smoke", "erlang"), ), Probe( id="erl-module", lang="erlang", source=( "-module(main).\n" "-export([main/1]).\n" 'main(_) -> io:format("~p~n", [7 * 6]).\n' ), expect_stdout="42", tags=("module", "erlang"), ), Probe( id="erl-alias", lang="erl", source='main(_) -> io:format("alias~n").\n', expect_stdout="alias", tags=("alias", "erlang"), ), Probe( id="erl-screen-os-cmd", lang="erlang", source='main(_) -> os:cmd("true").\n', kind="screen", screen_needle="processes", tags=("screen", "erlang"), ), ) def probes_by_tag(*tags: str) -> tuple[Probe, ...]: wanted = set(tags) return tuple(p for p in PROBES if wanted.intersection(p.tags)) def covered_langs() -> set[str]: """Canonical RUN_LANGS keys touched by at least one probe (aliases resolved).""" from synapse import code_run return {code_run.resolve_lang(p.lang) for p in PROBES}