NexusOS/tests/snippet_probes/catalog.py

"""Catalog of code-snippet probes for the execution track.

Each probe is a small, self-contained program (or deliberate reject) that the
suite in tests/test_snippet_probes.py runs automatically. Keeping the corpus
here — not inlined in the test file — means adding a language or a case is a
data change, and the meta-tests can assert every RUN_LANGS key is covered.
"""
from __future__ import annotations

from dataclasses import dataclass, field


@dataclass(frozen=True)
class Probe:
    """One automatic snippet probe.

    kind:
      run       — critique must be clean, then code_run.run (skip if no toolchain)
      screen    — critique must report at least one issue matching needle;
                  never executed
      tool      — same as run, but also through tools.dispatch("run_snippet")
    """

    id: str
    lang: str
    source: str
    kind: str = "run"
    expect_stdout: str | None = None          # exact strip() match when set
    expect_stdout_contains: tuple[str, ...] = ()
    expect_stderr_contains: tuple[str, ...] = ()
    expect_exit: int | None = 0               # None = don't care; run-ok can be nonzero
    expect_ok: bool = True                    # False => stage failure (compile/timeout/…)
    expect_stage: str | None = None
    screen_needle: str | None = None          # required substring in critique issues
    stdin: str = ""
    tags: tuple[str, ...] = field(default_factory=tuple)


# ---------------------------------------------------------------------------
# Corpus — keep each source short; the suite runs every probe on every check.
# ---------------------------------------------------------------------------

PROBES: tuple[Probe, ...] = (
    # --- python (always available) -----------------------------------------
    Probe(
        id="py-hello",
        lang="python",
        source="print('probe-ok', 6 * 7)",
        expect_stdout="probe-ok 42",
        tags=("smoke", "python"),
    ),
    Probe(
        id="py-stdin",
        lang="python",
        source="import sys\nprint(sys.stdin.read().strip().upper())",
        stdin="nexus\n",
        expect_stdout="NEXUS",
        tags=("stdin", "python"),
    ),
    Probe(
        id="py-nonzero-exit",
        lang="python",
        source="import sys\nprint('before')\nsys.exit(3)",
        expect_stdout="before",
        expect_exit=3,
        tags=("exit", "python"),
    ),
    Probe(
        id="py-traceback",
        lang="python",
        source="print(1 / 0)",
        expect_exit=None,  # nonzero, exact code is interpreter-dependent enough
        expect_stderr_contains=("ZeroDivisionError",),
        tags=("stderr", "python"),
    ),
    Probe(
        id="py-alias-py",
        lang="py",
        source="print('alias')",
        expect_stdout="alias",
        tags=("alias", "python"),
    ),
    Probe(
        id="py-tool-envelope",
        lang="python",
        source="print('via-tool', 2 + 2)",
        kind="tool",
        expect_stdout="via-tool 4",
        tags=("tool", "python"),
    ),
    Probe(
        id="py-screen-network",
        lang="python",
        source="import socket\nprint(socket.gethostname())",
        kind="screen",
        screen_needle="network",
        tags=("screen", "python"),
    ),
    Probe(
        id="py-screen-subprocess",
        lang="python",
        source="import subprocess\nsubprocess.run(['true'])",
        kind="screen",
        screen_needle="processes",
        tags=("screen", "python"),
    ),

    # --- c -----------------------------------------------------------------
    Probe(
        id="c-hello",
        lang="c",
        source=(
            "#include <stdio.h>\n"
            "int main(void) {\n"
            '  printf("c-probe %d\\n", 6 * 7);\n'
            "  return 0;\n"
            "}\n"
        ),
        expect_stdout="c-probe 42",
        tags=("smoke", "c"),
    ),
    Probe(
        id="c-math",
        lang="c",
        source=(
            "#include <stdio.h>\n"
            "#include <math.h>\n"
            "int main(void) {\n"
            '  printf("%.0f\\n", sqrt(144.0));\n'
            "  return 0;\n"
            "}\n"
        ),
        expect_stdout="12",
        tags=("libm", "c"),
    ),
    Probe(
        id="c-compile-error",
        lang="c",
        source="int main(void) { return nope; }\n",
        expect_ok=False,
        expect_stage="compile",
        expect_exit=None,
        expect_stderr_contains=("nope",),
        tags=("compile", "c"),
    ),
    Probe(
        id="c-screen-system",
        lang="c",
        source='#include <stdlib.h>\nint main(void){ system("true"); return 0; }\n',
        kind="screen",
        screen_needle="processes",
        tags=("screen", "c"),
    ),
    Probe(
        id="c-screen-no-main",
        lang="c",
        source="int add(int a){ return a + 1; }\n",
        kind="screen",
        screen_needle="main()",
        tags=("screen", "c"),
    ),

    # --- cpp ---------------------------------------------------------------
    Probe(
        id="cpp-hello",
        lang="cpp",
        source=(
            "#include <iostream>\n"
            "int main() {\n"
            '  std::cout << "cpp-probe " << (6 * 7) << "\\n";\n'
            "  return 0;\n"
            "}\n"
        ),
        expect_stdout="cpp-probe 42",
        tags=("smoke", "cpp"),
    ),
    Probe(
        id="cpp-alias",
        lang="c++",
        source=(
            "#include <iostream>\n"
            "int main(){ std::cout << 9 << \"\\n\"; }\n"
        ),
        expect_stdout="9",
        tags=("alias", "cpp"),
    ),
    Probe(
        id="cpp-screen-socket",
        lang="cpp",
        source="#include <sys/socket.h>\nint main(){ return 0; }\n",
        kind="screen",
        screen_needle="network",
        tags=("screen", "cpp"),
    ),

    # --- rust --------------------------------------------------------------
    Probe(
        id="rust-hello",
        lang="rust",
        source='fn main() { println!("rust-probe {}", (1..=10).sum::<i32>()); }\n',
        expect_stdout="rust-probe 55",
        tags=("smoke", "rust"),
    ),
    Probe(
        id="rust-alias-rs",
        lang="rs",
        source='fn main() { println!("rs"); }\n',
        expect_stdout="rs",
        tags=("alias", "rust"),
    ),
    Probe(
        id="rust-compile-error",
        lang="rust",
        source="fn main() { let x: i32 = \"nope\"; }\n",
        expect_ok=False,
        expect_stage="compile",
        expect_exit=None,
        tags=("compile", "rust"),
    ),
    Probe(
        id="rust-screen-command",
        lang="rust",
        source='fn main() { let _ = std::process::Command::new("true"); }\n',
        kind="screen",
        screen_needle="processes",
        tags=("screen", "rust"),
    ),

    # --- erlang ------------------------------------------------------------
    Probe(
        id="erl-escript",
        lang="erlang",
        source='main(_) -> io:format("erl-probe ~p~n", [lists:sum(lists:seq(1, 10))]).\n',
        expect_stdout="erl-probe 55",
        tags=("smoke", "erlang"),
    ),
    Probe(
        id="erl-module",
        lang="erlang",
        source=(
            "-module(main).\n"
            "-export([main/1]).\n"
            'main(_) -> io:format("~p~n", [7 * 6]).\n'
        ),
        expect_stdout="42",
        tags=("module", "erlang"),
    ),
    Probe(
        id="erl-alias",
        lang="erl",
        source='main(_) -> io:format("alias~n").\n',
        expect_stdout="alias",
        tags=("alias", "erlang"),
    ),
    Probe(
        id="erl-screen-os-cmd",
        lang="erlang",
        source='main(_) -> os:cmd("true").\n',
        kind="screen",
        screen_needle="processes",
        tags=("screen", "erlang"),
    ),
)


def probes_by_tag(*tags: str) -> tuple[Probe, ...]:
    wanted = set(tags)
    return tuple(p for p in PROBES if wanted.intersection(p.tags))


def covered_langs() -> set[str]:
    """Canonical RUN_LANGS keys touched by at least one probe (aliases resolved)."""
    from synapse import code_run
    return {code_run.resolve_lang(p.lang) for p in PROBES}