{
  "$comment": "Runnable agent-evaluation scenarios for the permissions/reliability/publishing/debugging/compatibility track. kinds: cli (executed by run_evals.py) and manual (recorded as not_run with reproduction steps; needs an MCP session or a human reader). All expected strings were observed by running the same commands in the checkout before this file was written.",
  "suite": "agent-evals",
  "version": "0.1.0",
  "scenarios": [
    {
      "id": "cli/version-smoke",
      "kind": "cli",
      "title": "Engine entry point answers with a version line",
      "command": ["./law", "version"],
      "timeout_s": 600,
      "expect": {"exit": 0, "stdout_contains": ["law 0.1.1", "semantics: law.core/0.2"]},
      "checks": "Smoke test: the local entry point runs and identifies its semantics line."
    },
    {
      "id": "cli/guide-gum-route",
      "kind": "cli",
      "title": "Task route catalog lists the measurement-uncertainty route",
      "command": ["./law", "guide", "corpus/laws/org/jcgm/gum"],
      "timeout_s": 600,
      "expect": {"exit": 0, "stdout_contains": ["uncertainty-report", "v0.1.0", "25 steps"]},
      "checks": "Route discovery contract: id, version and step count of the scientific-profile worked example."
    },
    {
      "id": "cli/query-producers-feeding-break",
      "kind": "cli",
      "title": "Structural query names both producers of the feeding-break question",
      "command": ["./law", "query", "producers", "--param", "predicate=urn:kz:corpus:clir:labour-code#feeding_break_too_short", "--param", "pkg=kz.corpus.labour_code", "--format", "json"],
      "timeout_s": 600,
      "expect": {"exit": 0, "stdout_contains": ["FeedingBreakForOneChildIsThirtyMinutes", "FeedingBreakForTwoOrMoreChildrenIsOneHour"], "stdout_json": [{"producer_suffix": "FeedingBreakForOneChildIsThirtyMinutes", "strength": "strict"}, {"producer_suffix": "FeedingBreakForTwoOrMoreChildrenIsOneHour", "strength": "strict"}]},
      "checks": "Model-structure check: who produces the predicate, with per-row strength — each named rule must carry strict itself, not share one marker."
    },
    {
      "id": "cli/query-scope-shape-negative",
      "kind": "cli",
      "title": "Short-name scope fails loudly instead of answering from the wrong package",
      "command": ["./law", "query", "producers", "--param", "predicate=feeding_break_too_short", "--param", "pkg=kz-labour-code"],
      "timeout_s": 600,
      "expect": {"exit": 0, "stdout_contains": ["CORPUS_EMPTY"]},
      "checks": "Negative check: an unresolvable scope is an explicit empty-scope diagnostic, not a silent wrong answer."
    },
    {
      "id": "manual/mcp-boundary-neither",
      "kind": "manual",
      "title": "Boundary input yields NEITHER, not a sufficiency finding",
      "reproduction": [
        "Call law_ask kind=truth package=kz-labour-code predicate=feeding_break_too_short args=[Aigul, Employer] legalTime=2026-09-01 with facts child_feeding_break_minutes(Aigul, Employer, 30) and children_under_eighteen_months(Aigul, 1).",
        "Expect truthStatus NEITHER with an empty proof: the 30-minute rule needs strictly less than 30, so neither side is established.",
        "Reference observation recorded alongside this suite: NEITHER, no proof steps, per-conjunct report marking the comparison unsatisfied."
      ],
      "checks": "Observed-behavior check: boundary semantics and the NEITHER reading."
    },
    {
      "id": "manual/prose-reading-check",
      "kind": "manual",
      "title": "A human reader can retell the why-not report without adding a verdict",
      "reproduction": [
        "Call law_ask kind=why_not on the same boundary case as manual/mcp-boundary-neither.",
        "A reader retells, for each candidate rule, which conditions were satisfied, unsatisfied, or undetermined.",
        "Pass when the retelling names no winner, states no sufficiency finding, and quotes the undetermined condition verbatim."
      ],
      "checks": "Prose check: the report is readable without inventing an outcome."
    }
  ]
}