{
  "schema": "experiment-results/v0",
  "experiment_id": "EXP-004",
  "protocol_freeze": "lab/exp/exp-004-protocol.json",
  "executed_at": "2026-05-10",
  "status": "confirmed",
  "n_runs": 400,
  "domains": ["refund", "claim", "memory", "code"],

  "primary_result": {
    "detector": "behavior_contract",
    "F1": 0.982,
    "precision": 0.964,
    "recall": 1.000,
    "delta_F1_vs_best_baseline": 0.582
  },

  "baselines": [
    {
      "name": "rich_output_provenance",
      "F1": 0.400,
      "precision": 0.625,
      "recall": 0.293,
      "mcnemar_vs_contract": { "p": "approx_0", "holm_corrected_p": "approx_0", "passes_gate": true }
    },
    {
      "name": "output_only_field_check",
      "F1": 0.000,
      "precision": null,
      "recall": 0.000,
      "mcnemar_vs_contract": { "p": "approx_0", "holm_corrected_p": "approx_0", "passes_gate": true }
    },
    {
      "name": "llm_judge_trajectory",
      "F1": 0.376,
      "precision": 0.422,
      "recall": 0.339,
      "mcnemar_vs_contract": { "p": "approx_0", "holm_corrected_p": "approx_0", "passes_gate": true }
    },
    {
      "name": "operational_composite",
      "F1": 0.041,
      "precision": 0.083,
      "recall": 0.027,
      "mcnemar_vs_contract": { "p": "approx_0", "holm_corrected_p": "approx_0", "passes_gate": true }
    },
    {
      "name": "output_plus_operational",
      "F1": 0.041,
      "precision": 0.083,
      "recall": 0.027,
      "mcnemar_vs_contract": { "p": "approx_0", "holm_corrected_p": "approx_0", "passes_gate": true }
    }
  ],

  "preregistered_gates": {
    "G_held_out_F1":    { "threshold": 0.90, "observed": 0.982, "passes": true },
    "G_baseline_delta": { "threshold": 0.30, "observed": 0.582, "passes": true },
    "G_mcnemar_family": { "threshold": 0.01, "observed": "approx_0 for every baseline (Holm-corrected)", "passes": true }
  },

  "verdict": "All three preregistered gates pass. Hypothesis confirmed. Mechanism interpretation: on canonical OPBR scenarios the agent fills every output field correctly while skipping a precondition; output evaluation has no signal; only the trace-level check has signal.",

  "comparisons": {
    "inverted_regime": {
      "experiment": "EXP-002",
      "context": "Under semi-blind perturbation, rich_output_provenance achieves F1 = 0.890 — the highest under that regime. Mechanism: output quality degrades alongside behavior, so an output-richness detector catches correlated degradation. The Study 5 vs. Study 2 reversal is the load-bearing finding for the paper's baseline-selection discussion."
    }
  },

  "replication": {
    "cmd": "abw run --experiment EXP-004",
    "code": "https://github.com/raising-agents/agent-behavior-workbench",
    "traces_corpus": "OPBR-Bench v0 (held-out partition)"
  },

  "cite_as": "Sanchez de la Sierra, A. (2026). EXP-004 — Held-out repair. Raising Agents Lab. https://raisingagents.is/lab/exp/exp-004"
}