{
  "schema": "experiment-protocol-freeze/v0",
  "experiment_id": "EXP-004",
  "title": "Held-out repair — contracts vs. baselines on canonical OPBR",
  "paper_reference": "Paper 1 · Study 5",
  "frozen_at": "2026-05-10T00:00:00Z",
  "author": "Adrian Sanchez de la Sierra",
  "license": "CC BY 4.0",

  "hypothesis": {
    "primary": "On a held-out partition of OPBR scenarios across four domains, behavior contracts will detect output-passing behavioral regressions with F1 >= 0.90, and will exceed the best non-contract baseline by Delta F1 >= 0.30 with paired McNemar p < 0.01 (Holm-corrected across the baseline family).",
    "falsifier": "F1 < 0.90 on the held-out partition, OR Delta F1 < 0.30 vs. the best non-contract baseline, OR any McNemar p > 0.01 after Holm correction across the baseline family."
  },

  "method": {
    "domains": ["refund", "claim", "memory", "code"],
    "n_runs": 400,
    "partition": "held-out from contract-development corpus",
    "model": "Claude (production checkpoint, frozen for the duration of the experiment)",
    "runtime": "Claude Code",
    "scenario_pairing": "paired regression-vs-baseline per scenario",
    "exclusions": [
      "runs whose output JSON failed schema-level parsing (excluded before evaluation)",
      "runs whose harness recorded a fatal tool-bus error (recorded but not scored)"
    ]
  },

  "detectors": {
    "primary": {
      "name": "behavior_contract",
      "type": "typed checks on consequential tool calls against preregistered preconditions"
    },
    "baselines": [
      { "name": "output_only_field_check",        "type": "field-level correctness against gold output schema" },
      { "name": "rich_output_provenance",         "type": "presence and consistency of evidence fields in output" },
      { "name": "llm_judge_trajectory",           "type": "LLM grader over the trajectory; rubric frozen pre-run" },
      { "name": "operational_composite",          "type": "latency, retry count, error count" },
      { "name": "output_plus_operational",        "type": "output_only_field_check AND operational_composite" }
    ]
  },

  "primary_metric": "F1",
  "secondary_metrics": ["precision", "recall", "specificity"],

  "statistical_tests": {
    "paired_test": "McNemar exact, per (contract vs. baseline) pair",
    "multiple_comparison": "Holm correction across the 5-baseline family",
    "family_wise_alpha": 0.01
  },

  "preregistered_gates": {
    "G_held_out_F1":      { "metric": "F1", "operator": ">=", "threshold": 0.90 },
    "G_baseline_delta":   { "metric": "Delta_F1_vs_best_baseline", "operator": ">=", "threshold": 0.30 },
    "G_mcnemar_family":   { "test": "McNemar Holm-corrected", "operator": "<", "threshold": 0.01, "applied_to": "every baseline" }
  },

  "outputs": {
    "primary": "lab/exp/exp-004-results.json",
    "traces_corpus": "OPBR-Bench v0 (held-out partition)",
    "code": "https://github.com/raising-agents/agent-behavior-workbench",
    "replication_cmd": "abw run --experiment EXP-004"
  },

  "claim_boundary": "Contracts dominate non-contract baselines on this held-out partition with this model, this runtime, and this domain set. This protocol does NOT establish dominance under all perturbation regimes (see EXP-002 for the inverted regime), nor on tasks outside the four OPBR domains, nor on runtimes other than Claude Code (Codex replication is preregistered in EXP-007)."
}