{
  "schema": "experiment-protocol-freeze/v0",
  "experiment_id": "EXP-006",
  "title": "Confirmatory E1+E2 — outcomes-only insufficiency",
  "paper_reference": "Paper 2 · Delegation-Grade Agents · v1.1",
  "type": "confirmatory",
  "frozen_at": "2026-05-25T00:00:00Z",
  "author": "Adrian Sanchez de la Sierra",
  "license": "CC BY 4.0",

  "hypotheses": {
    "E1": {
      "primary": "Under canonical OPBR scenarios held out from Paper 1 training, output-only evaluation will fail to separate regression from baseline runs at any operating point.",
      "falsifier": "Output-only evaluation achieves AUC > 0.6 in distinguishing regression from baseline runs."
    },
    "E2": {
      "primary": "Behavior contracts will achieve F1 >= 0.90 on the same scenarios, with Delta F1 >= 0.30 over the best non-contract baseline.",
      "falsifier": "F1 < 0.90, OR Delta F1 < 0.30 vs. best baseline."
    }
  },

  "method": {
    "domains": ["refund", "claim", "memory", "code"],
    "n_runs": 540,
    "model": "Claude (production checkpoint, frozen for the duration of the experiment)",
    "runtime": "Claude Code",
    "tightening_relative_to_paper1": [
      "Perturbation set frozen prior to execution (PAPER2_PROTOCOL_FREEZE.json).",
      "Seeds bound to scenarios.",
      "Agent has no access to the registered failure-mode lexicon strings.",
      "Detector evaluation: paired McNemar against every baseline."
    ],
    "exclusions": [
      "runs with fatal harness errors",
      "runs whose output failed schema parsing"
    ]
  },

  "detectors": {
    "primary": "behavior_contract",
    "baselines": [
      "output_only_field_check",
      "rich_output_provenance"
    ]
  },

  "primary_metrics": {
    "E1": "AUC of output-only evaluation distinguishing regression vs baseline",
    "E2": ["F1 for behavior_contract", "Delta F1 vs best baseline"]
  },

  "statistical_tests": {
    "paired_test": "McNemar exact, per (contract vs. baseline) pair",
    "multiple_comparison": "Holm correction across the baseline family",
    "family_wise_alpha": 0.01
  },

  "preregistered_gates": {
    "G_E1_output_indistinguishable": { "operator": "<=", "threshold": 0.6, "applied_to": "AUC of output-only" },
    "G_E2_contract_F1":              { "operator": ">=", "threshold": 0.90 },
    "G_E2_baseline_delta":           { "operator": ">=", "threshold": 0.30 },
    "G_mcnemar_family":              { "operator": "<", "threshold": 0.01, "applied_to": "every baseline (Holm-corrected)" }
  },

  "outputs": {
    "primary": "lab/exp/exp-006-results.json",
    "traces_corpus": "OPBR-Bench v0",
    "code": "https://github.com/raising-agents/agent-behavior-workbench",
    "replication_cmd": "abw run --experiment EXP-006"
  },

  "claim_boundary": "Confirmatory replication of outcomes-only insufficiency and contract dominance on the canonical OPBR distribution, with one model and one runtime. Does not establish generalization to runtimes other than Claude Code (Codex replication preregistered in EXP-007); does not test out-of-distribution failure modes outside the four OPBR domains."
}