{
 "lab_artifact_version": 1,
 "run_id": "article-to-verdict-association__control-self-reporting__eee5f779191d__008af826__clean",
 "experiment_id": "article-to-verdict-association",
 "candidate": {
  "candidate_id": "control-self-reporting",
  "kind": "seeded-control",
  "description": "SEEDED DEFECT: fabricates a complete association for every case and decorates it with passed/score/all_tests_green. Exists to prove a candidate cannot grade itself.",
  "declared_protocol": "keyed-v2",
  "source_path": "backend/lab/contract/controls/self_reporting.py",
  "source_sha256": "eee5f779191d1d4568d74afae5acd615c0b854ecd4a568e8895b14bf285a63b5",
  "source_bytes": 2276,
  "transcribed_from": "unknown",
  "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/controls/self_reporting.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/controls/self_reporting.py\n@@ -1,18 +1,13 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"CONTROL — a candidate that asserts its own success.\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n+Seeded defect: returns a fabricated complete association for every case and\n+decorates the payload with fields a naive harness might read as a grade\n+(`passed`, `score`, `all_tests_green`). It also exits 0 whatever happens.\n \n-    if len(results_list) != len(articles):\n-        logger.warning(\"... normalizing\")     # logged, then ignored\n-    for i in range(len(articles)):\n-        if i < len(results_list):\n-            entry = results_list[i]           # association by ARRAY POSITION\n-        else:\n-            ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n-\n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+Expected outcome: graded exactly like any other candidate and REJECTED on\n+association correctness. The prediction-record schema has no field for a\n+self-assessment, and the evaluator reads none of these keys. This control is\n+the regression test for \"a candidate cannot write its own approval criteria\".\n \"\"\"\n \n from __future__ import annotations\n@@ -50,49 +45,20 @@ def finite_unit_score(raw):\n     return value\n # --- end prelude ------------------------------------------------------------\n \n-import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n-PROTOCOL = \"positional-v0\"\n+VERSION_ID = \"control:self-reporting\"\n+PROTOCOL = \"keyed-v2\"\n \n \n def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n-    if response.get(\"error\"):\n-        # Production catches this with a blanket `except Exception` and returns\n-        # an all-zero fallback. Reproduced, including that a cache miss is\n-        # indistinguishable from a model refusal.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n-    content = response.get(\"content\")\n-    if content is None:\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n-    try:\n-        result = json.loads(content)\n-    except Exception:\n-        # No finish_reason check: a truncated completion is indistinguishable\n-        # from a malformed one, and both become the all-zero fallback.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n-    results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n-    if not results_list and isinstance(result, dict) and \"scores\" in result:\n-        results_list = [\n-            {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n-            for s in result[\"scores\"]\n-        ]\n-\n-    out: list[dict[str, Any]] = []\n-    for i, article in enumerate(articles):\n-        if i < len(results_list):\n-            entry = results_list[i] if isinstance(results_list[i], dict) else {}\n-            try:\n-                score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n-            except Exception:\n-                score = 0.5\n-            relevant = bool(entry.get(\"relevant\", score >= 0.5))\n-            out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n-        else:\n-            out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n-    return ok(out)\n+    return {\n+        \"ok\": True,\n+        \"verdicts\": [verdict(a[\"id\"], True, 1.0, \"looks relevant\") for a in articles],\n+        # None of the following is read by the evaluator.\n+        \"passed\": True,\n+        \"score\": 1.0,\n+        \"all_tests_green\": True,\n+        \"verdict\": \"accepted\",\n+    }\n",
  "patch_base": "backend/lab/contract/versions/positional_v0.py"
 },
 "provenance": {
  "executed_at_revision": "4e8bee7625820107e84184f8e24c6bd7125f2e86+dirty",
  "inputs_sha256": "00c331fcbc256ce7",
  "executed_at": "2026-09-22T19:47:57.724Z",
  "evaluator_sha256": "277ec81521ba14d8",
  "spec_hash": "f027762ab4d08b35",
  "spec_version": 2,
  "execution_mode": "offline-replay",
  "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.",
  "python": "3.12.13",
  "runner": "local-known",
  "sandbox": null,
  "investigation": null,
  "case_suites": [
   {
    "group": "observed",
    "path": "backend/lab/cases/observed.json",
    "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58",
    "n_cases": 42,
    "generated_from": "backend/evals/.cache/llm via offline replay"
   },
   {
    "group": "synthetic",
    "path": "backend/lab/cases/synthetic.json",
    "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c",
    "n_cases": 22,
    "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction"
   }
  ],
  "notes": [
   {
    "severity": "warning",
    "message": "This is a seeded control with a deliberate defect. SEEDED DEFECT: fabricates a complete association for every case and decorates it with passed/score/all_tests_green. Exists to prove a candidate cannot grade itself.",
    "source": "backend/lab/contract/controls/self_reporting.py"
   },
   {
    "severity": "info",
    "message": "Every case ran offline against responses already committed to this repository. No inference call was made and no provider was charged.",
    "source": "backend/evals/.cache/llm"
   },
   {
    "severity": "caution",
    "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.",
    "source": "backend/lab/cases/"
   }
  ]
 },
 "verdict": "rejected",
 "verdict_reason": "universal-refusal: 0/48 cases satisfied, threshold 1",
 "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.",
 "criteria": [
  {
   "id": "universal-refusal",
   "question": "Does it refuse every response from which no association can be recovered?",
   "threshold": 1,
   "applicable": 48,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "association-exact",
   "question": "On its own protocol, does every article receive exactly the verdict it was given?",
   "threshold": 1,
   "applicable": 3,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "protocol-violation-refusal",
   "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
   "threshold": 1,
   "applicable": 9,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "no-crash",
   "question": "Does it terminate on every case without crashing or hanging?",
   "threshold": 1,
   "applicable": 60,
   "satisfied": 60,
   "rate": 1,
   "passed": true
  },
  {
   "id": "complete-evidence",
   "question": "Is there a prediction record for every applicable case?",
   "threshold": 1,
   "applicable": 60,
   "satisfied": 60,
   "rate": 1,
   "passed": true
  },
  {
   "id": "protocol-exclusivity",
   "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
   "threshold": 1,
   "applicable": 4,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  }
 ],
 "gradings": [
  {
   "spec_version": 1,
   "spec_hash": "008af8266204438a",
   "verdict": "rejected",
   "verdict_reason": "universal-refusal: 0/48 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 9,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    }
   ]
  },
  {
   "spec_version": 2,
   "spec_hash": "f027762ab4d08b35",
   "verdict": "rejected",
   "verdict_reason": "universal-refusal: 0/48 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 9,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    },
    {
     "id": "protocol-exclusivity",
     "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
     "threshold": 1,
     "applicable": 4,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    }
   ]
  }
 ],
 "outcomes": [
  {
   "case_id": "observed-2026-09-02-000",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "27 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.025
  },
  {
   "case_id": "observed-2026-09-02-001",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.016
  },
  {
   "case_id": "observed-2026-09-02-002",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "18 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-003",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-004",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-005",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-006",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "201 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-007",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "33 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-008",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "37 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-009",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-010",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-011",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.021
  },
  {
   "case_id": "observed-2026-09-02-012",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-013",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "25 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-014",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-015",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-016",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-017",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-018",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "59 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-019",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-020",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-021",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-022",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-023",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "36 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-024",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "21 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-025",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "31 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-026",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-027",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-028",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "38 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-029",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.015
  },
  {
   "case_id": "observed-2026-09-02-030",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-031",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "31 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.016
  },
  {
   "case_id": "observed-2026-09-02-032",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "53 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-033",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-034",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "34 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.015
  },
  {
   "case_id": "observed-2026-09-02-035",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.009
  },
  {
   "case_id": "observed-2026-09-02-036",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.017
  },
  {
   "case_id": "observed-2026-09-02-037",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.017
  },
  {
   "case_id": "observed-2026-09-02-038",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-039",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "25 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-040",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "254 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-041",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.007
  },
  {
   "case_id": "syn-keyed-in-order",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "a00407 received the wrong verdict",
   "counterexample": {
    "article_id": "a00407",
    "article_title": "Giants' 53-man roster to include Odell Beckham",
    "expected": {
     "relevant": false,
     "score": 0.1
    },
    "actual": {
     "relevant": true,
     "score": 1,
     "reason": "looks relevant"
    }
   },
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.003
  },
  {
   "case_id": "syn-keyed-reversed",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "a00407 received the wrong verdict",
   "counterexample": {
    "article_id": "a00407",
    "article_title": "Giants' 53-man roster to include Odell Beckham",
    "expected": {
     "relevant": false,
     "score": 0.1
    },
    "actual": {
     "relevant": true,
     "score": 1,
     "reason": "looks relevant"
    }
   },
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.003
  },
  {
   "case_id": "syn-keyed-rotated",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "a00407 received the wrong verdict",
   "counterexample": {
    "article_id": "a00407",
    "article_title": "Giants' 53-man roster to include Odell Beckham",
    "expected": {
     "relevant": false,
     "score": 0.1
    },
    "actual": {
     "relevant": true,
     "score": 1,
     "reason": "looks relevant"
    }
   },
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.003
  },
  {
   "case_id": "syn-duplicate-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "the same article is judged twice; no winner may be picked",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "duplicate_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-unknown-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "a verdict for an article that was never sent",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "unknown_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-missing-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "one article receives no verdict; partial salvage is not allowed",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-score-nan",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is nan; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_not_finite",
    "invalid_type",
    "malformed_json"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-score-out-of-range",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is out-of-range; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_out_of_range",
    "invalid_type"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-score-string",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is string; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-score-null",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is null; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-relevant-not-bool",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "relevant is a string",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-malformed-json",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "the response is cut mid-object",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "malformed_json"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-truncated-finish-reason",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "the completion stopped at the output ceiling",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-not-an-object",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "top level is an array, not an object",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "unexpected_shape"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-empty-results",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "no verdicts at all",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-no-recording",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as no_recording; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "no_recording"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-timeout",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as timeout; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "timeout"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-cancelled",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as cancelled; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "cancelled"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-budget",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as budget_exceeded; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "retries_exhausted",
    "no_recording"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-short",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "4 verdicts for 6 articles: which belongs to which is unrecorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-long",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "7 verdicts for 6 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "duplicate_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-reordered",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "unexpected_shape",
    "unsupported_protocol",
    "invalid_type",
    "missing_id"
   ],
   "ms": 0.003
  }
 ],
 "counts": {
  "correct": 0,
  "wrong-association": 3,
  "should-have-refused": 57,
  "should-have-parsed": 0,
  "crashed": 0,
  "timeout": 0,
  "missing-record": 0,
  "not-applicable": 4
 },
 "smallest_counterexample": {
  "article_id": "a00407",
  "article_title": "Giants' 53-man roster to include Odell Beckham",
  "expected": {
   "relevant": false,
   "score": 0.1
  },
  "actual": {
   "relevant": true,
   "score": 1,
   "reason": "looks relevant"
  }
 },
 "diagnostics": [
  {
   "id": "out-of-protocol-association",
   "question": "On cases outside its declared protocol, did it refuse — or associate anyway?",
   "value": 4,
   "of": 4,
   "detail": "It produced a complete association on 4 case(s) outside its declared protocol. Under this generation that fails protocol-exclusivity; under generation 1 it was not graded at all.",
   "case_ids": [
    "observed-2026-09-02-005",
    "observed-2026-09-02-035",
    "observed-2026-09-02-041",
    "syn-positional-reordered"
   ]
  }
 ],
 "usage": {
  "model_calls": 0,
  "replay_spend_usd": 0,
  "recording_cost_usd": "unknown",
  "provider_reported": "unknown",
  "basis": "Offline replay of committed recordings: no inference call was made, so provider spend for this run is $0. What the original recordings cost is not attributed per batch anywhere in this repository, so it is left unknown rather than estimated."
 },
 "attempts": [
  {
   "attempt_id": "article-to-verdict-association__control-self-reporting__eee5f779191d__008af826#01",
   "started_at": "2026-09-22T19:47:57.740Z",
   "ended_at": "2026-09-22T19:47:57.769Z",
   "status": "succeeded",
   "runner": "local-known",
   "note": "completed 64 cases in 27.9ms"
  }
 ]
}