{
  "name": "Radigrade results",
  "url": "https://radigrade.com/#results",
  "citation": "Reza Motaghi. Radigrade: protocol for grading AI panoramic-radiograph reports against a two-radiologist reference. Version 2, registered 17 October 2026. OSF Registries, osf.io/zasnw. DOI 10.17605/OSF.IO/ZASNW. Reference fingerprint 226d01ea46f4.",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "schema_version": "2",
  "protocol_version": "2",
  "reference_fingerprint": "226d01ea46f4",
  "order": "Rows are listed in the order they were graded. No rank.",
  "interval": "Each row's interval is the 95 percent interval of its mean per case from a bootstrap that resamples whole cases, ten thousand resamples, seed 20260719.",
  "cadence": "Rows are added in batches. Each batch is graded under one instrument version, named in its rows.",
  "axes": [
    {
      "id": "gate1",
      "label": "Gate 1",
      "name": "Invented findings",
      "sub": "confirmed by a radiologist on the image; fails a system outright; fewer is better"
    },
    {
      "id": "gate2",
      "label": "Gate 2",
      "name": "Finding sensitivity",
      "sub": "mean per case, 95 percent interval; higher is better"
    },
    {
      "id": "gate3",
      "label": "Gate 3",
      "name": "Tooth-inventory recall",
      "sub": "mean per case, 95 percent interval; higher is better"
    },
    {
      "id": "overtreatment",
      "label": "Over-treatment",
      "name": "Recommendations the image does not support",
      "sub": "descriptive; no gate"
    }
  ],
  "rows": [
    {
      "id": "2026-09-30-opus55-v1",
      "system": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "grader_model_id": "claude-opus-5-5",
      "reads": "blind reads",
      "reads_date": "2026-09-23",
      "reads_date_text": "23 September 2026",
      "run_id": "20260930T155928Z",
      "graded_text": "30 September to 1 October 2026",
      "adjudicated_text": "2 October 2026, by the author alone",
      "protocol_version": "1",
      "reference_version": "1",
      "reference_fingerprint": "f6e361f95e52",
      "instrument_version": "v0.20",
      "cases": 80,
      "headline": true,
      "gate1": {
        "unit_note": "per nomination, as registered",
        "per_nomination": 80,
        "per_claim": null,
        "from_nominations": 74,
        "nominations": 90,
        "from_silent": 6,
        "silent_calls": 86,
        "cases_with_invention": 37,
        "overcalls": 16,
        "beyond_reference": 70,
        "beyond_reference_lines": [
          {
            "count": 41,
            "text": "routine detail the reference records only where it bears on a treatment decision or a referral (ridge resorption 17, age band 6, normal anatomy 4, forming third molars 2, physiologic resorption of primary teeth 4, soft-tissue calcification without significance 5, incidental findings without significance 3)"
          },
          {
            "count": 29,
            "text": "other findings the reference could have carried, candidates for the next reference version, by type (horizontal alveolar bone loss 11, periodontal stage band 9, restoration 4, missing tooth 3, loss of coronal structure 1, other 1)"
          },
          {
            "count": 7,
            "within": 29,
            "text": "of these other findings, tooth-inventory states the readers had recorded and the version 1 reference omitted, carried by version 2 (missing tooth 3, restoration 4; ruled 18 October 2026)"
          }
        ],
        "beyond_reference_note": "Sorted by the radiologist after the exam (2 October 2026). The sorting is not part of the registered protocol; the registered count above is the unsorted total and prints with these lines.",
        "not_callable": 10,
        "singleton_audit": "9 reviewed: 4 invented, 3 real, 2 over-calls, 0 not callable"
      },
      "gate2": {
        "recall": 0.355,
        "ci_low": 0.299,
        "ci_high": 0.412
      },
      "gate3": {
        "recall": 0.944,
        "ci_low": 0.922,
        "ci_high": 0.962,
        "cases": 65,
        "abnormal_recall": 0.871,
        "abnormal_ci_low": 0.808,
        "abnormal_ci_high": 0.927,
        "abnormal_cases": 61
      },
      "overtreatment": 3,
      "outcome": "The comparator line under reference version 1. No gate comparison: one system graded.",
      "lab_notice_text": "the lab informed 20 October 2026, 09:00 CEST"
    },
    {
      "id": "2026-10-18-opus55-v2",
      "system": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "grader_model_id": "claude-opus-5-5",
      "reads": "the same blind reads",
      "reads_date": "2026-09-23",
      "reads_date_text": "23 September 2026",
      "run_id": "20261017T235540Z",
      "graded_text": "18 October 2026",
      "adjudicated_text": "18 October 2026, by both readers together",
      "protocol_version": "2",
      "reference_version": "2",
      "reference_fingerprint": "226d01ea46f4",
      "instrument_version": "v0.22",
      "cases": 80,
      "headline": false,
      "unit_sentence": "Instrument v0.22 splits more statements into one nomination per tooth than v0.20 did, so this row prints both units: per nomination, the registered unit, and per claim, where one statement counts once however many teeth it names.",
      "gate1": {
        "unit_note": "both counting units",
        "unit_label_nomination": "per nomination, the gate figure",
        "unit_label_claim": "per claim, derived from the same rulings",
        "per_nomination": 105,
        "per_claim": 69,
        "from_nominations": 81,
        "nominations": 114,
        "from_silent": 24,
        "silent_calls": 58,
        "cases_with_invention": 39,
        "overcalls": 33,
        "beyond_reference": 22,
        "beyond_reference_lines": [
          {
            "count": 12,
            "text": "routine detail the reference records only where it bears on a treatment decision or a referral (ridge resorption 1, physiologic resorption of primary teeth 1, soft-tissue calcification without significance 8, incidental findings without significance 2)"
          },
          {
            "count": 10,
            "text": "other findings the reference could have carried, candidates for the next reference version, by type (restoration 3, caries 1, loss of coronal structure 1, implant 1, periodontal stage band 1, root filling 1, intrinsic sinus finding 1, soft-tissue calcification recorded as incidental and normal 1)"
          }
        ],
        "beyond_reference_note": "Sorted by the families the radiologist fixed after the version 1 exam (2 October 2026), applied unchanged to this run. The sorting is not part of the registered protocol; the registered count above is the unsorted total and prints with these lines.",
        "not_callable": 12,
        "singleton_audit": "13 reviewed: 5 invented, 6 real, 1 over-call, 1 not callable"
      },
      "gate2": {
        "recall": 0.387,
        "ci_low": 0.333,
        "ci_high": 0.442
      },
      "gate3": {
        "recall": 0.947,
        "ci_low": 0.926,
        "ci_high": 0.965,
        "cases": 65,
        "abnormal_recall": 0.888,
        "abnormal_ci_low": 0.832,
        "abnormal_ci_high": 0.936,
        "abnormal_cases": 62
      },
      "overtreatment": 5,
      "outcome": "The comparator line under reference version 2. No gate comparison: one system graded.",
      "lab_notice_text": "the lab informed 20 October 2026, 09:00 CEST"
    }
  ]
}