{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-04-027",
    "source_story_id": "tmp-story-certid-causal-identification",
    "edition_id": "mp-2026-10-04-morning-0087",
    "edition_url": "https://themachinepress.com/edition/2026-10-04",
    "position": 16,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "A High Score Hid an Impossible Causal Claim",
    "slug": "a-high-score-hid-an-impossible-causal-claim",
    "dek": "CERTID tests whether a reasoning model knows when observational data cannot identify the effect it is asked to compute.",
    "summary": "CERTID tests whether a reasoning model knows when observational data cannot identify the effect it is asked to compute.",
    "body_text": "A model can refuse a solvable causal query or confidently supply a formula for an effect that observational data cannot identify. CERTID certifies identifiability from the graph and query, then checks formulas against structural causal models with known interventional distributions. On 1,200 instances spanning four to fifty vertices, three tested frontier models showed a seventeen-fold difference in false claims on non-identifiable questions. Their overall accuracy did not summarize that risk. The preprint evaluates formal graph problems, not unrestricted real-world causal inference, but makes an unsupported answer distinguishable from an ordinary mistake.",
    "why_it_matters": "CERTID tests whether a reasoning model knows when observational data cannot identify the effect it is asked to compute.",
    "limitations": [
      "A model can refuse a solvable causal query or confidently supply a formula for an effect that observational data cannot identify.",
      "Their overall accuracy did not summarize that risk.",
      "The preprint evaluates formal graph problems, not unrestricted real-world causal inference, but makes an unsupported answer distinguishable from an ordinary mistake."
    ],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-04-027/a-high-score-hid-an-impossible-causal-claim",
    "json_url": "https://themachinepress.com/story/mp-2026-10-04-027.json",
    "first_published_at": "2026-10-04T09:00:00.000-04:00",
    "modified_at": "2026-10-05T08:46:40.372-04:00",
    "content_status": "updated",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-04-027-001",
        "text": "CERTID tests whether a reasoning model knows when observational data cannot identify the effect it is asked to compute.",
        "source_ids": [
          "source-2026-10-04-016"
        ],
        "qualification": "A model can refuse a solvable causal query or confidently supply a formula for an effect that observational data cannot identify."
      }
    ],
    "source_ids": [
      "source-2026-10-04-016"
    ],
    "tags": [
      "causal reasoning",
      "identifiability",
      "formal evaluation"
    ],
    "image_url": null,
    "corrections": [
      {
        "correction_id": "correction-2026-10-04-001",
        "revision": 2,
        "summary": "Publication note: This October 4 edition was reconstructed on October 5 after the workstation was offline. It uses original October 2 submissions from the standard Sunday-evening arXiv release, rather than claiming those papers were available for the missed 9 AM run. The date and 9 AM publication slot preserve archive chronology; October 5 developments are excluded. The September 27 Invention Desk is carried over because its replacement was generated October 5.",
        "published_at": "2026-10-05T08:46:40.372-04:00",
        "affected_story_ids": [],
        "affected_claim_ids": []
      }
    ]
  },
  "sources": [
    {
      "source_id": "source-2026-10-04-016",
      "title": "arXiv preprint 2610.03519, original v1",
      "publisher": "arXiv / submitting research authors",
      "url": "https://arxiv.org/abs/2610.03519v1",
      "canonical_url": "https://arxiv.org/abs/2610.03519v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-02T12:12:21.000-04:00",
      "accessed_at": "2026-10-05T08:32:11.058-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-04-027-001"
      ]
    }
  ],
  "corrections": [
    {
      "correction_id": "correction-2026-10-04-001",
      "revision": 2,
      "summary": "Publication note: This October 4 edition was reconstructed on October 5 after the workstation was offline. It uses original October 2 submissions from the standard Sunday-evening arXiv release, rather than claiming those papers were available for the missed 9 AM run. The date and 9 AM publication slot preserve archive chronology; October 5 developments are excluded. The September 27 Invention Desk is carried over because its replacement was generated October 5.",
      "published_at": "2026-10-05T08:46:40.372-04:00",
      "affected_story_ids": [],
      "affected_claim_ids": []
    }
  ],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "A High Score Hid an Impossible Causal Claim",
    "publisher": "The Machine Press",
    "published_at": "2026-10-04T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-04-027/a-high-score-hid-an-impossible-causal-claim"
  }
}