{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-27-005",
    "source_story_id": "tmp-story-rare-disease-evaluation-leakage",
    "edition_id": "mp-2026-10-27-morning-0109",
    "edition_url": "https://themachinepress.com/edition/2026-10-27",
    "position": 5,
    "story_type": "dispatch",
    "section": "research",
    "editorial_classification": "editorial",
    "headline": "The Diagnostic Benchmark Knew Too Much About Its Answers",
    "slug": "the-diagnostic-benchmark-knew-too-much-about-its-answers",
    "dek": "Strict cross-dataset isolation cut mean Recall@1 from 54.3 to 40.3 percent in a reconstructed rare-disease agentic framework.",
    "summary": "Strict cross-dataset isolation cut mean Recall@1 from 54.3 to 40.3 percent in a reconstructed rare-disease agentic framework.",
    "body_text": "High benchmark scores for augmented language-model diagnosis can reflect evaluation structure rather than clinical reasoning. An audit covering more than 60,000 cases reconstructed a state-of-the-art agentic framework and separated candidate sets across datasets. Mean Recall@1 fell from 54.3 percent under conventional one-versus-rest evaluation to 40.3 percent under strict isolation. Supervised classifiers showed a second generalization gap, declining from 42.9 percent internally to 16.7 percent across external cohorts. The authors also found that retrospective treatment cues can reveal diagnoses and that time mismatches in knowledge bases move results. This does not establish clinical performance for every system; it shows why rare-disease evaluations need data isolation, temporal controls and safeguards against therapeutic clues.",
    "why_it_matters": "Strict cross-dataset isolation cut mean Recall@1 from 54.3 to 40.3 percent in a reconstructed rare-disease agentic framework.",
    "limitations": [
      "This does not establish clinical performance for every system; it shows why rare-disease evaluations need data isolation, temporal controls and safeguards against therapeutic clues."
    ],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-27-005/the-diagnostic-benchmark-knew-too-much-about-its-answers",
    "json_url": "https://themachinepress.com/story/mp-2026-10-27-005.json",
    "first_published_at": "2026-10-27T09:00:00.000-04:00",
    "modified_at": "2026-10-27T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-27-005-001",
        "text": "Strict cross-dataset isolation cut mean Recall@1 from 54.3 to 40.3 percent in a reconstructed rare-disease agentic framework.",
        "source_ids": [
          "source-2026-10-27-005"
        ],
        "qualification": "This does not establish clinical performance for every system; it shows why rare-disease evaluations need data isolation, temporal controls and safeguards against therapeutic clues."
      }
    ],
    "source_ids": [
      "source-2026-10-27-005"
    ],
    "tags": [
      "medical AI",
      "data leakage",
      "evaluation",
      "rare disease"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-27-005",
      "title": "arXiv preprint 2610.33592, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.33592v1",
      "canonical_url": "https://arxiv.org/abs/2610.33592v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-25T20:00:00.000-04:00",
      "accessed_at": "2026-10-27T08:18:12.114-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-27-005-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Diagnostic Benchmark Knew Too Much About Its Answers",
    "publisher": "The Machine Press",
    "published_at": "2026-10-27T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-27-005/the-diagnostic-benchmark-knew-too-much-about-its-answers"
  }
}