{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-30-008",
    "source_story_id": "tmp-story-paperbenchx-science-reproduction",
    "edition_id": "mp-2026-10-30-morning-0112",
    "edition_url": "https://themachinepress.com/edition/2026-10-30",
    "position": 8,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "The Best Agent Reproduced Only 14 Percent of the Papers",
    "slug": "the-best-agent-reproduced-only-14-percent-of-the-papers",
    "dek": "PAPERBENCHX replays submissions from scratch across 93 expert-built tasks and 12 scientific areas, scoring regenerated evidence rather than plausible-looking outputs.",
    "summary": "PAPERBENCHX replays submissions from scratch across 93 expert-built tasks and 12 scientific areas, scoring regenerated evidence rather than plausible-looking outputs.",
    "body_text": "Before an AI system discovers new science, it should be able to reproduce established results. PAPERBENCHX spans 93 expert-constructed tasks in 12 areas, including electromagnetics, photonics, chemistry, materials, biology and robotics. Agents receive a paper, a task specification and a domain-native software environment, then must rebuild an experiment and produce a runnable workflow. Every submission is replayed from scratch and only regenerated evidence is scored. Across 10 frontier-agent configurations, the best fully reproduced 14 percent of tasks at the 90 percent threshold; mean rubric scores ranged from 27.53 to 59.08 percent. The benchmark measures computational reproduction, not all of science, but shows how partial progress can look much stronger than end-to-end evidence.",
    "why_it_matters": "PAPERBENCHX replays submissions from scratch across 93 expert-built tasks and 12 scientific areas, scoring regenerated evidence rather than plausible-looking outputs.",
    "limitations": [
      "Every submission is replayed from scratch and only regenerated evidence is scored.",
      "The benchmark measures computational reproduction, not all of science, but shows how partial progress can look much stronger than end-to-end evidence."
    ],
    "importance": 10,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-30-008/the-best-agent-reproduced-only-14-percent-of-the-papers",
    "json_url": "https://themachinepress.com/story/mp-2026-10-30-008.json",
    "first_published_at": "2026-10-30T09:00:00.000-04:00",
    "modified_at": "2026-10-30T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-30-008-001",
        "text": "PAPERBENCHX replays submissions from scratch across 93 expert-built tasks and 12 scientific areas, scoring regenerated evidence rather than plausible-looking outputs.",
        "source_ids": [
          "source-2026-10-30-008"
        ],
        "qualification": "Every submission is replayed from scratch and only regenerated evidence is scored."
      }
    ],
    "source_ids": [
      "source-2026-10-30-008"
    ],
    "tags": [
      "AI scientists",
      "reproducibility",
      "benchmarks",
      "scientific software"
    ],
    "image_url": "https://themachinepress.com/issues/2026-10-30/paperbenchx-reproduction.webp",
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-30-008",
      "title": "arXiv preprint 2610.39744, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.39744v1",
      "canonical_url": "https://arxiv.org/abs/2610.39744v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-28T20:00:00.000-04:00",
      "accessed_at": "2026-10-30T08:14:40.422-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-30-008-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Best Agent Reproduced Only 14 Percent of the Papers",
    "publisher": "The Machine Press",
    "published_at": "2026-10-30T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-30-008/the-best-agent-reproduced-only-14-percent-of-the-papers"
  }
}