{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-11-01-014",
    "source_story_id": "tmp-story-rtaste-research-agents",
    "edition_id": "mp-2026-11-01-morning-0114",
    "edition_url": "https://themachinepress.com/edition/2026-11-01",
    "position": 14,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "Research Agents Propose Useful Experiments. Ranking Them Is Harder.",
    "slug": "research-agents-propose-useful-experiments-ranking-them-is-harder",
    "dek": "Of 415 proposed minimum viable experiments, 194 were judged sound, valuable and new, but only 73 also moved beyond the seed paper and the obvious.",
    "summary": "Of 415 proposed minimum viable experiments, 194 were judged sound, valuable and new, but only 73 also moved beyond the seed paper and the obvious.",
    "body_text": "RTaste asks whether research agents can choose which experiment deserves to run before seeing the outcome. Six frontier models proposed, forecast and ranked 415 minimum viable experiments based on seven recent NLP papers; a separate executor attempted every proposal, and model auditors judged the results. Auditors labeled 194 sound, valuable and new, but only 73 also beyond the seed paper and non-obvious. First choices beat random selection yet did not outperform simply choosing the first proposed experiment, and teams cost more without a clear overall quality gain. Because model auditors supplied the judgments, the harness is evidence about this setup rather than a definitive measure of scientific taste.",
    "why_it_matters": "Of 415 proposed minimum viable experiments, 194 were judged sound, valuable and new, but only 73 also moved beyond the seed paper and the obvious.",
    "limitations": [
      "Auditors labeled 194 sound, valuable and new, but only 73 also beyond the seed paper and non-obvious.",
      "First choices beat random selection yet did not outperform simply choosing the first proposed experiment, and teams cost more without a clear overall quality gain."
    ],
    "importance": 8,
    "canonical_url": "https://themachinepress.com/story/mp-2026-11-01-014/research-agents-propose-useful-experiments-ranking-them-is-harder",
    "json_url": "https://themachinepress.com/story/mp-2026-11-01-014.json",
    "first_published_at": "2026-11-01T09:00:00.000-05:00",
    "modified_at": "2026-11-01T09:00:00.000-05:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-11-01-014-001",
        "text": "Of 415 proposed minimum viable experiments, 194 were judged sound, valuable and new, but only 73 also moved beyond the seed paper and the obvious.",
        "source_ids": [
          "source-2026-11-01-014"
        ],
        "qualification": "Auditors labeled 194 sound, valuable and new, but only 73 also beyond the seed paper and non-obvious."
      }
    ],
    "source_ids": [
      "source-2026-11-01-014"
    ],
    "tags": [
      "research agents",
      "experiment design",
      "evaluation",
      "forecasting"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-11-01-014",
      "title": "arXiv preprint 2610.40488, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.40488v1",
      "canonical_url": "https://arxiv.org/abs/2610.40488v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-28T20:00:00.000-04:00",
      "accessed_at": "2026-11-01T08:20:00.000-05:00",
      "supports_claim_ids": [
        "claim-mp-2026-11-01-014-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "Research Agents Propose Useful Experiments. Ranking Them Is Harder.",
    "publisher": "The Machine Press",
    "published_at": "2026-11-01T09:00:00.000-05:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-11-01-014/research-agents-propose-useful-experiments-ranking-them-is-harder"
  }
}