{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-06-019",
    "source_story_id": "tmp-sidebar-test-set-tuning",
    "edition_id": "mp-2026-10-06-morning-0089",
    "edition_url": "https://themachinepress.com/edition/2026-10-06",
    "position": 21,
    "story_type": "ticker",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "Test-Set Tuning Moved Scores More Than Rankings",
    "slug": "test-set-tuning-moved-scores-more-than-rankings",
    "dek": "Experiments found real performance inflation but often preserved which models ranked ahead.",
    "summary": "Experiments found real performance inflation but often preserved which models ranked ahead.",
    "body_text": "Across MNIST-1D, CIFAR-10 and three GLUE tasks, tuning hyperparameters on the test set produced statistically significant optimism, yet the magnitude was often small beside other noise and model rankings were largely preserved. The authors argue for transparent reporting and a more measured diagnosis—not permission to erase the validation boundary.",
    "why_it_matters": "Experiments found real performance inflation but often preserved which models ranked ahead.",
    "limitations": [
      "The authors argue for transparent reporting and a more measured diagnosis—not permission to erase the validation boundary."
    ],
    "importance": 5,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-06-019/test-set-tuning-moved-scores-more-than-rankings",
    "json_url": "https://themachinepress.com/story/mp-2026-10-06-019.json",
    "first_published_at": "2026-10-06T09:00:00.000-04:00",
    "modified_at": "2026-10-06T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-06-019-001",
        "text": "Experiments found real performance inflation but often preserved which models ranked ahead.",
        "source_ids": [
          "source-2026-10-06-021"
        ],
        "qualification": "The authors argue for transparent reporting and a more measured diagnosis—not permission to erase the validation boundary."
      }
    ],
    "source_ids": [
      "source-2026-10-06-021"
    ],
    "tags": [
      "test sets",
      "hyperparameters",
      "benchmark integrity"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-06-021",
      "title": "arXiv preprint 2610.05902",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.05902",
      "canonical_url": "https://arxiv.org/abs/2610.05902",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-05T03:21:50.000-04:00",
      "accessed_at": "2026-10-06T08:25:42.165-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-06-019-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "Test-Set Tuning Moved Scores More Than Rankings",
    "publisher": "The Machine Press",
    "published_at": "2026-10-06T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-06-019/test-set-tuning-moved-scores-more-than-rankings"
  }
}