{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-07-10-012",
    "source_story_id": "tmp-story-swe-bench-pro-audit",
    "edition_id": "mp-2026-07-10-morning-0001",
    "edition_url": "https://themachinepress.com/edition/2026-07-10",
    "position": 12,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "A Coding Benchmark Loses Its Signal",
    "slug": "a-coding-benchmark-loses-its-signal",
    "dek": "An OpenAI audit estimates that roughly thirty percent of SWE-Bench Pro's public tasks are broken or unfairly specified.",
    "summary": "An OpenAI audit estimates that roughly thirty percent of SWE-Bench Pro's public tasks are broken or unfairly specified.",
    "body_text": "OpenAI audited the 731-task public split of SWE-Bench Pro after pass rates climbed sharply. Its automated review flagged 27.4 percent of tasks as broken, while a separate campaign with five experienced engineers per task marked 34.1 percent. The recurring problems were overly strict tests, underspecified prompts, low test coverage, and misleading instructions. Because coding benchmark scores inform capability and safety judgments, OpenAI withdrew its earlier recommendation to use SWE-Bench Pro and called for evaluations designed explicitly for model testing.",
    "why_it_matters": "An OpenAI audit estimates that roughly thirty percent of SWE-Bench Pro's public tasks are broken or unfairly specified.",
    "limitations": [],
    "importance": 7,
    "canonical_url": "https://themachinepress.com/story/mp-2026-07-10-012/a-coding-benchmark-loses-its-signal",
    "json_url": "https://themachinepress.com/story/mp-2026-07-10-012.json",
    "first_published_at": "2026-07-10T09:00:00.000-04:00",
    "modified_at": "2026-07-10T10:47:01.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-07-10-012-001",
        "text": "An OpenAI audit estimates that roughly thirty percent of SWE-Bench Pro's public tasks are broken or unfairly specified.",
        "source_ids": [
          "source-2026-07-10-012"
        ],
        "qualification": null
      }
    ],
    "source_ids": [
      "source-2026-07-10-012"
    ],
    "tags": [
      "OpenAI",
      "SWE-Bench Pro",
      "coding agents",
      "evaluations",
      "benchmarks"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-07-10-012",
      "title": "OpenAI: Separating signal from noise in coding evaluations",
      "publisher": "openai.com",
      "url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations/",
      "canonical_url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations/",
      "source_type": "secondary_reporting",
      "is_primary_source": false,
      "published_at": "2026-07-07T20:00:00.000-04:00",
      "accessed_at": "2026-07-11T01:05:00.000-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-07-10-012-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "A Coding Benchmark Loses Its Signal",
    "publisher": "The Machine Press",
    "published_at": "2026-07-10T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-07-10-012/a-coding-benchmark-loses-its-signal"
  }
}