{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-28-007",
    "source_story_id": "tmp-story-infradev-coding-agents",
    "edition_id": "mp-2026-10-28-morning-0110",
    "edition_url": "https://themachinepress.com/edition/2026-10-28",
    "position": 7,
    "story_type": "dispatch",
    "section": "developer-tools",
    "editorial_classification": "editorial",
    "headline": "Official Tests Accepted a Third of the Wrong Infrastructure Patches",
    "slug": "official-tests-accepted-a-third-of-the-wrong-infrastructure-patches",
    "dek": "InfraDev adds hidden probes to 36 multi-file AI-infrastructure tasks; the best frontier model resolves 38.9 percent, and probes reject 32 percent of official-test passers.",
    "summary": "InfraDev adds hidden probes to 36 multi-file AI-infrastructure tasks; the best frontier model resolves 38.9 percent, and probes reject 32 percent of official-test passers.",
    "body_text": "Production AI infrastructure changes span distributed training, inference and reinforcement-learning systems, but most coding benchmarks stop at a kernel or one GPU. InfraDev reconstructs 36 pull requests from Megatron-LM, vLLM and verl as repository tasks that must run across many GPUs. Hidden probes cover requirements omitted by the original tests while an explicit prompt names every interface those probes invoke. Across six frontier models, the strongest resolved 38.9 percent. The probes rejected 32 percent of submissions that passed all official tests; using original pull-request descriptions as prompts produced no solved tasks. The benchmark is small and infrastructure-specific, yet it demonstrates why state, memory, time and traffic behavior need evaluation beyond nominal test suites.",
    "why_it_matters": "InfraDev adds hidden probes to 36 multi-file AI-infrastructure tasks; the best frontier model resolves 38.9 percent, and probes reject 32 percent of official-test passers.",
    "limitations": [],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-28-007/official-tests-accepted-a-third-of-the-wrong-infrastructure-patches",
    "json_url": "https://themachinepress.com/story/mp-2026-10-28-007.json",
    "first_published_at": "2026-10-28T09:00:00.000-04:00",
    "modified_at": "2026-10-28T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-28-007-001",
        "text": "InfraDev adds hidden probes to 36 multi-file AI-infrastructure tasks; the best frontier model resolves 38.9 percent, and probes reject 32 percent of official-test passers.",
        "source_ids": [
          "source-2026-10-28-007"
        ],
        "qualification": null
      }
    ],
    "source_ids": [
      "source-2026-10-28-007"
    ],
    "tags": [
      "coding agents",
      "AI infrastructure",
      "hidden tests",
      "distributed systems"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-28-007",
      "title": "arXiv preprint 2610.36589, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.36589v1",
      "canonical_url": "https://arxiv.org/abs/2610.36589v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-26T20:00:00.000-04:00",
      "accessed_at": "2026-10-28T08:24:14.982-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-28-007-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "Official Tests Accepted a Third of the Wrong Infrastructure Patches",
    "publisher": "The Machine Press",
    "published_at": "2026-10-28T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-28-007/official-tests-accepted-a-third-of-the-wrong-infrastructure-patches"
  }
}