{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-28-006",
    "source_story_id": "tmp-story-requirements-to-tests",
    "edition_id": "mp-2026-10-28-morning-0110",
    "edition_url": "https://themachinepress.com/edition/2026-10-28",
    "position": 6,
    "story_type": "dispatch",
    "section": "developer-tools",
    "editorial_classification": "editorial",
    "headline": "Passing Tests Covered Only a Third of the Required Behavior",
    "slug": "passing-tests-covered-only-a-third-of-the-required-behavior",
    "dek": "On Req2Test-Bench, the strongest agent configuration produced runnable tests for 82.8 percent of cases but behaviorally adequate tests for 33.8 percent.",
    "summary": "On Req2Test-Bench, the strongest agent configuration produced runnable tests for 82.8 percent of cases but behaviorally adequate tests for 33.8 percent.",
    "body_text": "A generated test can execute cleanly while failing to constrain the behavior that a requirement actually describes. Req2Test-Bench reconstructs 5,086 developer-authored behavior-driven-development instances from eight open-source projects, five languages and four frameworks. Agents receive a Gherkin requirement and target interfaces, then must generate the withheld test implementation. The best of eight evaluated configurations produced passing tests for 82.8 percent of instances, but only 33.8 percent killed every mutant killed by the developer test. Under-specified setup and implicit expected outcomes accounted for much of the information gap. The benchmark measures fidelity to recovered project artifacts; it does not establish that every developer-written test is complete.",
    "why_it_matters": "On Req2Test-Bench, the strongest agent configuration produced runnable tests for 82.8 percent of cases but behaviorally adequate tests for 33.8 percent.",
    "limitations": [
      "The best of eight evaluated configurations produced passing tests for 82.8 percent of instances, but only 33.8 percent killed every mutant killed by the developer test.",
      "The benchmark measures fidelity to recovered project artifacts; it does not establish that every developer-written test is complete."
    ],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-28-006/passing-tests-covered-only-a-third-of-the-required-behavior",
    "json_url": "https://themachinepress.com/story/mp-2026-10-28-006.json",
    "first_published_at": "2026-10-28T09:00:00.000-04:00",
    "modified_at": "2026-10-28T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-28-006-001",
        "text": "On Req2Test-Bench, the strongest agent configuration produced runnable tests for 82.8 percent of cases but behaviorally adequate tests for 33.8 percent.",
        "source_ids": [
          "source-2026-10-28-006"
        ],
        "qualification": "The best of eight evaluated configurations produced passing tests for 82.8 percent of instances, but only 33.8 percent killed every mutant killed by the developer test."
      }
    ],
    "source_ids": [
      "source-2026-10-28-006"
    ],
    "tags": [
      "software testing",
      "coding agents",
      "requirements",
      "mutation testing"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-28-006",
      "title": "arXiv preprint 2610.36584, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.36584v1",
      "canonical_url": "https://arxiv.org/abs/2610.36584v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-26T20:00:00.000-04:00",
      "accessed_at": "2026-10-28T08:24:14.982-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-28-006-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "Passing Tests Covered Only a Third of the Required Behavior",
    "publisher": "The Machine Press",
    "published_at": "2026-10-28T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-28-006/passing-tests-covered-only-a-third-of-the-required-behavior"
  }
}