{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-03-008",
    "source_story_id": "tmp-story-keyword-harness-fail-open",
    "edition_id": "mp-2026-10-03-morning-0086",
    "edition_url": "https://themachinepress.com/edition/2026-10-03",
    "position": 8,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "The Tool Benchmark Passed Keywords, Not Commands",
    "slug": "the-tool-benchmark-passed-keywords-not-commands",
    "dek": "A diagnostic ladder found that small-model tool-use claims could survive weak lexical checks while failing execution structure.",
    "summary": "A diagnostic ladder found that small-model tool-use claims could survive weak lexical checks while failing execution structure.",
    "body_text": "The Keyword Harnesses Fail Open study examines evaluations that reward outputs for containing expected words rather than forming valid tool calls. Its staged diagnostic separates lexical overlap from argument binding, ordering and executable structure, exposing cases in which a reported success disappears under stricter checks. The lesson is narrow but practical: tool-use claims need verifiers tied to the real interface contract, not a permissive keyword proxy.",
    "why_it_matters": "A diagnostic ladder found that small-model tool-use claims could survive weak lexical checks while failing execution structure.",
    "limitations": [
      "The lesson is narrow but practical: tool-use claims need verifiers tied to the real interface contract, not a permissive keyword proxy."
    ],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-03-008/the-tool-benchmark-passed-keywords-not-commands",
    "json_url": "https://themachinepress.com/story/mp-2026-10-03-008.json",
    "first_published_at": "2026-10-03T09:00:00.000-04:00",
    "modified_at": "2026-10-03T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-03-008-001",
        "text": "A diagnostic ladder found that small-model tool-use claims could survive weak lexical checks while failing execution structure.",
        "source_ids": [
          "source-2026-10-03-008"
        ],
        "qualification": "The lesson is narrow but practical: tool-use claims need verifiers tied to the real interface contract, not a permissive keyword proxy."
      }
    ],
    "source_ids": [
      "source-2026-10-03-008"
    ],
    "tags": [
      "tool use",
      "evaluation",
      "verification"
    ],
    "image_url": "https://themachinepress.com/issues/2026-10-03/keyword-harness-code.webp",
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-03-008",
      "title": "arXiv preprint 2610.02142",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.02142",
      "canonical_url": "https://arxiv.org/abs/2610.02142",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-01T13:46:38.000-04:00",
      "accessed_at": "2026-10-03T08:31:37.000-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-03-008-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Tool Benchmark Passed Keywords, Not Commands",
    "publisher": "The Machine Press",
    "published_at": "2026-10-03T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-03-008/the-tool-benchmark-passed-keywords-not-commands"
  }
}