{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-09-30-002",
    "source_story_id": "tmp-feature-userproxybench-simulated-users",
    "edition_id": "mp-2026-09-30-morning-0083",
    "edition_url": "https://themachinepress.com/edition/2026-09-30",
    "position": 2,
    "story_type": "secondary",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "The Benchmark Changed When Only the Simulated User Changed",
    "slug": "the-benchmark-changed-when-only-the-simulated-user-changed",
    "dek": "UserProxyBench found that agent reward can stay high even when a proxy user breaks its private instructions.",
    "summary": "UserProxyBench found that agent reward can stay high even when a proxy user breaks its private instructions.",
    "body_text": "UserProxyBench evaluates the language model assigned to play the user in interactive agent tests, scoring adherence to private instructions independently of whether the agent completes the task. With the agent fixed and only the user proxy varied across 375 enterprise tasks, mean reward moved by 15.2 points. The authors found user-specification violations in 24.4% of successful episodes; premature disclosure was the dominant failure, and successful episodes with it used 1.06 fewer tool calls on average. The result shows that task reward can conceal a changed interaction, within the tested tau-bench-derived setting and seven proxy models.",
    "why_it_matters": "UserProxyBench found that agent reward can stay high even when a proxy user breaks its private instructions.",
    "limitations": [
      "With the agent fixed and only the user proxy varied across 375 enterprise tasks, mean reward moved by 15.2 points."
    ],
    "importance": 10,
    "canonical_url": "https://themachinepress.com/story/mp-2026-09-30-002/the-benchmark-changed-when-only-the-simulated-user-changed",
    "json_url": "https://themachinepress.com/story/mp-2026-09-30-002.json",
    "first_published_at": "2026-09-30T09:00:00.000-04:00",
    "modified_at": "2026-09-30T09:02:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-09-30-002-001",
        "text": "UserProxyBench found that agent reward can stay high even when a proxy user breaks its private instructions.",
        "source_ids": [
          "source-2026-09-30-002"
        ],
        "qualification": "With the agent fixed and only the user proxy varied across 375 enterprise tasks, mean reward moved by 15.2 points."
      }
    ],
    "source_ids": [
      "source-2026-09-30-002"
    ],
    "tags": [
      "agent benchmarks",
      "user simulation",
      "evaluation fidelity"
    ],
    "image_url": "https://themachinepress.com/issues/2026-09-30/feature-user-proxy-benchmark.png",
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-09-30-002",
      "title": "arXiv preprint 2609.38043",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2609.38043",
      "canonical_url": "https://arxiv.org/abs/2609.38043",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-09-29T13:17:24.000-04:00",
      "accessed_at": "2026-09-30T08:24:00.000-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-09-30-002-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Benchmark Changed When Only the Simulated User Changed",
    "publisher": "The Machine Press",
    "published_at": "2026-09-30T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-09-30-002/the-benchmark-changed-when-only-the-simulated-user-changed"
  }
}