{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-11-03-012",
    "source_story_id": "tmp-story-da-robot-reward-description",
    "edition_id": "mp-2026-11-03-morning-0116",
    "edition_url": "https://themachinepress.com/edition/2026-11-03",
    "position": 12,
    "story_type": "dispatch",
    "section": "robotics",
    "editorial_classification": "editorial",
    "headline": "Describe the Attempt Before Scoring It",
    "slug": "describe-the-attempt-before-scoring-it",
    "dek": "D&A separates video description from rubric-based assessment so a vision-language model is less likely to hallucinate robot-task progress.",
    "summary": "D&A separates video description from rubric-based assessment so a vision-language model is less likely to hallucinate robot-task progress.",
    "body_text": "Vision-language models used as zero-shot reward models often assume the task is succeeding and reduce progress to a binary verdict. D&A first requires a task-grounded description of scene dynamics, then asks for an assessment against an explicit definition of absolute success. That separation anchors the score outside the video's own timeline and makes partial progress and failure easier to distinguish. The authors report better calibration, less confirmation bias and performance matching or exceeding tested post-trained reward models across tasks and embodiments; the scores also supported downstream reinforcement learning. Because the method is prompt-based, its result depends on the underlying model and rubric, and it does not make video-derived rewards automatically safe or correct.",
    "why_it_matters": "D&A separates video description from rubric-based assessment so a vision-language model is less likely to hallucinate robot-task progress.",
    "limitations": [
      "Because the method is prompt-based, its result depends on the underlying model and rubric, and it does not make video-derived rewards automatically safe or correct."
    ],
    "importance": 8,
    "canonical_url": "https://themachinepress.com/story/mp-2026-11-03-012/describe-the-attempt-before-scoring-it",
    "json_url": "https://themachinepress.com/story/mp-2026-11-03-012.json",
    "first_published_at": "2026-11-03T09:00:00.000-05:00",
    "modified_at": "2026-11-03T09:00:00.000-05:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-11-03-012-001",
        "text": "D&A separates video description from rubric-based assessment so a vision-language model is less likely to hallucinate robot-task progress.",
        "source_ids": [
          "source-2026-11-03-012"
        ],
        "qualification": "Because the method is prompt-based, its result depends on the underlying model and rubric, and it does not make video-derived rewards automatically safe or correct."
      }
    ],
    "source_ids": [
      "source-2026-11-03-012"
    ],
    "tags": [
      "robot learning",
      "reward models",
      "vision-language models",
      "evaluation"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-11-03-012",
      "title": "arXiv preprint 2611.03622, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2611.03622v1",
      "canonical_url": "https://arxiv.org/abs/2611.03622v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-11-02T13:52:11.000-05:00",
      "accessed_at": "2026-11-03T08:18:17.000-05:00",
      "supports_claim_ids": [
        "claim-mp-2026-11-03-012-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "Describe the Attempt Before Scoring It",
    "publisher": "The Machine Press",
    "published_at": "2026-11-03T09:00:00.000-05:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-11-03-012/describe-the-attempt-before-scoring-it"
  }
}