{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-17-027",
    "source_story_id": "tmp-story-agent-memory-intrinsic-evaluation",
    "edition_id": "mp-2026-10-17-morning-0099",
    "edition_url": "https://themachinepress.com/edition/2026-10-17",
    "position": 16,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "A Correct Answer Can Hide a Broken Memory",
    "slug": "a-correct-answer-can-hide-a-broken-memory",
    "dek": "FEB-30 inspects the store after each fact change instead of using downstream question answering as the only proxy.",
    "summary": "FEB-30 inspects the store after each fact change instead of using downstream question answering as the only proxy.",
    "body_text": "Question-answering accuracy can blame a good memory for a retrieval failure or reward a broken memory when generation guesses correctly. FEB-30 inverts the usual benchmark: known fact-evolution trajectories generate the documents, emails and conversations, producing deterministic ground truth for direct store inspection. On one representative system, intrinsic and downstream evaluation differed by 16 points on average, with 44 percent of trajectories crossing the authors' meaningful-divergence threshold. The system classified create, update, reinforce and supersede operations correctly only 20 percent of the time. Thirty trajectories are not a comprehensive memory benchmark, but the result exposes failures that answer-only scoring cannot see.",
    "why_it_matters": "FEB-30 inspects the store after each fact change instead of using downstream question answering as the only proxy.",
    "limitations": [
      "The system classified create, update, reinforce and supersede operations correctly only 20 percent of the time.",
      "Thirty trajectories are not a comprehensive memory benchmark, but the result exposes failures that answer-only scoring cannot see."
    ],
    "importance": 8,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-17-027/a-correct-answer-can-hide-a-broken-memory",
    "json_url": "https://themachinepress.com/story/mp-2026-10-17-027.json",
    "first_published_at": "2026-10-17T09:00:00.000-04:00",
    "modified_at": "2026-10-17T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-17-027-001",
        "text": "FEB-30 inspects the store after each fact change instead of using downstream question answering as the only proxy.",
        "source_ids": [
          "source-2026-10-17-016"
        ],
        "qualification": "The system classified create, update, reinforce and supersede operations correctly only 20 percent of the time."
      }
    ],
    "source_ids": [
      "source-2026-10-17-016"
    ],
    "tags": [
      "agent memory",
      "evaluation",
      "fact evolution"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-17-016",
      "title": "arXiv preprint 2610.22209, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.22209v1",
      "canonical_url": "https://arxiv.org/abs/2610.22209v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-15T12:58:12.000-04:00",
      "accessed_at": "2026-10-17T08:26:35.645-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-17-027-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "A Correct Answer Can Hide a Broken Memory",
    "publisher": "The Machine Press",
    "published_at": "2026-10-17T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-17-027/a-correct-answer-can-hide-a-broken-memory"
  }
}