{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-09-027",
    "source_story_id": "tmp-story-harmful-refusal-psychometric-audit",
    "edition_id": "mp-2026-10-09-morning-0091",
    "edition_url": "https://themachinepress.com/edition/2026-10-09",
    "position": 16,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "The Safety Score Had More Than One Meaning",
    "slug": "the-safety-score-had-more-than-one-meaning",
    "dek": "A psychometric audit argues that aggregate refusal scores need evidence of what they actually measure before models are compared.",
    "summary": "A psychometric audit argues that aggregate refusal scores need evidence of what they actually measure before models are compared.",
    "body_text": "The audit examined four HELM Safety datasets that could plausibly measure harmful refusal and found three saturated. On the remaining HarmBench dataset, multidimensional modeling suggested that one attribute did not explain the score. Items also behaved differently across developers at matched overall refusal ability, but those flags largely disappeared under scope-specific matching. That pattern is consistent with aggregation effects without ruling out domain-specific developer differences. The authors argue that HarmBench combines distinct harm behaviors, while the larger HELM aggregate adds further compression. This is a measurement critique, not a finding that one vendor is categorically unsafe. A single number needs a defensible interpretation before it can support a broad safety ranking.",
    "why_it_matters": "A psychometric audit argues that aggregate refusal scores need evidence of what they actually measure before models are compared.",
    "limitations": [
      "On the remaining HarmBench dataset, multidimensional modeling suggested that one attribute did not explain the score.",
      "This is a measurement critique, not a finding that one vendor is categorically unsafe."
    ],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-09-027/the-safety-score-had-more-than-one-meaning",
    "json_url": "https://themachinepress.com/story/mp-2026-10-09-027.json",
    "first_published_at": "2026-10-09T09:00:00.000-04:00",
    "modified_at": "2026-10-10T03:43:44.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-09-027-001",
        "text": "A psychometric audit argues that aggregate refusal scores need evidence of what they actually measure before models are compared.",
        "source_ids": [
          "source-2026-10-09-016"
        ],
        "qualification": "On the remaining HarmBench dataset, multidimensional modeling suggested that one attribute did not explain the score."
      }
    ],
    "source_ids": [
      "source-2026-10-09-016"
    ],
    "tags": [
      "safety measurement",
      "psychometrics",
      "benchmark validity"
    ],
    "image_url": null,
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-09-016",
      "title": "arXiv preprint 2610.12409, original v1",
      "publisher": "arXiv / submitting research authors",
      "url": "https://arxiv.org/abs/2610.12409v1",
      "canonical_url": "https://arxiv.org/abs/2610.12409v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-08T13:46:43.000-04:00",
      "accessed_at": "2026-10-10T03:27:42.309-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-09-027-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Safety Score Had More Than One Meaning",
    "publisher": "The Machine Press",
    "published_at": "2026-10-09T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-09-027/the-safety-score-had-more-than-one-meaning"
  }
}