{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-09-14-003",
    "source_story_id": "tmp-story-competence-gated-forecast-pooling",
    "edition_id": "mp-2026-09-14-morning-0067",
    "edition_url": "https://themachinepress.com/edition/2026-09-14",
    "position": 3,
    "story_type": "dispatch",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "The Forecasting Gate Learned When to Ignore the Model",
    "slug": "the-forecasting-gate-learned-when-to-ignore-the-model",
    "dek": "Across 2,357 resolved questions, domain-level competence weights improved the main external baseline’s Brier score from 0.0771 to 0.0732.",
    "summary": "Across 2,357 resolved questions, domain-level competence weights improved the main external baseline’s Brier score from 0.0771 to 0.0732.",
    "body_text": "A language model is not always the best signal in a forecast, especially when a market, crowd or statistical prior already exists. This study estimates each source’s marginal value by domain, shrinks uncertain weights toward a global value and recalibrates the pooled result. Across 2,357 resolved binary questions and five language models, the gate improved the main external baseline’s Brier score from 0.0771 to 0.0732 and beat global combinations under the reported leakage controls. On ForecastBench’s official market subset it found no significant gain and mostly deferred to the market. Verbal confidence across four Qwen models did not reliably identify when the model added value.",
    "why_it_matters": "Across 2,357 resolved questions, domain-level competence weights improved the main external baseline’s Brier score from 0.0771 to 0.0732.",
    "limitations": [
      "A language model is not always the best signal in a forecast, especially when a market, crowd or statistical prior already exists.",
      "Verbal confidence across four Qwen models did not reliably identify when the model added value."
    ],
    "importance": 9,
    "canonical_url": "https://themachinepress.com/story/mp-2026-09-14-003/the-forecasting-gate-learned-when-to-ignore-the-model",
    "json_url": "https://themachinepress.com/story/mp-2026-09-14-003.json",
    "first_published_at": "2026-09-14T09:00:00.000-04:00",
    "modified_at": "2026-09-14T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-09-14-003-001",
        "text": "Across 2,357 resolved questions, domain-level competence weights improved the main external baseline’s Brier score from 0.0771 to 0.0732.",
        "source_ids": [
          "source-2026-09-14-003"
        ],
        "qualification": "A language model is not always the best signal in a forecast, especially when a market, crowd or statistical prior already exists."
      }
    ],
    "source_ids": [
      "source-2026-09-14-003"
    ],
    "tags": [
      "forecasting",
      "calibration",
      "language models"
    ],
    "image_url": "https://themachinepress.com/issues/2026-09-14/forecast-competence-code-file.webp",
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-09-14-003",
      "title": "arXiv preprint 2609.12101",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2609.12101",
      "canonical_url": "https://arxiv.org/abs/2609.12101",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": null,
      "accessed_at": "2026-09-14T08:24:00.000-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-09-14-003-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Forecasting Gate Learned When to Ignore the Model",
    "publisher": "The Machine Press",
    "published_at": "2026-09-14T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-09-14-003/the-forecasting-gate-learned-when-to-ignore-the-model"
  }
}