{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-19-013",
    "source_story_id": "tmp-story-futransformer-mixed-precision",
    "edition_id": "mp-2026-10-19-morning-0101",
    "edition_url": "https://themachinepress.com/edition/2026-10-19",
    "position": 13,
    "story_type": "dispatch",
    "section": "chips-infrastructure",
    "editorial_classification": "editorial",
    "headline": "Prefill and Decode Stop Sharing the Same Number Format",
    "slug": "prefill-and-decode-stop-sharing-the-same-number-format",
    "dek": "FuTransformer uses FP8 where tokens reuse weights and INT4 where single-token decoding is limited by reading them.",
    "summary": "FuTransformer uses FP8 where tokens reuse weights and INT4 where single-token decoding is limited by reading them.",
    "body_text": "Transformer prefill and decode stress hardware differently, so FuTransformer gives them different kernels and precision. On Hopper GPUs, prefill uses FP8 with fused epilogues and direct cache placement; decode uses INT4 projections and a rotated INT4 key-value cache to reduce outlier error. On one H100, the author reports prefill up to 1.48 times faster than compiled PyTorch FP8 and INT4 decode 1.3 to 2.4 times faster than the best evaluated PyTorch path across tested shapes and batches. Full Llama-3.1-8B decoding reached 2.8 milliseconds per step at batch one in the reported setup, with a perplexity cost from quantization. These are implementation-specific comparisons, not universal model-serving gains.",
    "why_it_matters": "FuTransformer uses FP8 where tokens reuse weights and INT4 where single-token decoding is limited by reading them.",
    "limitations": [
      "These are implementation-specific comparisons, not universal model-serving gains."
    ],
    "importance": 8,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-19-013/prefill-and-decode-stop-sharing-the-same-number-format",
    "json_url": "https://themachinepress.com/story/mp-2026-10-19-013.json",
    "first_published_at": "2026-10-19T09:00:00.000-04:00",
    "modified_at": "2026-10-19T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-19-013-001",
        "text": "FuTransformer uses FP8 where tokens reuse weights and INT4 where single-token decoding is limited by reading them.",
        "source_ids": [
          "source-2026-10-19-013"
        ],
        "qualification": "These are implementation-specific comparisons, not universal model-serving gains."
      }
    ],
    "source_ids": [
      "source-2026-10-19-013"
    ],
    "tags": [
      "inference",
      "quantization",
      "GPU kernels"
    ],
    "image_url": "https://themachinepress.com/issues/2026-10-19/futransformer-chip.webp",
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-19-013",
      "title": "arXiv preprint 2610.23796, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.23796v1",
      "canonical_url": "https://arxiv.org/abs/2610.23796v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-16T13:22:01.000-04:00",
      "accessed_at": "2026-10-19T08:27:48.782-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-19-013-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "Prefill and Decode Stop Sharing the Same Number Format",
    "publisher": "The Machine Press",
    "published_at": "2026-10-19T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-19-013/prefill-and-decode-stop-sharing-the-same-number-format"
  }
}