{
  "$schema": "https://themachinepress.com/schemas/story-v1.schema.json",
  "schema_version": "1.0.0",
  "document_type": "machine_press_story",
  "story": {
    "story_id": "mp-2026-10-18-001",
    "source_story_id": "tmp-lead-premisesql-abstention",
    "edition_id": "mp-2026-10-18-morning-0100",
    "edition_url": "https://themachinepress.com/edition/2026-10-18",
    "position": 1,
    "story_type": "lead",
    "section": "benchmarks-evals",
    "editorial_classification": "editorial",
    "headline": "The Database Answers the Question That Should Have Stopped It",
    "slug": "the-database-answers-the-question-that-should-have-stopped-it",
    "dek": "PremiseSQL pairs 1,800 flawed requests with their valid originals and finds that text-to-SQL systems rarely challenge a bad premise by default.",
    "summary": "PremiseSQL pairs 1,800 flawed requests with their valid originals and finds that text-to-SQL systems rarely challenge a bad premise by default.",
    "body_text": "A database assistant can produce perfectly executable SQL for a question the database cannot truthfully answer. PremiseSQL tests whether text-to-SQL systems recognize that failure before writing the query. The benchmark pairs 1,800 flawed requests across four kinds of premise error with 1,800 valid originals, then evaluates models under three prompting settings. The authors report that frontier systems are strongly premise-trusting by default: they rarely critique a flawed request or abstain from generating misleading SQL. Prompts that explicitly encourage critique improve detection and refusal, but they also create a second failure mode—valid questions are more often mislabeled as invalid, and useful SQL is suppressed. The benchmark does not establish how often flawed premises occur in production databases, and its results depend on the tested schemas, models and prompts. Its sharper lesson is architectural: execution success is not the same as question validity. A reliable data assistant needs a gate that can inspect what the user assumes, preserve that judgment through query planning and still avoid turning healthy skepticism into blanket refusal.",
    "why_it_matters": "PremiseSQL pairs 1,800 flawed requests with their valid originals and finds that text-to-SQL systems rarely challenge a bad premise by default.",
    "limitations": [
      "A database assistant can produce perfectly executable SQL for a question the database cannot truthfully answer.",
      "The benchmark does not establish how often flawed premises occur in production databases, and its results depend on the tested schemas, models and prompts.",
      "Its sharper lesson is architectural: execution success is not the same as question validity."
    ],
    "importance": 10,
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-18-001/the-database-answers-the-question-that-should-have-stopped-it",
    "json_url": "https://themachinepress.com/story/mp-2026-10-18-001.json",
    "first_published_at": "2026-10-18T09:00:00.000-04:00",
    "modified_at": "2026-10-18T09:00:00.000-04:00",
    "content_status": "new",
    "is_carryover": false,
    "carryover_reason": null,
    "key_claims": [
      {
        "claim_id": "claim-mp-2026-10-18-001-001",
        "text": "PremiseSQL pairs 1,800 flawed requests with their valid originals and finds that text-to-SQL systems rarely challenge a bad premise by default.",
        "source_ids": [
          "source-2026-10-18-001"
        ],
        "qualification": "A database assistant can produce perfectly executable SQL for a question the database cannot truthfully answer."
      }
    ],
    "source_ids": [
      "source-2026-10-18-001"
    ],
    "tags": [
      "text-to-SQL",
      "abstention",
      "premise critique",
      "benchmarks"
    ],
    "image_url": "https://themachinepress.com/issues/2026-10-18/lead-premise-sql.png",
    "corrections": []
  },
  "sources": [
    {
      "source_id": "source-2026-10-18-001",
      "title": "arXiv preprint 2610.22158, original v1",
      "publisher": "arXiv",
      "url": "https://arxiv.org/abs/2610.22158v1",
      "canonical_url": "https://arxiv.org/abs/2610.22158v1",
      "source_type": "primary_research",
      "is_primary_source": true,
      "published_at": "2026-10-15T12:23:17.000-04:00",
      "accessed_at": "2026-10-18T08:23:52.920-04:00",
      "supports_claim_ids": [
        "claim-mp-2026-10-18-001-001"
      ]
    }
  ],
  "corrections": [],
  "publisher": {
    "name": "The Machine Press",
    "url": "https://themachinepress.com",
    "description": "A daily newspaper for the age of artificial intelligence."
  },
  "cite_this_report": {
    "title": "The Database Answers the Question That Should Have Stopped It",
    "publisher": "The Machine Press",
    "published_at": "2026-10-18T09:00:00.000-04:00",
    "canonical_url": "https://themachinepress.com/story/mp-2026-10-18-001/the-database-answers-the-question-that-should-have-stopped-it"
  }
}