Generative AI Part 8: Structured Outputs for Reliable JSON from LLMs

import json

# A typical reply when you only *ask* for JSON in the prompt
reply = """Sure! Here is the data you asked for:
```json
{"name": "Asha", "age": "twenty-six", "skills": ["Python", "SQL"],}
```
Let me know if you need anything else."""

try:
    data = json.loads(reply)
    print("Parsed:", data)
except json.JSONDecodeError as err:
    print("JSONDecodeError:", err)

$ python naive_json.py
JSONDecodeError: Expecting value: line 1 column 1 (char 0)

// A typical reply when you only *ask* for JSON in the prompt
const reply = `Sure! Here is the data you asked for:
\`\`\`json
{"name": "Asha", "age": "twenty-six", "skills": ["Python", "SQL"],}
\`\`\`
Let me know if you need anything else.`;

try {
  const data = JSON.parse(reply);
  console.log("Parsed:", data);
} catch (err) {
  console.log(`${err.name}: ${err.message}`);
}

$ node naive-json.mjs
SyntaxError: Unexpected token 'S', "Sure! Here"... is not valid JSON

# pip install pydantic
from typing import Literal
from pydantic import BaseModel, Field

class Ticket(BaseModel):
    """A customer support ticket extracted from an email."""
    customer_name: str
    category: Literal["billing", "bug", "feature_request", "other"]
    priority: int = Field(ge=1, le=5, description="1 = low, 5 = urgent")
    summary: str = Field(description="One short sentence")
    order_id: str | None  # nullable, but the key must always be present

if __name__ == "__main__":
    schema = Ticket.model_json_schema()
    print("required:", schema["required"])
    print("category:", schema["properties"]["category"]["enum"])
    print("priority:", schema["properties"]["priority"])

$ python schema.py
required: ['customer_name', 'category', 'priority', 'summary', 'order_id']
category: ['billing', 'bug', 'feature_request', 'other']
priority: {'description': '1 = low, 5 = urgent', 'maximum': 5, 'minimum': 1, 'title': 'Priority', 'type': 'integer'}

// npm install zod
import { z } from "zod";

export const Ticket = z.object({
  customer_name: z.string(),
  category: z.enum(["billing", "bug", "feature_request", "other"]),
  priority: z.number().int().min(1).max(5).describe("1 = low, 5 = urgent"),
  summary: z.string().describe("One short sentence"),
  order_id: z.string().nullable(), // nullable, but the key must always be present
});

if (process.argv[1]?.endsWith("schema.mjs")) {
  const schema = z.toJSONSchema(Ticket);
  console.log("required:", schema.required);
  console.log("category:", schema.properties.category.enum);
  console.log("priority:", schema.properties.priority);
}

$ node schema.mjs
required: [ 'customer_name', 'category', 'priority', 'summary', 'order_id' ]
category: [ 'billing', 'bug', 'feature_request', 'other' ]
priority: {
  type: 'integer',
  minimum: 1,
  maximum: 5,
  description: '1 = low, 5 = urgent'
}

# pip install openai pydantic python-dotenv
import os
from dotenv import load_dotenv
from openai import OpenAI
from schema import Ticket  # the Pydantic model from schema.py

load_dotenv()
client = OpenAI()  # reads OPENAI_API_KEY from .env
email = open("email.txt", encoding="utf-8").read()

response = client.responses.parse(
    model=os.getenv("OPENAI_MODEL", "gpt-6-luna"),
    instructions="Extract a support ticket from the customer email.",
    input=email,
    text_format=Ticket,  # <- the SDK turns the model into a strict JSON Schema
)

ticket = response.output_parsed  # a real Ticket object, already validated
if ticket is None:
    print("No ticket. Status:", response.status)  # refusal or cut off
else:
    print(ticket.category, ticket.priority, ticket.order_id)
    print(ticket.model_dump_json(indent=2))

$ python extract_openai.py
billing 5 A-4821
{
  "customer_name": "Rahul Mehta",
  "category": "billing",
  "priority": 5,
  "summary": "Customer was charged twice for order A-4821 and wants a refund today.",
  "order_id": "A-4821"
}

// npm install openai zod
// run with: node --env-file=.env extract-openai.mjs
import { readFile } from "node:fs/promises";
import OpenAI from "openai";
import { zodTextFormat } from "openai/helpers/zod";
import { Ticket } from "./schema.mjs";

const client = new OpenAI(); // reads OPENAI_API_KEY
const email = await readFile("email.txt", "utf8");

const response = await client.responses.parse({
  model: process.env.OPENAI_MODEL ?? "gpt-6-luna",
  instructions: "Extract a support ticket from the customer email.",
  input: email,
  text: { format: zodTextFormat(Ticket, "ticket") }, // <- strict JSON Schema
});

const ticket = response.output_parsed; // typed and validated
if (!ticket) {
  console.log("No ticket. Status:", response.status); // refusal or cut off
} else {
  console.log(ticket.category, ticket.priority, ticket.order_id);
  console.log(JSON.stringify(ticket, null, 2));
}

$ node --env-file=.env extract-openai.mjs
billing 5 A-4821
{
  "customer_name": "Rahul Mehta",
  "category": "billing",
  "priority": 5,
  "summary": "Customer was charged twice for order A-4821 and wants a refund today.",
  "order_id": "A-4821"
}

Hi team, I'm Rahul Mehta. I was charged twice for order #A-4821 this morning
and I need the extra payment refunded today, please. Thanks!

# pip install ollama pydantic   (and: ollama pull llama3.2)
from ollama import chat
from schema import Ticket

email = open("email.txt", encoding="utf-8").read()

response = chat(
    model="llama3.2",
    messages=[
        {"role": "system", "content": "Extract a support ticket from the customer email. Reply in JSON."},
        {"role": "user", "content": email},
    ],
    format=Ticket.model_json_schema(),  # <- Ollama only generates JSON that fits
    options={"temperature": 0},
)

ticket = Ticket.model_validate_json(response.message.content)  # still validate!
print(ticket.model_dump_json(indent=2))

$ python extract_ollama.py
{
  "customer_name": "Rahul Mehta",
  "category": "billing",
  "priority": 5,
  "summary": "Customer was charged twice for order A-4821 and wants a refund today.",
  "order_id": "A-4821"
}

// npm install ollama zod   (and: ollama pull llama3.2)
import { readFile } from "node:fs/promises";
import ollama from "ollama";
import { z } from "zod";
import { Ticket } from "./schema.mjs";

const email = await readFile("email.txt", "utf8");

const response = await ollama.chat({
  model: "llama3.2",
  messages: [
    { role: "system", content: "Extract a support ticket from the customer email. Reply in JSON." },
    { role: "user", content: email },
  ],
  format: z.toJSONSchema(Ticket), // <- Ollama only generates JSON that fits
  options: { temperature: 0 },
});

const ticket = Ticket.parse(JSON.parse(response.message.content)); // still validate!
console.log(JSON.stringify(ticket, null, 2));

$ node extract-ollama.mjs
{
  "customer_name": "Rahul Mehta",
  "category": "billing",
  "priority": 5,
  "summary": "Customer was charged twice for order A-4821 and wants a refund today.",
  "order_id": "A-4821"
}

import json
from pydantic import ValidationError
from schema import Ticket

# Pretend LLM: the first reply is broken, the second one is fixed.
fake_replies = iter([
    '{"customer_name": "Rahul", "category": "payment", "priority": 9, "summary": "Charged twice"}',
    '{"customer_name": "Rahul", "category": "billing", "priority": 5, '
    '"summary": "Charged twice for one order", "order_id": "A-4821"}',
])

def ask_llm(messages):
    return next(fake_replies)  # swap in a real API call here

def get_ticket(email, max_tries=3):
    messages = [{"role": "user", "content": f"Extract a ticket as JSON:\n{email}"}]
    for attempt in range(1, max_tries + 1):
        reply = ask_llm(messages)
        try:
            return Ticket.model_validate_json(reply)
        except ValidationError as err:
            print(f"Attempt {attempt} failed with {err.error_count()} errors:")
            for e in err.errors():
                print("  -", ".".join(map(str, e["loc"])), "->", e["msg"])
            # show the model its mistakes and ask again
            messages += [
                {"role": "assistant", "content": reply},
                {"role": "user", "content": f"Fix these errors and reply with JSON only:\n{err}"},
            ]
    raise RuntimeError("No valid JSON after retries")

print("OK:", get_ticket("I was charged twice for order #A-4821"))

$ python validate_retry.py
Attempt 1 failed with 3 errors:
  - category -> Input should be 'billing', 'bug', 'feature_request' or 'other'
  - priority -> Input should be less than or equal to 5
  - order_id -> Field required
OK: customer_name='Rahul' category='billing' priority=5 summary='Charged twice for one order' order_id='A-4821'

import { Ticket } from "./schema.mjs";

// Pretend LLM: the first reply is broken, the second one is fixed.
const fakeReplies = [
  '{"customer_name": "Rahul", "category": "payment", "priority": 9, "summary": "Charged twice"}',
  '{"customer_name": "Rahul", "category": "billing", "priority": 5, "summary": "Charged twice for one order", "order_id": "A-4821"}',
];
const askLLM = async (messages) => fakeReplies.shift(); // swap in a real API call

async function getTicket(email, maxTries = 3) {
  const messages = [{ role: "user", content: `Extract a ticket as JSON:\n${email}` }];
  for (let attempt = 1; attempt <= maxTries; attempt++) {
    const reply = await askLLM(messages);
    const result = Ticket.safeParse(JSON.parse(reply));
    if (result.success) return result.data;
    console.log(`Attempt ${attempt} failed with ${result.error.issues.length} errors:`);
    for (const issue of result.error.issues) {
      console.log("  -", issue.path.join("."), "->", issue.message);
    }
    messages.push(
      { role: "assistant", content: reply },
      { role: "user", content: `Fix these errors and reply with JSON only:\n${result.error.message}` },
    );
  }
  throw new Error("No valid JSON after retries");
}

console.log("OK:", await getTicket("I was charged twice for order #A-4821"));

$ node validate-retry.mjs
Attempt 1 failed with 3 errors:
  - category -> Invalid option: expected one of "billing"|"bug"|"feature_request"|"other"
  - priority -> Too big: expected number to be <=5
  - order_id -> Invalid input: expected string, received undefined
OK: {
  customer_name: 'Rahul',
  category: 'billing',
  priority: 5,
  summary: 'Charged twice for one order',
  order_id: 'A-4821'
}