Preference data for RLHF

from typing import Literal

from agno.agent import Agent
from agno.models.openai import OpenAIResponses
from pydantic import BaseModel, Field


class Preference(BaseModel):
    winner: Literal["A", "B", "tie"] = Field(
        ..., description="Which response is better, or 'tie' if equal"
    )


agent = Agent(
    model=OpenAIResponses(id="gpt-5.5"),
    instructions=(
        "Decide which response better answers the prompt. Return 'A', 'B', "
        "or 'tie'. Use 'tie' only when the two are genuinely "
        "indistinguishable in quality."
    ),
    output_schema=Preference,
)


def build_input(prompt: str, a: str, b: str) -> str:
    return f"Prompt:\n{prompt}\n\nResponse A:\n{a}\n\nResponse B:\n{b}"


prompt = "Explain why the sky is blue, in one sentence."
a = "Shorter blue wavelengths scatter more off air molecules, so the sky looks blue."
b = "Because of physics."
result = agent.run(build_input(prompt, a, b)).content
# Preference(winner='A')

from typing import Literal

from pydantic import BaseModel, Field


class Preference(BaseModel):
    winner: Literal["A", "B", "tie"] = Field(..., description="Better response")
    rationale: str = Field(..., description="Why the winner is better")

instructions = """\
Compare the two responses on these criteria, in priority order:
1. Correctness - is the information accurate
2. Completeness - does it fully answer the prompt
3. Clarity - is it easy to follow

Return the response that wins on the highest-priority criterion where
they differ. Use 'tie' only if they are equal on all three.
"""