LLM-as-a-judge scorers and classifiers

import { Eval } from "braintrust";
import { LLMClassifierFromTemplate } from "autoevals";
import OpenAI from "openai";

const client = new OpenAI();

const MOVIE_DATASET = [
  {
    input:
      "A detective investigates a series of murders based on the seven deadly sins.",
    expected: "Se7en",
  },
  {
    input:
      "A thief who steals corporate secrets through the use of dream-sharing technology is given the inverse task of planting an idea into the mind of a C.E.O.",
    expected: "Inception",
  },
];

async function task(input: string): Promise<string> {
  const response = await client.responses.create({
    model: "gpt-5-mini",
    input: [
      {
        role: "system",
        content:
          "Based on the following description, identify the movie. Reply with only the movie title.",
      },
      { role: "user", content: input },
    ],
  });
  return response.output_text ?? "";
}

const correctnessScorer = LLMClassifierFromTemplate({
  name: "Correctness",
  promptTemplate: `You are evaluating a movie-identification task.

Output (model's answer): {{output}}
Expected (correct movie): {{expected}}

Does the output correctly identify the same movie as the expected answer?
Consider alternate titles (e.g. "Harry Potter 1" vs "Harry Potter and the Sorcerer's Stone") as correct.

Return only "correct" if the output is the right movie (exact or equivalent title).
Return only "incorrect" otherwise.`,
  choiceScores: {
    correct: 1,
    incorrect: 0,
  },
  useCoT: true,
});

Eval("Movie Matcher", {
  data: MOVIE_DATASET,
  task,
  scores: [correctnessScorer],
});

import { Eval, wrapOpenAI, wrapTraced, type Scorer } from "braintrust";
import OpenAI from "openai";

const client = new OpenAI();
const wrappedClient = wrapOpenAI(new OpenAI());

const SUPPORT_DATASET = [
  { input: "My order hasn't arrived yet. Order #12345." },
  { input: "I need help resetting my password." },
];

const callLLM = wrapTraced(async function callLLM(messages: Array<{ role: string; content: string }>) {
  const response = await wrappedClient.chat.completions.create({
    model: "gpt-5-mini",
    messages,
  });
  return response.choices[0].message.content || "";
});

async function supportTask(input: string): Promise<string> {
  const messages: Array<{ role: string; content: string }> = [
    { role: "system", content: "You are a helpful customer support agent." }
  ];

  messages.push({ role: "user", content: input });
  const response1 = await callLLM(messages);
  messages.push({ role: "assistant", content: response1 });

  messages.push({ role: "user", content: "Can you provide more details?" });
  const response2 = await callLLM(messages);
  messages.push({ role: "assistant", content: response2 });

  messages.push({ role: "user", content: "Thank you for your help!" });
  const response3 = await callLLM(messages);

  return response3;
}

const conversationCoherence: Scorer = async ({ trace }) => {
  if (!trace) return null;

  const thread = await trace.getThread();
  const threadText = thread
    .map(msg => `${msg.role}: ${msg.content}`)
    .join("\n\n");

  const response = await client.responses.create({
    model: "gpt-5-mini",
    input: [
      {
        role: "user",
        content: `Evaluate the coherence of this customer support conversation:

${threadText}

Rate the conversation coherence:
- "A" for highly coherent with natural flow and consistent context
- "B" for mostly coherent with minor gaps or context issues
- "C" for incoherent, disjointed, or lost context

Return only the letter (A, B, or C).`,
      },
    ],
  });

  const rating = response.output_text?.trim().toUpperCase() || "C";
  const choiceScores = { A: 1, B: 0.6, C: 0 };
  const score = choiceScores[rating as keyof typeof choiceScores] ?? 0;

  return {
    name: "Conversation coherence",
    score,
    metadata: { rating, thread_length: thread.length },
  };
};

Eval("Support Conversation Quality", {
  data: SUPPORT_DATASET,
  task: supportTask,
  scores: [conversationCoherence],
});