Custom code scorers and classifiers

import { Eval, type Scorer } from "braintrust";
import OpenAI from "openai";

const client = new OpenAI();

const DATASET = [
  {
    input: "What is 2+2?",
    expected: "4",
  },
  {
    input: "What is the capital of France?",
    expected: "Paris",
  },
];

async function task(input: string): Promise<string> {
  const response = await client.responses.create({
    model: "gpt-5-mini",
    input: [
      { role: "user", content: input },
    ],
  });
  return response.output_text ?? "";
}

const equalityScorer: Scorer = ({ output, expected }) => {
  if (!expected) return null;
  const matches = output === expected;
  return {
    name: "Equality",
    score: matches ? 1 : 0,
    metadata: { exact_match: matches },
  };
};

const containsScorer: Scorer = ({ output, expected }) => {
  if (!expected) return null;
  const contains = output.toLowerCase().includes(expected.toLowerCase());
  return {
    name: "Contains expected",
    score: contains ? 1 : 0,
  };
};

Eval("Custom Code Scorer Example", {
  data: DATASET,
  task,
  scores: [equalityScorer, containsScorer],
});

import { Eval, wrapOpenAI, wrapTraced, type Scorer } from "braintrust";
import OpenAI from "openai";

const client = wrapOpenAI(new OpenAI());

const SUPPORT_DATASET = [
  { input: "My order hasn't arrived yet. Order #12345." },
  { input: "I need help resetting my password." },
];

const callLLM = wrapTraced(async function callLLM(messages: Array<{ role: string; content: string }>) {
  const response = await client.chat.completions.create({
    model: "gpt-5-mini",
    messages,
  });
  return response.choices[0].message.content || "";
});

async function supportTask(input: string): Promise<string> {
  const messages: Array<{ role: string; content: string }> = [
    { role: "system", content: "You are a helpful customer support agent." }
  ];

  messages.push({ role: "user", content: input });
  const response1 = await callLLM(messages);
  messages.push({ role: "assistant", content: response1 });

  messages.push({ role: "user", content: "Can you provide more details?" });
  const response2 = await callLLM(messages);
  messages.push({ role: "assistant", content: response2 });

  messages.push({ role: "user", content: "Thank you for your help!" });
  const response3 = await callLLM(messages);

  return response3;
}

const politenessScorer: Scorer = async ({ trace }) => {
  if (!trace) return 0;

  const thread = await trace.getThread();
  const lastAssistantMsg = thread.reverse().find(msg => msg.role === "assistant");
  const content = lastAssistantMsg?.content?.toLowerCase() || "";

  const politeWords = ["welcome", "glad", "happy", "pleasure", "thank"];
  const isPolite = politeWords.some(word => content.includes(word));

  return {
    name: "Politeness",
    score: isPolite ? 1 : 0,
    metadata: { checked_message_preview: content.slice(0, 80) },
  };
};

const efficiencyScorer: Scorer = async ({ trace }) => {
  if (!trace) return 0;

  const llmSpans = await trace.getSpans({ spanType: ["llm"] });
  const isEfficient = llmSpans.length >= 3 && llmSpans.length <= 5;

  return {
    name: "Efficiency",
    score: isEfficient ? 1 : 0,
    metadata: { llm_calls: llmSpans.length },
  };
};

Eval("Support Quality", {
  data: SUPPORT_DATASET,
  task: supportTask,
  scores: [politenessScorer, efficiencyScorer],
});