Thread injection in trace scorers (js)

js/index.ts

@@ -40,3 +40,11 @@ export * from "./ragas";
 export * from "./value";
 export { Evaluators } from "./manifest";
 export { makePartial, ScorerWithPartial } from "./partial";
+export {
+  computeThreadTemplateVars,
+  formatMessageArrayAsText,
+  isLLMMessageArray,
+  isRoleContentMessage,
+  type LLMMessage,
+  type ThreadTemplateVars,
+} from "./thread-utils";

js/llm.ts

@@ -8,6 +8,42 @@ import {
 } from "openai/resources";
 import { makePartial, ScorerWithPartial } from "./partial";
 import { renderMessages } from "./render-messages";
+import {
+  computeThreadTemplateVars,
+  type ThreadTemplateVars,
+} from "./thread-utils";
+
+/**
+ * Minimal interface for a Trace object that can provide thread data.
+ * This is compatible with the Trace interface from the braintrust SDK.
+ */
+export interface TraceForScorer {
+  getThread(options?: { preprocessor?: string }): Promise<unknown[]>;
+}
+
+// Thread-related template variable names that require preprocessor invocation
+export const THREAD_VARIABLE_NAMES = [
+  "thread",
+  "thread_count",
+  "first_message",
+  "last_message",
+  "user_messages",
+  "assistant_messages",
+  "human_ai_pairs",
+];
+
+// Pattern to match thread variables in template syntax: {{thread, {{ thread, {%...thread, etc.
+export const THREAD_VARIABLE_PATTERN = new RegExp(
+  `\\{[\\{%]\\s*(${THREAD_VARIABLE_NAMES.join("|")})`,
+);
+
+/**
+ * Check if a template string might use thread-related template variables.
+ * This is a heuristic - looks for variable names after {{ or {% syntax.
+ */
+export function templateUsesThreadVariables(template: string): boolean {
+  return THREAD_VARIABLE_PATTERN.test(template);
+}
 
 const NO_COT_SUFFIX =
   "Answer the question by calling `select_choice` with a single choice from {{__choices}}.";
@@ -193,6 +229,12 @@ function parseResponse(
 export type LLMClassifierArgs<RenderArgs> = {
   model?: string;
   useCoT?: boolean;
+  /**
+   * Optional trace object for multi-turn scoring.
+   * When provided, thread template variables (thread_text, thread_count, etc.)
+   * are automatically computed and made available in the template.
+   */
+  trace?: TraceForScorer;
 } & LLMArgs &
   RenderArgs;
 
@@ -217,6 +259,21 @@ export function LLMClassifierFromTemplate<RenderArgs>({
   ) => {
     const useCoT = runtimeArgs.useCoT ?? useCoTArg ?? true;
 
+    // Compute thread template variables if trace is available AND the template uses them.
+    // These become available in templates as {{thread}}, {{thread_count}}, etc.
+    // Note: {{thread}} automatically renders as human-readable text via smart escape.
+    // Only call getThread() if the template actually uses thread variables to avoid
+    // creating unnecessary preprocessor spans.
+    let threadVars: Record<string, unknown> = {};
+    if (runtimeArgs.trace && templateUsesThreadVariables(promptTemplate)) {
+      const thread = await runtimeArgs.trace.getThread();
+      const computed = computeThreadTemplateVars(thread);
+      // Build threadVars from THREAD_VARIABLE_NAMES to keep in sync with the pattern
+      for (const name of THREAD_VARIABLE_NAMES) {
+        threadVars[name] = computed[name as keyof ThreadTemplateVars];
+      }
+    }
+
     const prompt =
       promptTemplate + "\n" + (useCoT ? COT_SUFFIX : NO_COT_SUFFIX);
 
@@ -228,7 +285,8 @@ export function LLMClassifierFromTemplate<RenderArgs>({
       },
     ];
 
-    return await OpenAIClassifier({
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const classifierArgs: any = {
       name,
       messages,
       choiceScores,
@@ -237,12 +295,15 @@ export function LLMClassifierFromTemplate<RenderArgs>({
       maxTokens,
       temperature,
       __choices: choiceStrings,
+      // Thread template vars come first so explicit args can override
+      ...threadVars,
       ...runtimeArgs,
-
       // Since the logic is a bit funky for computing this, include
       // it at the end to prevent overrides
       useCoT,
-    });
+    };
+
+    return await OpenAIClassifier(classifierArgs);
   };
   Object.defineProperty(ret, "name", {
     value: name,

js/render-messages.test.ts

@@ -37,3 +37,149 @@ describe("renderMessages", () => {
     expect(rendered[0].content).toBe("");
   });
 });
+
+describe("renderMessages with thread variables", () => {
+  const sampleThread = [
+    { role: "user", content: "Hello, how are you?" },
+    { role: "assistant", content: "I am doing well, thank you!" },
+    { role: "user", content: "What is the weather like?" },
+    { role: "assistant", content: "It is sunny and warm today." },
+  ];
+
+  it("{{thread}} renders full conversation as human-readable text", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "{{thread}}" },
+    ];
+    const rendered = renderMessages(messages, { thread: sampleThread });
+
+    expect(rendered[0].content).toContain("User:");
+    expect(rendered[0].content).toContain("Hello, how are you?");
+    expect(rendered[0].content).toContain("Assistant:");
+    expect(rendered[0].content).toContain("I am doing well, thank you!");
+    expect(rendered[0].content).toContain("What is the weather like?");
+    expect(rendered[0].content).toContain("It is sunny and warm today.");
+  });
+
+  it("{{thread.0}} renders first message as formatted text", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "First message: {{thread.0}}" },
+    ];
+    const rendered = renderMessages(messages, { thread: sampleThread });
+
+    expect(rendered[0].content).toBe(
+      "First message: user: Hello, how are you?",
+    );
+  });
+
+  it("{{thread.1}} renders second message as formatted text", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "Second message: {{thread.1}}" },
+    ];
+    const rendered = renderMessages(messages, { thread: sampleThread });
+
+    expect(rendered[0].content).toBe(
+      "Second message: assistant: I am doing well, thank you!",
+    );
+  });
+
+  it("{{first_message}} renders single message formatted", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "First: {{first_message}}" },
+    ];
+    const rendered = renderMessages(messages, {
+      first_message: sampleThread[0],
+    });
+
+    expect(rendered[0].content).toBe("First: user: Hello, how are you?");
+  });
+
+  it("{{thread_count}} renders as a number", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "Count: {{thread_count}}" },
+    ];
+    const rendered = renderMessages(messages, { thread_count: 4 });
+
+    expect(rendered[0].content).toBe("Count: 4");
+  });
+
+  it("{{user_messages}} renders array of user messages", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "Users said: {{user_messages}}" },
+    ];
+    const userMessages = sampleThread.filter((m) => m.role === "user");
+    const rendered = renderMessages(messages, { user_messages: userMessages });
+
+    expect(rendered[0].content).toContain("User:");
+    expect(rendered[0].content).toContain("Hello, how are you?");
+    expect(rendered[0].content).toContain("What is the weather like?");
+    expect(rendered[0].content).not.toContain("Assistant:");
+  });
+
+  it("{{user_messages.0}} renders first user message", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "First user: {{user_messages.0}}" },
+    ];
+    const userMessages = sampleThread.filter((m) => m.role === "user");
+    const rendered = renderMessages(messages, { user_messages: userMessages });
+
+    expect(rendered[0].content).toBe("First user: user: Hello, how are you?");
+  });
+
+  it("{{human_ai_pairs}} renders array of paired turns", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "Pairs: {{human_ai_pairs}}" },
+    ];
+    const pairs = [
+      { human: sampleThread[0], assistant: sampleThread[1] },
+      { human: sampleThread[2], assistant: sampleThread[3] },
+    ];
+    const rendered = renderMessages(messages, { human_ai_pairs: pairs });
+
+    // Pairs are objects, so they get JSON stringified
+    expect(rendered[0].content).toContain("Pairs:");
+    expect(rendered[0].content).toContain("human");
+    expect(rendered[0].content).toContain("assistant");
+  });
+
+  it("{{#thread}}...{{/thread}} iterates over messages", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      {
+        role: "user",
+        content: "Messages:{{#thread}}\n- {{role}}: {{content}}{{/thread}}",
+      },
+    ];
+    const rendered = renderMessages(messages, { thread: sampleThread });
+
+    expect(rendered[0].content).toBe(
+      "Messages:\n- user: Hello, how are you?\n- assistant: I am doing well, thank you!\n- user: What is the weather like?\n- assistant: It is sunny and warm today.",
+    );
+  });
+
+  it("handles empty thread gracefully", () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "Thread: {{thread}}" },
+    ];
+    const rendered = renderMessages(messages, { thread: [] });
+
+    expect(rendered[0].content).toBe("Thread: ");
+  });
+
+  it("handles thread with complex content (arrays)", () => {
+    const complexThread = [
+      {
+        role: "user",
+        content: [{ type: "text", text: "Hello with structured content" }],
+      },
+      { role: "assistant", content: "Simple response" },
+    ];
+    const messages: ChatCompletionMessageParam[] = [
+      { role: "user", content: "{{thread}}" },
+    ];
+    const rendered = renderMessages(messages, { thread: complexThread });
+
+    expect(rendered[0].content).toContain("User:");
+    expect(rendered[0].content).toContain("Hello with structured content");
+    expect(rendered[0].content).toContain("Assistant:");
+    expect(rendered[0].content).toContain("Simple response");
+  });
+});

js/render-messages.ts

@@ -1,5 +1,32 @@
 import mustache from "mustache";
 import { ChatCompletionMessageParam } from "openai/resources";
+import {
+  isLLMMessageArray,
+  isRoleContentMessage,
+  formatMessageArrayAsText,
+} from "./thread-utils";
+
+/**
+ * Smart escape function for Mustache templates.
+ * - Strings are passed through unchanged
+ * - LLM message arrays are formatted as human-readable text
+ * - Single messages are formatted with role and content
+ * - Other values are JSON-stringified
+ */
+function escapeValue(v: unknown): string {
+  if (typeof v === "string") {
+    return v;
+  }
+  if (isLLMMessageArray(v)) {
+    return formatMessageArrayAsText(v);
+  }
+  if (isRoleContentMessage(v)) {
+    const content =
+      typeof v.content === "string" ? v.content : JSON.stringify(v.content);
+    return `${v.role}: ${content}`;
+  }
+  return JSON.stringify(v);
+}
 
 export function renderMessages(
   messages: ChatCompletionMessageParam[],
@@ -9,8 +36,7 @@ export function renderMessages(
     ...m,
     content: m.content
       ? mustache.render(m.content as string, renderArgs, undefined, {
-          escape: (v: unknown) =>
-            typeof v === "string" ? v : JSON.stringify(v),
+          escape: escapeValue,
         })
       : "",
   }));