{ "$schema": "https://ui.shadcn.com/schema/registry-item.json", "name": "trace-eval-agent", "title": "Trace Eval Agent", "description": "A live research chat that surfaces execution trace, deterministic eval checks, and an LLM judge over the same run.", "dependencies": [ "@ai-sdk/openai", "@ai-sdk/react", "@base-ui/react", "@phosphor-icons/react", "ai", "class-variance-authority", "lucide-react", "zod" ], "registryDependencies": [ "utils", "badge", "breadcrumb", "button", "textarea", "https://elements.ai-sdk.dev/api/registry/conversation.json", "https://elements.ai-sdk.dev/api/registry/message.json", "https://elements.ai-sdk.dev/api/registry/reasoning.json", "https://elements.ai-sdk.dev/api/registry/shimmer.json", "https://elements.ai-sdk.dev/api/registry/sources.json", "https://elements.ai-sdk.dev/api/registry/task.json", "https://elements.ai-sdk.dev/api/registry/test-results.json", "https://elements.ai-sdk.dev/api/registry/tool.json" ], "files": [ { "path": "registry/trace-eval-agent/app/demos/trace-eval-agent/page.tsx", "content": "import { TraceEvalAgentScreen } from \"@/components/trace-eval-agent/trace-eval-agent-screen\";\n\nexport default function TraceEvalAgentPage() {\n return ;\n}\n", "type": "registry:page", "target": "app/demos/trace-eval-agent/page.tsx" }, { "path": "registry/trace-eval-agent/app/api/demos/trace-eval-agent/route.ts", "content": "import { handleTraceEvalAgentRequest } from \"@/lib/trace-eval-agent/server/runtime\";\n\nexport const runtime = \"nodejs\";\n\nexport function POST(request: Request) {\n return handleTraceEvalAgentRequest(request);\n}\n", "type": "registry:page", "target": "app/api/demos/trace-eval-agent/route.ts" }, { "path": "registry/trace-eval-agent/app/api/demos/trace-eval-agent/evaluate/route.ts", "content": "import { handleTraceEvalAgentEvaluationRequest } from \"@/lib/trace-eval-agent/server/evaluation\";\n\nexport const runtime = \"nodejs\";\n\nexport function POST(request: Request) {\n return handleTraceEvalAgentEvaluationRequest(request);\n}\n", "type": "registry:page", "target": "app/api/demos/trace-eval-agent/evaluate/route.ts" }, { "path": "registry/trace-eval-agent/app/api/demos/trace-eval-agent/evaluate/stream/route.ts", "content": "import { handleTraceEvalAgentEvaluationStreamRequest } from \"@/lib/trace-eval-agent/server/evaluation\";\n\nexport const runtime = \"nodejs\";\n\nexport function POST(request: Request) {\n return handleTraceEvalAgentEvaluationStreamRequest(request);\n}\n", "type": "registry:page", "target": "app/api/demos/trace-eval-agent/evaluate/stream/route.ts" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-screen.tsx", "content": "import { Badge } from \"@/components/ui/badge\";\nimport {\n Breadcrumb,\n BreadcrumbItem,\n BreadcrumbLink,\n BreadcrumbList,\n BreadcrumbPage,\n BreadcrumbSeparator,\n} from \"@/components/ui/breadcrumb\";\nimport { ArrowLeft } from \"lucide-react\";\n\nimport { getTraceEvalAgentRuntimeState } from \"@/lib/trace-eval-agent/server/runtime\";\nimport { TraceEvalAgentWorkspace } from \"./trace-eval-agent-workspace\";\n\nexport function TraceEvalAgentScreen() {\n const runtimeState = getTraceEvalAgentRuntimeState();\n\n return (\n
\n
\n
\n
\n \n \n \n \n \n Demo\n \n \n \n /\n \n \n \n Trace and Eval Agent\n \n \n \n \n

\n Live research chat with session trace and evaluation\n

\n

\n This slice keeps a real research conversation at the top, then\n scores the same session below with execution trace, source\n coverage, answer-shape checks, and expected-path evaluation.\n

\n
\n\n
\n {runtimeState.statusLabel}\n {runtimeState.chatModel}\n
\n
\n\n
\n \n
\n
\n
\n );\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-screen.tsx" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-workspace.tsx", "content": "\"use client\";\n\nimport { MagnifyingGlassIcon, StopIcon } from \"@phosphor-icons/react\";\nimport {\n Conversation,\n ConversationContent,\n ConversationEmptyState,\n ConversationScrollButton,\n} from \"@/components/ai-elements/conversation\";\nimport {\n Message,\n MessageContent,\n MessageResponse,\n} from \"@/components/ai-elements/message\";\nimport {\n PromptInput,\n PromptInputBody,\n PromptInputFooter,\n PromptInputSubmit,\n PromptInputTextarea,\n} from \"@/components/ai-elements/prompt-input\";\nimport { Badge } from \"@/components/ui/badge\";\nimport { Button } from \"@/components/ui/button\";\nimport { cn } from \"@/lib/utils\";\nimport { useDeferredValue } from \"react\";\nimport { classifyTraceEvalRunOutcome } from \"@/lib/trace-eval-agent/model/trace-eval-run-outcome\";\nimport { buildTraceEvalRunRecord } from \"@/lib/trace-eval-agent/model/trace-eval-run-record\";\nimport { buildTraceEvalSnapshotFromRunRecord } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\nimport type { TraceEvalAgentRuntimeState } from \"@/lib/trace-eval-agent/server/runtime\";\nimport { TraceEvalAgentAssistantMessage } from \"./trace-eval-agent-assistant-message\";\nimport { TraceEvalAgentEvalPanel } from \"./trace-eval-agent-eval-panel\";\nimport {\n getTextContent,\n traceEvalAgentSamplePrompts,\n} from \"./trace-eval-agent-model\";\nimport { TraceEvalAgentRuntimeSidebar } from \"./trace-eval-agent-runtime-sidebar\";\nimport { TraceEvalAgentTracePanel } from \"./trace-eval-agent-trace-panel\";\nimport { useTraceEvalAgentChat } from \"./use-trace-eval-agent-chat\";\nimport { useTraceEvalJudge } from \"./use-trace-eval-judge\";\n\ninterface TraceEvalAgentWorkspaceProps {\n runtimeState: TraceEvalAgentRuntimeState;\n}\n\nexport function TraceEvalAgentWorkspace({\n runtimeState,\n}: TraceEvalAgentWorkspaceProps) {\n const {\n clearError,\n error,\n hasMessages,\n isBusy,\n messages,\n regenerate,\n sendMessage,\n status,\n stop,\n } = useTraceEvalAgentChat();\n const deferredMessages = useDeferredValue(messages);\n const runRecord = buildTraceEvalRunRecord(deferredMessages, isBusy);\n const snapshot = buildTraceEvalSnapshotFromRunRecord(runRecord);\n const runOutcome = classifyTraceEvalRunOutcome({\n error,\n record: runRecord,\n });\n const judge = useTraceEvalJudge({\n judgeModel: runtimeState.chatModel,\n outcome: runOutcome,\n snapshot,\n });\n\n function resetTransientFailure() {\n clearError();\n }\n\n async function handleSendMessage(text: string) {\n resetTransientFailure();\n await sendMessage({ text });\n }\n\n async function handleRetry() {\n resetTransientFailure();\n await regenerate();\n }\n\n return (\n
\n
\n
\n {runtimeState.isChatAvailable ? null : (\n
\n {runtimeState.setupMessage}\n
\n )}\n\n {runOutcome.kind === \"failed-run\" ? (\n
\n

{runOutcome.detail}

\n {\n handleRetry();\n }}\n size=\"sm\"\n type=\"button\"\n variant=\"outline\"\n >\n Retry\n \n
\n ) : null}\n\n \n \n {hasMessages ? (\n messages.map((message, index) => (\n \n \n {message.role === \"assistant\" ? (\n \n ) : (\n \n {getTextContent(message)}\n \n )}\n \n \n ))\n ) : (\n }\n title=\"Research agent is ready\"\n />\n )}\n \n \n \n\n
\n
\n {\n handleSendMessage(text);\n }}\n >\n \n \n \n \n
\n ResearchAgent\n Trace + Eval\n {runtimeState.chatModel}\n
\n
\n {isBusy ? (\n \n \n Stop\n \n ) : null}\n {hasMessages ? (\n {\n handleRetry();\n }}\n size=\"sm\"\n type=\"button\"\n variant=\"outline\"\n >\n Retry\n \n ) : null}\n \n
\n \n \n\n {hasMessages ? null : (\n
\n {traceEvalAgentSamplePrompts.map((item) => (\n {\n handleSendMessage(item.prompt);\n }}\n size=\"sm\"\n type=\"button\"\n variant=\"outline\"\n >\n \n {item.label}\n \n ))}\n
\n )}\n
\n
\n
\n\n
\n \n \n
\n
\n\n {\n handleSendMessage(prompt);\n }}\n runOutcome={runOutcome}\n runtimeState={runtimeState}\n snapshot={snapshot}\n />\n
\n );\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-workspace.tsx" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-assistant-message.tsx", "content": "\"use client\";\n\nimport { MessageResponse } from \"@/components/ai-elements/message\";\nimport {\n Reasoning,\n ReasoningContent,\n ReasoningTrigger,\n} from \"@/components/ai-elements/reasoning\";\nimport { Shimmer } from \"@/components/ai-elements/shimmer\";\nimport {\n Source,\n Sources,\n SourcesContent,\n SourcesTrigger,\n} from \"@/components/ai-elements/sources\";\nimport {\n Tool,\n ToolContent,\n ToolHeader,\n ToolInput,\n ToolOutput,\n} from \"@/components/ai-elements/tool\";\nimport type { UIMessage } from \"ai\";\n\nimport { buildTraceEvalSnapshot } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\nimport {\n getReasoningText,\n getTextContent,\n getToolDisplayName,\n getToolParts,\n hasVisibleToolPayload,\n} from \"./trace-eval-agent-model\";\n\ninterface TraceEvalAgentAssistantMessageProps {\n isStreaming: boolean;\n message: UIMessage;\n}\n\nexport function TraceEvalAgentAssistantMessage({\n isStreaming,\n message,\n}: TraceEvalAgentAssistantMessageProps) {\n const reasoning = getReasoningText(message);\n const answer = getTextContent(message);\n const toolParts = getToolParts(message).filter(hasVisibleToolPayload);\n const hasReasoning = reasoning.length > 0;\n const hasAnswer = answer.length > 0;\n const lastPart = message.parts.at(-1);\n const isReasoningStreaming = isStreaming && lastPart?.type === \"reasoning\";\n const showThinking = isStreaming && !hasReasoning && !hasAnswer;\n const snapshot = buildTraceEvalSnapshot([message], isStreaming);\n\n return (\n
\n {hasReasoning ? (\n \n \n {reasoning}\n \n ) : null}\n\n {toolParts.map((part) => (\n \n {part.type === \"dynamic-tool\" ? (\n \n ) : (\n \n )}\n \n {part.input ? : null}\n \n \n \n ))}\n\n {snapshot.sources.length > 0 ? (\n \n \n \n {snapshot.sources.map((source) => (\n \n ))}\n \n \n ) : null}\n\n {hasAnswer ? {answer} : null}\n {showThinking ? Thinking... : null}\n
\n );\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-assistant-message.tsx" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-trace-panel.tsx", "content": "\"use client\";\n\nimport {\n CheckCircleIcon,\n CircleIcon,\n CircleNotchIcon,\n XCircleIcon,\n} from \"@phosphor-icons/react\";\nimport {\n Task,\n TaskContent,\n TaskItem,\n TaskTrigger,\n} from \"@/components/ai-elements/task\";\nimport { Badge } from \"@/components/ui/badge\";\nimport { cn } from \"@/lib/utils\";\n\nimport type { TraceEvalRunOutcome } from \"@/lib/trace-eval-agent/model/trace-eval-run-outcome\";\nimport type { TraceEvalSnapshot } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\n\nconst traceStatusStyles = {\n failed: \"text-red-600\",\n passed: \"text-green-600\",\n pending: \"text-muted-foreground\",\n running: \"text-blue-600\",\n} as const;\n\nconst traceStatusIcons = {\n failed: XCircleIcon,\n passed: CheckCircleIcon,\n pending: CircleIcon,\n running: CircleNotchIcon,\n} as const;\n\nexport function TraceEvalAgentTracePanel({\n runOutcome,\n snapshot,\n}: {\n runOutcome: TraceEvalRunOutcome;\n snapshot: TraceEvalSnapshot;\n}) {\n if (runOutcome.kind === \"failed-run\") {\n return (\n
\n
\n
\n

Trace

\n

\n Current conversation execution path\n

\n
\n failed\n
\n\n \n \n \n \n
\n \n
\n

\n Run failed before trace evaluation\n

\n

\n {runOutcome.detail}\n

\n
\n
\n \n \n \n
\n );\n }\n\n return (\n
\n
\n
\n

Trace

\n

\n Current conversation execution path\n

\n
\n {snapshot.status}\n
\n\n \n \n \n {snapshot.trace.map((item) => {\n const Icon = traceStatusIcons[item.status];\n\n return (\n \n
\n
\n \n
\n

\n {item.title}\n

\n

\n {item.detail}\n

\n
\n
\n {item.metric ? (\n \n {item.metric}\n \n ) : null}\n
\n \n );\n })}\n \n \n
\n );\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-trace-panel.tsx" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-eval-panel.tsx", "content": "\"use client\";\n\nimport {\n Test,\n TestName,\n TestResults,\n TestResultsContent,\n TestResultsDuration,\n TestResultsHeader,\n TestResultsProgress,\n TestResultsSummary,\n TestStatus,\n TestSuite,\n TestSuiteContent,\n TestSuiteName,\n TestSuiteStats,\n} from \"@/components/ai-elements/test-results\";\n\nimport type { TraceEvalRunOutcome } from \"@/lib/trace-eval-agent/model/trace-eval-run-outcome\";\nimport type { TraceEvalSnapshot } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\nimport type { TraceEvalJudgeViewState } from \"./use-trace-eval-judge\";\n\nexport function TraceEvalAgentEvalPanel({\n judge,\n runOutcome,\n snapshot,\n}: {\n judge: TraceEvalJudgeViewState;\n runOutcome: TraceEvalRunOutcome;\n snapshot: TraceEvalSnapshot;\n}) {\n if (runOutcome.kind === \"empty\") {\n return (\n \n \n
\n

Eval

\n

\n Deterministic gate and LLM judge\n

\n
\n \n\n \n

\n {runOutcome.detail}\n

\n \n \n );\n }\n\n if (runOutcome.kind === \"skipped\") {\n return (\n \n \n
\n

Eval

\n

\n Deterministic gate and LLM judge\n

\n
\n
\n \n \n
\n \n\n \n \n
\n \n \n
\n \n \n \n
\n \n

\n {runOutcome.detail}\n

\n
\n \n \n \n \n \n );\n }\n\n if (runOutcome.kind === \"failed-run\") {\n return (\n \n \n
\n

Eval

\n

\n Deterministic gate and LLM judge\n

\n
\n
\n \n \n
\n \n\n \n \n
\n \n \n
\n \n \n \n
\n \n

\n {runOutcome.detail}\n

\n
\n \n \n \n \n \n );\n }\n\n let suiteStatus: \"failed\" | \"passed\" | \"running\" = \"passed\";\n\n if (snapshot.summary.failed > 0) {\n suiteStatus = \"failed\";\n } else if (snapshot.status === \"running\") {\n suiteStatus = \"running\";\n }\n\n const judgeSuiteStatus = getJudgeSuiteStatus(judge);\n\n return (\n \n \n
\n

Eval

\n

\n Deterministic gate and LLM judge\n

\n
\n
\n \n \n
\n \n\n \n
\n
\n Gate score\n {formatScore(snapshot.score)}\n
\n {judge.result ? (\n
\n Judge score\n \n {formatScore(judge.result.overallScore)}\n \n
\n ) : null}\n {judge.status === \"running\" ? (\n
\n
\n Judge progress\n \n {formatScore(getJudgeProgressValue(judge))}\n \n
\n
\n \n
\n
\n ) : null}\n \n
\n\n \n
\n \n \n
\n \n {snapshot.checks.map((check) => (\n \n \n
\n \n

\n {check.detail}\n

\n
\n \n ))}\n \n \n\n \n
\n \n {judge.result ? (\n dimension.score >= 0.8\n ).length\n }\n skipped={\n judge.result.dimensions.filter(\n (dimension) =>\n dimension.score >= 0.6 && dimension.score < 0.8\n ).length\n }\n />\n ) : null}\n
\n \n {renderJudgeContent(judge, runOutcome, snapshot)}\n \n \n \n \n );\n}\n\nfunction getJudgeSuiteStatus(\n judge: TraceEvalJudgeViewState\n): \"failed\" | \"passed\" | \"running\" | \"skipped\" {\n if (judge.status === \"running\") {\n return \"running\";\n }\n\n if (judge.status === \"idle\") {\n return \"skipped\";\n }\n\n if (judge.status === \"failed\") {\n return \"failed\";\n }\n\n if (\n judge.result &&\n (judge.result.overallScore < 0.8 ||\n judge.result.deterministicFailures.length > 0)\n ) {\n return \"failed\";\n }\n\n return \"passed\";\n}\n\nfunction getDimensionStatus(score: number): \"failed\" | \"passed\" | \"skipped\" {\n if (score >= 0.8) {\n return \"passed\";\n }\n\n return score >= 0.6 ? \"skipped\" : \"failed\";\n}\n\nfunction formatScore(score: number) {\n return `${Math.round(score * 100)}%`;\n}\n\nfunction getJudgeProgressValue(judge: TraceEvalJudgeViewState) {\n return judge.progress?.progress ?? 0;\n}\n\nfunction formatJudgeElapsed(judge: TraceEvalJudgeViewState) {\n if (judge.elapsedMs === null) {\n return \"Elapsed time is still starting.\";\n }\n\n return `Elapsed ${Math.max(0.1, judge.elapsedMs / 1000).toFixed(1)}s`;\n}\n\nfunction formatJudgeStageLabel(\n step: \"starting\" | \"summary\" | \"dimensions\" | \"finalizing\"\n) {\n switch (step) {\n case \"starting\":\n return \"Starting\";\n case \"summary\":\n return \"Streaming summary\";\n case \"dimensions\":\n return \"Scoring dimensions\";\n case \"finalizing\":\n return \"Finalizing recommendation\";\n default:\n return \"Judge stage\";\n }\n}\n\nfunction renderJudgeContent(\n judge: TraceEvalJudgeViewState,\n runOutcome: TraceEvalRunOutcome,\n snapshot: TraceEvalSnapshot\n) {\n if (judge.status === \"idle\") {\n if (runOutcome.kind === \"running\") {\n return (\n \n \n
\n \n

\n The run is still streaming. The judge starts after the research\n answer completes.\n

\n
\n \n );\n }\n\n if (snapshot.latestPrompt && !snapshot.latestAnswer) {\n return (\n \n \n
\n \n

\n The judge only runs after a completed research answer is\n available. Retry the run to generate an answer before scoring it.\n

\n
\n \n );\n }\n\n return (\n \n \n
\n \n

\n The judge starts after the research answer finishes.\n

\n
\n \n );\n }\n\n if (judge.status === \"running\") {\n return (\n <>\n \n \n
\n \n

\n {judge.progress?.message ??\n \"Scoring final-answer quality and full-run quality.\"}\n

\n

\n {formatJudgeElapsed(judge)}\n

\n
\n \n\n {judge.partial?.summary ? (\n \n \n
\n \n

\n {judge.partial.summary}\n

\n
\n \n ) : null}\n\n {judge.partial?.dimensions?.map((dimension) =>\n dimension ? (\n \n \n
\n \n

\n {dimension.rationale ?? \"Scoring dimension.\"}\n

\n
\n \n ) : null\n )}\n \n );\n }\n\n if (judge.status === \"failed\") {\n return (\n \n \n
\n \n

{judge.error}

\n
\n \n );\n }\n\n if (!judge.result) {\n return null;\n }\n\n return (\n <>\n \n \n
\n \n

\n {judge.result.summary}\n

\n

\n {judge.result.rationale}\n

\n
\n \n\n {judge.result.deterministicFailures.length > 0 ? (\n \n \n
\n \n

\n {judge.result.deterministicFailures\n .map((failure) => failure.title)\n .join(\", \")}\n

\n
\n \n ) : null}\n\n {judge.result.dimensions.map((dimension) => (\n \n \n
\n \n

\n {dimension.rationale}\n

\n
\n \n ))}\n \n );\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-eval-panel.tsx" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-runtime-sidebar.tsx", "content": "\"use client\";\n\nimport { Badge } from \"@/components/ui/badge\";\nimport { Button } from \"@/components/ui/button\";\nimport type { ReactNode } from \"react\";\n\nimport type { TraceEvalRunOutcome } from \"@/lib/trace-eval-agent/model/trace-eval-run-outcome\";\nimport type { TraceEvalSnapshot } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\nimport { TRACE_EVAL_SEARCH_TOOL_NAME } from \"@/lib/trace-eval-agent/server/model\";\nimport type { TraceEvalAgentRuntimeState } from \"@/lib/trace-eval-agent/server/runtime\";\nimport {\n formatDuration,\n formatTokenCount,\n traceEvalAgentSamplePrompts,\n} from \"./trace-eval-agent-model\";\nimport type { TraceEvalJudgeViewState } from \"./use-trace-eval-judge\";\n\ninterface TraceEvalAgentRuntimeSidebarProps {\n judge: TraceEvalJudgeViewState;\n onPromptSelect: (prompt: string) => void;\n runOutcome: TraceEvalRunOutcome;\n runtimeState: TraceEvalAgentRuntimeState;\n snapshot: TraceEvalSnapshot;\n}\n\nfunction SidebarSection({\n children,\n title,\n}: {\n children: ReactNode;\n title: string;\n}) {\n return (\n
\n

{title}

\n
{children}
\n
\n );\n}\n\nexport function TraceEvalAgentRuntimeSidebar({\n judge,\n onPromptSelect,\n runOutcome,\n runtimeState,\n snapshot,\n}: TraceEvalAgentRuntimeSidebarProps) {\n return (\n \n );\n}\n\nfunction formatJudgeState(\n judge: TraceEvalJudgeViewState,\n runOutcome: TraceEvalRunOutcome\n) {\n if (judge.result) {\n return formatScore(judge.result.overallScore);\n }\n\n if (runOutcome.kind === \"skipped\") {\n return \"Skipped\";\n }\n\n if (runOutcome.kind === \"failed-run\") {\n return \"Not run\";\n }\n\n if (judge.status === \"running\") {\n return \"Running\";\n }\n\n if (judge.status === \"failed\") {\n return \"Failed\";\n }\n\n return \"Pending\";\n}\n\nfunction formatJudgeStage(judge: TraceEvalJudgeViewState) {\n if (judge.status !== \"running\") {\n return \"--\";\n }\n\n if (!judge.progress) {\n return \"Starting\";\n }\n\n return `${judge.progress.label} (${Math.round(judge.progress.progress * 100)}%)`;\n}\n\nfunction formatRunScore(runOutcome: TraceEvalRunOutcome, score: number) {\n if (runOutcome.kind === \"skipped\" || runOutcome.kind === \"failed-run\") {\n return \"--\";\n }\n\n return formatScore(score);\n}\n\nfunction formatScore(score: number) {\n return `${Math.round(score * 100)}%`;\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-runtime-sidebar.tsx" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/trace-eval-agent-model.ts", "content": "\"use client\";\n\nimport type { ToolPart } from \"@/components/ai-elements/tool\";\nimport { isReasoningUIPart, isToolUIPart, type UIMessage } from \"ai\";\n\nimport type { TraceEvalAgentMessage } from \"@/lib/trace-eval-agent/model/trace-eval-run-record\";\n\nexport interface TraceEvalAgentSamplePrompt {\n label: string;\n prompt: string;\n}\n\nexport const traceEvalAgentSamplePrompts: TraceEvalAgentSamplePrompt[] = [\n {\n label: \"Pick a tracing + eval stack for a Next.js AI agent\",\n prompt:\n \"Research the current Langfuse, Braintrust, and OpenTelemetry options for tracing and evaluating a Next.js AI agent. Use live web search, cite at least two sources, and recommend a production path.\",\n },\n {\n label: \"Compare Langfuse, Braintrust, and OpenTelemetry today\",\n prompt:\n \"Compare Langfuse, Braintrust, and OpenTelemetry for AI agent tracing and evaluation in a Next.js product team.\",\n },\n {\n label: \"When is AI Gateway web_search enough?\",\n prompt:\n \"Research the current Vercel AI Gateway search-tool story and explain when native web_search is enough versus when MCP is still needed.\",\n },\n];\n\nexport function getTextContent(message: UIMessage) {\n return message.parts\n .filter((part) => part.type === \"text\")\n .map((part) => part.text)\n .join(\"\\n\");\n}\n\nexport function getReasoningText(message: UIMessage) {\n return message.parts\n .filter(isReasoningUIPart)\n .map((part) => part.text)\n .join(\"\\n\\n\");\n}\n\nexport function getToolParts(message: UIMessage): ToolPart[] {\n return message.parts.filter(isToolUIPart);\n}\n\nexport function getToolDisplayName(part: ToolPart) {\n return part.type === \"dynamic-tool\"\n ? part.toolName\n : part.type.split(\"-\").slice(1).join(\"-\");\n}\n\nfunction hasMeaningfulToolValue(value: unknown): boolean {\n if (value === null || value === undefined) {\n return false;\n }\n\n if (typeof value === \"string\") {\n return value.trim().length > 0;\n }\n\n if (\n typeof value === \"number\" ||\n typeof value === \"boolean\" ||\n typeof value === \"bigint\"\n ) {\n return true;\n }\n\n if (Array.isArray(value)) {\n return value.some(hasMeaningfulToolValue);\n }\n\n if (typeof value === \"object\") {\n return Object.values(value).some(hasMeaningfulToolValue);\n }\n\n return false;\n}\n\nexport function hasVisibleToolPayload(part: ToolPart) {\n return Boolean(\n part.errorText ||\n hasMeaningfulToolValue(part.input) ||\n hasMeaningfulToolValue(part.output)\n );\n}\n\nexport function getLatestTraceEvalMetadata(messages: UIMessage[]) {\n const assistantMessages = messages.filter(\n (message): message is TraceEvalAgentMessage => message.role === \"assistant\"\n );\n\n return assistantMessages.at(-1)?.metadata;\n}\n\nexport function formatDuration(durationMs?: number) {\n if (durationMs === undefined) {\n return \"Pending\";\n }\n\n if (durationMs < 1000) {\n return `${durationMs}ms`;\n }\n\n return `${(durationMs / 1000).toFixed(2)}s`;\n}\n\nexport function formatTokenCount(totalTokens?: number) {\n return totalTokens === undefined ? \"Pending\" : `${totalTokens} tokens`;\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/trace-eval-agent-model.ts" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/use-trace-eval-agent-chat.ts", "content": "\"use client\";\n\nimport { Chat, useChat } from \"@ai-sdk/react\";\nimport { DefaultChatTransport } from \"ai\";\nimport { useState } from \"react\";\nimport type { TraceEvalAgentMessage } from \"@/lib/trace-eval-agent/model/trace-eval-run-record\";\n\nexport function useTraceEvalAgentChat() {\n const [chat] = useState(\n () =>\n new Chat({\n transport: new DefaultChatTransport({\n api: \"/api/demos/trace-eval-agent\",\n }),\n })\n );\n const controller = useChat({ chat });\n const hasMessages = controller.messages.length > 0;\n const isBusy =\n controller.status === \"submitted\" || controller.status === \"streaming\";\n\n return {\n ...controller,\n chat,\n hasMessages,\n isBusy,\n };\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/use-trace-eval-agent-chat.ts" }, { "path": "registry/trace-eval-agent/components/trace-eval-agent/use-trace-eval-judge.ts", "content": "\"use client\";\n\nimport { experimental_useObject as useObject } from \"@ai-sdk/react\";\nimport type { DeepPartial } from \"ai\";\nimport { useEffect, useMemo, useRef, useState } from \"react\";\n\nimport {\n completeTraceEvalJudgeResult,\n type TraceEvalJudgeResult,\n type TraceEvalJudgeStreamObject,\n traceEvalJudgeStreamSchema,\n} from \"@/lib/trace-eval-agent/model/trace-eval-judge\";\nimport {\n deriveTraceEvalJudgeProgress,\n type TraceEvalJudgeProgressState,\n} from \"@/lib/trace-eval-agent/model/trace-eval-judge-progress\";\nimport type { TraceEvalRunOutcome } from \"@/lib/trace-eval-agent/model/trace-eval-run-outcome\";\nimport type { TraceEvalSnapshot } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\n\nexport type TraceEvalJudgeViewStatus =\n | \"idle\"\n | \"running\"\n | \"complete\"\n | \"failed\";\n\nexport interface TraceEvalJudgeViewState {\n elapsedMs: number | null;\n error: string | null;\n partial: DeepPartial | null;\n progress: TraceEvalJudgeProgressState | null;\n result: TraceEvalJudgeResult | null;\n status: TraceEvalJudgeViewStatus;\n}\n\nconst invalidJudgeStreamObjectError =\n \"Trace eval judge stream ended before returning a valid structured result.\";\n\nconst initialJudgeState: TraceEvalJudgeViewState = {\n elapsedMs: null,\n error: null,\n partial: null,\n progress: null,\n result: null,\n status: \"idle\",\n};\n\nfunction buildEvaluationKey({\n outcome,\n snapshot,\n}: {\n outcome: TraceEvalRunOutcome;\n snapshot: TraceEvalSnapshot;\n}) {\n if (!outcome.shouldJudge) {\n return null;\n }\n\n return [\n snapshot.runId ?? \"no-run-id\",\n snapshot.latestPrompt,\n snapshot.latestAnswer,\n snapshot.totalTokens ?? \"no-usage\",\n snapshot.durationMs ?? \"no-duration\",\n ].join(\"\\n---\\n\");\n}\n\nfunction buildRunningJudgeState(\n partial: DeepPartial | null,\n elapsedMs: number | null\n): TraceEvalJudgeViewState {\n return {\n elapsedMs,\n error: null,\n partial,\n progress: deriveTraceEvalJudgeProgress(partial),\n result: null,\n status: \"running\",\n };\n}\n\nexport function useTraceEvalJudge({\n judgeModel,\n outcome,\n snapshot,\n}: {\n judgeModel: string;\n outcome: TraceEvalRunOutcome;\n snapshot: TraceEvalSnapshot;\n}): TraceEvalJudgeViewState {\n const evaluationKey = useMemo(\n () => buildEvaluationKey({ outcome, snapshot }),\n [outcome, snapshot]\n );\n const lastEvaluationKeyRef = useRef(null);\n const activeEvaluationRef = useRef<{\n evaluationKey: string;\n judgeModel: string;\n snapshot: TraceEvalSnapshot;\n } | null>(null);\n const startedAtRef = useRef(null);\n const latestObjectRef =\n useRef | null>(null);\n const [state, setState] =\n useState(initialJudgeState);\n const { clear, isLoading, object, stop, submit } = useObject<\n typeof traceEvalJudgeStreamSchema,\n TraceEvalJudgeStreamObject,\n { snapshot: TraceEvalSnapshot }\n >({\n api: \"/api/demos/trace-eval-agent/evaluate/stream\",\n onError(streamError) {\n setState({\n elapsedMs:\n startedAtRef.current === null\n ? null\n : Math.max(0, Date.now() - startedAtRef.current),\n error: streamError.message,\n partial: latestObjectRef.current,\n progress: latestObjectRef.current\n ? deriveTraceEvalJudgeProgress(latestObjectRef.current)\n : null,\n result: null,\n status: \"failed\",\n });\n },\n onFinish({ object: finalObject }) {\n const activeEvaluation = activeEvaluationRef.current;\n const parsedJudge = traceEvalJudgeStreamSchema.safeParse(\n finalObject ?? latestObjectRef.current\n );\n\n if (!activeEvaluation) {\n return;\n }\n\n if (!parsedJudge.success) {\n setState({\n elapsedMs:\n startedAtRef.current === null\n ? null\n : Math.max(0, Date.now() - startedAtRef.current),\n error: invalidJudgeStreamObjectError,\n partial: latestObjectRef.current,\n progress: latestObjectRef.current\n ? deriveTraceEvalJudgeProgress(latestObjectRef.current)\n : null,\n result: null,\n status: \"failed\",\n });\n return;\n }\n\n setState({\n elapsedMs:\n startedAtRef.current === null\n ? null\n : Math.max(0, Date.now() - startedAtRef.current),\n error: null,\n partial: parsedJudge.data,\n progress: null,\n result: completeTraceEvalJudgeResult({\n deterministicFailures: activeEvaluation.snapshot.checks.filter(\n (check) => check.status === \"failed\"\n ),\n evaluatedAt: new Date().toISOString(),\n judge: parsedJudge.data,\n model: activeEvaluation.judgeModel,\n }),\n status: \"complete\",\n });\n },\n schema: traceEvalJudgeStreamSchema,\n });\n\n useEffect(() => {\n latestObjectRef.current = object ?? null;\n }, [object]);\n\n useEffect(() => {\n if (!isLoading || startedAtRef.current === null) {\n return;\n }\n\n const intervalId = window.setInterval(() => {\n setState((current) => {\n if (current.status !== \"running\" || startedAtRef.current === null) {\n return current;\n }\n\n return {\n ...current,\n elapsedMs: Math.max(0, Date.now() - startedAtRef.current),\n };\n });\n }, 250);\n\n return () => {\n window.clearInterval(intervalId);\n };\n }, [isLoading]);\n\n useEffect(() => {\n if (!isLoading) {\n return;\n }\n\n setState((current) =>\n current.status === \"running\"\n ? buildRunningJudgeState(\n object ?? null,\n startedAtRef.current === null\n ? null\n : Math.max(0, Date.now() - startedAtRef.current)\n )\n : current\n );\n }, [isLoading, object]);\n\n useEffect(() => {\n if (snapshot.status !== \"complete\" || !evaluationKey) {\n lastEvaluationKeyRef.current = null;\n activeEvaluationRef.current = null;\n startedAtRef.current = null;\n latestObjectRef.current = null;\n stop();\n clear();\n setState((current) =>\n current.status === \"idle\" ? current : initialJudgeState\n );\n return;\n }\n\n if (lastEvaluationKeyRef.current === evaluationKey) {\n return;\n }\n\n lastEvaluationKeyRef.current = evaluationKey;\n activeEvaluationRef.current = {\n evaluationKey,\n judgeModel,\n snapshot,\n };\n startedAtRef.current = Date.now();\n latestObjectRef.current = null;\n stop();\n clear();\n setState(buildRunningJudgeState(null, 0));\n submit({ snapshot });\n }, [clear, evaluationKey, judgeModel, snapshot, stop, submit]);\n\n return state;\n}\n", "type": "registry:component", "target": "@components/trace-eval-agent/use-trace-eval-judge.ts" }, { "path": "registry/trace-eval-agent/components/ai-elements/prompt-input.tsx", "content": "\"use client\";\n\nimport { CornerDownLeftIcon, LoaderCircleIcon, SquareIcon } from \"lucide-react\";\nimport type {\n ComponentProps,\n FormEvent,\n HTMLAttributes,\n ReactNode,\n TextareaHTMLAttributes,\n} from \"react\";\nimport {\n createContext,\n useCallback,\n useContext,\n useMemo,\n useState,\n} from \"react\";\nimport { Button } from \"@/components/ui/button\";\nimport { Textarea } from \"@/components/ui/textarea\";\nimport { cn } from \"@/lib/utils\";\n\ntype PromptStatus = \"error\" | \"ready\" | \"streaming\" | \"submitted\";\n\ninterface PromptInputContextValue {\n setText: (text: string) => void;\n text: string;\n}\n\ninterface PromptInputMessage {\n text: string;\n}\n\ninterface PromptInputProps\n extends Omit, \"onSubmit\"> {\n children: ReactNode;\n onSubmit: (\n message: PromptInputMessage,\n event: FormEvent\n ) => void | Promise;\n}\n\ninterface PromptInputSubmitProps\n extends Omit, \"children\" | \"type\"> {\n onStop?: () => void;\n status?: PromptStatus;\n}\n\ntype PromptInputClickEvent = Parameters<\n Exclude[\"onClick\"], undefined>\n>[0];\n\nconst PromptInputContext = createContext(null);\n\nfunction usePromptInputContext() {\n const context = useContext(PromptInputContext);\n\n if (!context) {\n throw new Error(\n \"PromptInput components must be used inside .\"\n );\n }\n\n return context;\n}\n\nexport function PromptInput({\n children,\n className,\n onSubmit,\n ...props\n}: PromptInputProps) {\n const [text, setText] = useState(\"\");\n\n const contextValue = useMemo(\n () => ({\n setText,\n text,\n }),\n [text]\n );\n\n const handleSubmit = useCallback(\n async (event: FormEvent) => {\n event.preventDefault();\n\n const nextText = text.trim();\n if (!nextText) {\n return;\n }\n\n const result = onSubmit({ text: nextText }, event);\n\n try {\n await result;\n setText(\"\");\n } catch {\n // Keep the text for retry after a failed submit.\n }\n },\n [onSubmit, text]\n );\n\n return (\n \n \n {children}\n \n \n );\n}\n\nexport function PromptInputBody({\n className,\n ...props\n}: HTMLAttributes) {\n return
;\n}\n\nexport function PromptInputFooter({\n className,\n ...props\n}: HTMLAttributes) {\n return
;\n}\n\nexport function PromptInputTextarea({\n className,\n disabled,\n onChange,\n ...props\n}: TextareaHTMLAttributes) {\n const context = usePromptInputContext();\n\n return (\n {\n context.setText(event.currentTarget.value);\n onChange?.(event);\n }}\n value={context.text}\n {...props}\n />\n );\n}\n\nexport function PromptInputSubmit({\n className,\n disabled,\n onClick,\n onStop,\n size = \"icon\",\n status = \"ready\",\n variant = \"default\",\n ...props\n}: PromptInputSubmitProps) {\n const { text } = usePromptInputContext();\n const isBusy = status === \"submitted\" || status === \"streaming\";\n const isDisabled = Boolean(disabled) || (!isBusy && text.trim().length === 0);\n\n let icon = ;\n\n if (status === \"submitted\") {\n icon = ;\n } else if (status === \"streaming\") {\n icon = ;\n }\n\n return (\n {\n if (isBusy && onStop) {\n event.preventDefault();\n onStop();\n return;\n }\n\n onClick?.(event);\n }}\n size={size}\n type={isBusy && onStop ? \"button\" : \"submit\"}\n variant={variant}\n {...props}\n >\n {icon}\n \n );\n}\n", "type": "registry:component", "target": "@components/ai-elements/prompt-input.tsx" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/server/chat.ts", "content": "import { createOpenAI } from \"@ai-sdk/openai\";\nimport {\n convertToModelMessages,\n stepCountIs,\n streamText,\n type UIMessage,\n} from \"ai\";\n\nimport { projectTraceEvalHistoryForModel } from \"@/lib/trace-eval-agent/model/trace-eval-chat-history\";\nimport { getTraceEvalAgentConfig, type TraceEvalAgentEnv } from \"./env\";\nimport {\n resolveTraceEvalAgentChatModel,\n TRACE_EVAL_AGENT_PROVIDER_OPTIONS,\n TRACE_EVAL_SEARCH_TOOL_NAME,\n} from \"./model\";\n\nconst traceEvalAgentInstructions = [\n \"You are the Trace and Eval research agent demo.\",\n \"Turn each user request into a concise, source-grounded research answer.\",\n \"Assume the user wants a best-effort answer immediately.\",\n \"For non-trivial factual or current-event research, call web_search before finalizing the answer.\",\n \"Start with one broad web_search call.\",\n \"Only run a second web_search if the first search leaves a concrete evidence gap.\",\n \"Do not exceed two web_search calls in a single answer.\",\n \"Do not stop for clarification on broad research prompts unless a missing constraint would make the answer materially wrong.\",\n \"If the prompt is still broad, state the assumption briefly, run search, and continue.\",\n \"Use the search evidence to synthesize findings instead of copying snippets.\",\n \"Expose uncertainty clearly when the evidence is thin or conflicting.\",\n \"Whenever the tool returns enough evidence, cite at least two concrete sources in the response.\",\n \"Keep the final answer compact, useful, and easy to evaluate.\",\n].join(\" \");\n\nconst trailingSlashPattern = /\\/$/;\nconst v3AiSuffixPattern = /\\/v3\\/ai$/;\n\nfunction resolveOpenAICompatibleBaseURL(baseURL: string) {\n const normalizedBaseURL = baseURL.replace(trailingSlashPattern, \"\");\n\n if (normalizedBaseURL.endsWith(\"/v1\")) {\n return normalizedBaseURL;\n }\n\n if (normalizedBaseURL.endsWith(\"/v3/ai\")) {\n return normalizedBaseURL.replace(v3AiSuffixPattern, \"/v1\");\n }\n\n throw new Error(\n `Trace eval agent expects AI_GATEWAY_BASE_URL to end with /v3/ai or /v1. Received: ${baseURL}`\n );\n}\n\nexport async function streamTraceEvalAgent(\n messages: UIMessage[],\n env: TraceEvalAgentEnv\n) {\n const { apiKey, baseURL } = getTraceEvalAgentConfig(env);\n const chatModel = resolveTraceEvalAgentChatModel(env);\n const openai = createOpenAI({\n apiKey,\n baseURL: resolveOpenAICompatibleBaseURL(baseURL),\n name: \"gateway-openai\",\n });\n const runId = crypto.randomUUID();\n const startedAt = Date.now();\n const messageMetadataBase = {\n model: chatModel,\n runId,\n searchTool: TRACE_EVAL_SEARCH_TOOL_NAME,\n startedAt,\n };\n const replayableMessages = projectTraceEvalHistoryForModel(messages);\n const result = streamText({\n experimental_telemetry: {\n functionId: \"trace-eval-agent.run\",\n isEnabled: true,\n metadata: {\n demo: \"trace-eval-agent\",\n runId,\n searchTool: TRACE_EVAL_SEARCH_TOOL_NAME,\n },\n recordInputs: true,\n recordOutputs: true,\n },\n messages: await convertToModelMessages(replayableMessages),\n model: openai(chatModel),\n providerOptions: TRACE_EVAL_AGENT_PROVIDER_OPTIONS,\n stopWhen: stepCountIs(20),\n system: traceEvalAgentInstructions,\n tools: {\n [TRACE_EVAL_SEARCH_TOOL_NAME]: openai.tools.webSearch({\n searchContextSize: \"medium\",\n }),\n },\n });\n\n return result.toUIMessageStreamResponse({\n messageMetadata: ({ part }) => {\n if (part.type === \"start\") {\n return messageMetadataBase;\n }\n\n if (part.type === \"finish\") {\n return {\n ...messageMetadataBase,\n finishReason: part.finishReason,\n finishedAt: Date.now(),\n totalUsage: part.totalUsage,\n };\n }\n\n return;\n },\n originalMessages: messages,\n sendReasoning: true,\n sendSources: true,\n });\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/server/chat.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/server/evaluation.ts", "content": "import { generateText, Output, streamText } from \"ai\";\nimport { z } from \"zod\";\n\nimport {\n buildTraceEvalJudgeContext,\n completeTraceEvalJudgeResult,\n formatTraceEvalJudgePrompt,\n type TraceEvalJudgeResult,\n traceEvalJudgeStreamSchema,\n} from \"@/lib/trace-eval-agent/model/trace-eval-judge\";\nimport type { TraceEvalSnapshot } from \"@/lib/trace-eval-agent/model/trace-eval-snapshot\";\nimport {\n createTraceEvalAgentGateway,\n getTraceEvalAgentConfig,\n getTraceEvalAgentEnv,\n getTraceEvalAgentSetupState,\n type TraceEvalAgentEnv,\n} from \"./env\";\n\ninterface TraceEvalAgentEvaluationRequestBody {\n snapshot?: unknown;\n}\n\nconst invalidSnapshotError = 'Expected a JSON body with a \"snapshot\" object.';\nconst incompleteRunError =\n \"Trace eval judge requires a completed conversation with a user prompt and assistant answer.\";\n\nconst traceEvalCheckSchema = z.object({\n detail: z.string(),\n id: z.string().min(1),\n status: z.enum([\"passed\", \"failed\", \"running\", \"skipped\"]),\n title: z.string().min(1),\n});\n\nconst traceEvalSourceSchema = z.object({\n id: z.string().min(1),\n origin: z.enum([\"markdown-link\", \"source-part\"]),\n title: z.string().min(1),\n url: z.string().url(),\n});\n\nconst traceEvalTraceItemSchema = z.object({\n detail: z.string(),\n id: z.string().min(1),\n kind: z.enum([\"user\", \"model\", \"tool\", \"source\", \"usage\"]),\n metric: z.string().optional(),\n status: z.enum([\"pending\", \"running\", \"passed\", \"failed\"]),\n title: z.string().min(1),\n});\n\nconst traceEvalSummarySchema = z.object({\n failed: z.number().int().nonnegative(),\n passed: z.number().int().nonnegative(),\n skipped: z.number().int().nonnegative(),\n total: z.number().int().nonnegative(),\n});\n\nconst traceEvalSnapshotSchema = z.object({\n checks: z.array(traceEvalCheckSchema),\n durationMs: z.number().nonnegative().optional(),\n latestAnswer: z.string(),\n latestPrompt: z.string().nullable(),\n runId: z.string().min(1).optional(),\n score: z.number().min(0).max(1),\n sources: z.array(traceEvalSourceSchema),\n status: z.enum([\"empty\", \"running\", \"complete\"]),\n summary: traceEvalSummarySchema,\n totalTokens: z.number().nonnegative().optional(),\n trace: z.array(traceEvalTraceItemSchema),\n});\n\nconst judgeInstructions = [\n \"You are the Trace and Eval Agent LLM-as-judge evaluator.\",\n \"Evaluate both the final answer quality and the full run process.\",\n \"Deterministic checks are hard constraints. Do not hide missing search, missing sources, provider errors, or research-shape failures behind a broad qualitative score.\",\n \"Use normalized 0-1 scores for overallScore and every dimension score. Never return percentages or 1-5 ratings.\",\n \"Treat token usage as observability context only. Do not score down only because token usage is high.\",\n \"Use the provided rubric dimensions exactly.\",\n \"Return concise rationale that a production team can act on.\",\n].join(\" \");\n\nfunction readTraceEvalSnapshot(body: unknown): TraceEvalSnapshot {\n const { snapshot } = (body ?? {}) as TraceEvalAgentEvaluationRequestBody;\n const parsedSnapshot = traceEvalSnapshotSchema.safeParse(snapshot);\n\n if (!parsedSnapshot.success) {\n throw new Error(invalidSnapshotError);\n }\n\n return parsedSnapshot.data satisfies TraceEvalSnapshot;\n}\n\nasync function readTraceEvalSnapshotRequest(\n request: Request\n): Promise {\n const bodyText = await request.text();\n\n if (!bodyText.trim()) {\n throw new Error(invalidSnapshotError);\n }\n\n try {\n return readTraceEvalSnapshot(JSON.parse(bodyText));\n } catch (error) {\n if (error instanceof SyntaxError) {\n throw new Error(invalidSnapshotError);\n }\n\n throw error;\n }\n}\n\nfunction assertJudgeableRun({\n answer,\n prompt,\n}: {\n answer: string;\n prompt: string;\n}) {\n if (!(prompt.trim() && answer.trim())) {\n throw new Error(incompleteRunError);\n }\n}\n\nexport async function evaluateTraceEvalRun(\n snapshot: TraceEvalSnapshot,\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n): Promise<{\n judge: TraceEvalJudgeResult;\n}> {\n const gateway = createTraceEvalAgentGateway(env);\n const { chatModel } = getTraceEvalAgentConfig(env);\n const context = buildTraceEvalJudgeContext(snapshot);\n\n assertJudgeableRun(context);\n\n const result = await generateText({\n experimental_telemetry: {\n functionId: \"trace-eval-agent.judge\",\n isEnabled: true,\n metadata: {\n demo: \"trace-eval-agent\",\n deterministicFailures: context.deterministicFailures.length,\n judge: \"llm-as-judge\",\n },\n recordInputs: true,\n recordOutputs: true,\n },\n model: gateway(chatModel),\n output: Output.object({\n description:\n \"A structured LLM-as-judge result for a trace/eval research-agent run.\",\n name: \"TraceEvalJudgeResult\",\n schema: traceEvalJudgeStreamSchema,\n }),\n prompt: formatTraceEvalJudgePrompt(context),\n system: judgeInstructions,\n });\n\n return {\n judge: completeTraceEvalJudgeResult({\n deterministicFailures: context.deterministicFailures,\n evaluatedAt: new Date().toISOString(),\n judge: result.output,\n model: chatModel,\n }),\n };\n}\n\nfunction createTraceEvalJudgeStreamResponse({\n snapshot,\n env,\n}: {\n snapshot: TraceEvalSnapshot;\n env: TraceEvalAgentEnv;\n}) {\n const gateway = createTraceEvalAgentGateway(env);\n const { chatModel } = getTraceEvalAgentConfig(env);\n const context = buildTraceEvalJudgeContext(snapshot);\n\n assertJudgeableRun(context);\n\n const result = streamText({\n experimental_telemetry: {\n functionId: \"trace-eval-agent.judge\",\n isEnabled: true,\n metadata: {\n demo: \"trace-eval-agent\",\n deterministicFailures: context.deterministicFailures.length,\n judge: \"llm-as-judge\",\n transport: \"object-stream\",\n },\n recordInputs: true,\n recordOutputs: true,\n },\n model: gateway(chatModel),\n output: Output.object({\n description:\n \"A structured LLM-as-judge result for a trace/eval research-agent run.\",\n name: \"TraceEvalJudgeResult\",\n schema: traceEvalJudgeStreamSchema,\n }),\n prompt: formatTraceEvalJudgePrompt(context),\n system: judgeInstructions,\n });\n\n const encoder = new TextEncoder();\n\n return new Response(\n new ReadableStream({\n async start(controller) {\n try {\n for await (const chunk of result.textStream) {\n controller.enqueue(encoder.encode(chunk));\n }\n\n controller.close();\n } catch (error) {\n controller.error(error);\n }\n },\n }),\n {\n headers: {\n \"cache-control\": \"no-cache\",\n \"content-type\": \"text/plain; charset=utf-8\",\n },\n }\n );\n}\n\nexport async function handleTraceEvalAgentEvaluationRequest(\n request: Request,\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n) {\n const setup = getTraceEvalAgentSetupState(env);\n\n if (!setup.isReady) {\n return Response.json(\n {\n error: setup.issues.join(\" \"),\n },\n { status: 500 }\n );\n }\n\n try {\n const snapshot = await readTraceEvalSnapshotRequest(request);\n const result = await evaluateTraceEvalRun(snapshot, env);\n\n return Response.json(result);\n } catch (error) {\n if (\n error instanceof Error &&\n [invalidSnapshotError, incompleteRunError].includes(error.message)\n ) {\n return Response.json(\n {\n error: error.message,\n },\n { status: 400 }\n );\n }\n\n throw error;\n }\n}\n\nexport async function handleTraceEvalAgentEvaluationStreamRequest(\n request: Request,\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n) {\n const setup = getTraceEvalAgentSetupState(env);\n\n if (!setup.isReady) {\n return Response.json(\n {\n error: setup.issues.join(\" \"),\n },\n { status: 500 }\n );\n }\n\n try {\n const snapshot = await readTraceEvalSnapshotRequest(request);\n return createTraceEvalJudgeStreamResponse({ env, snapshot });\n } catch (error) {\n if (\n error instanceof Error &&\n [invalidSnapshotError, incompleteRunError].includes(error.message)\n ) {\n return Response.json(\n {\n error: error.message,\n },\n { status: 400 }\n );\n }\n\n throw error;\n }\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/server/evaluation.ts" }, { "path": "registry/trace-eval-agent/lib/ai-gateway/contract.ts", "content": "import { createGateway } from \"ai\";\n\nexport const DEFAULT_GATEWAY_BASE_URL = \"https://ai-gateway.vercel.sh/v3/ai\";\nexport const MINIMUM_NODE_VERSION = \"22.13.0\";\nconst nodeVersionPattern = /^v?(\\d+)\\.(\\d+)\\.(\\d+)(?:[-+].*)?$/;\n\nexport type AiGatewayEnvRecord = Record;\n\nexport interface ParsedNodeVersion {\n major: number;\n minor: number;\n patch: number;\n}\n\nexport interface AiGatewayContractConfig {\n apiKey: string;\n baseURL: string;\n chatModel: string;\n}\n\nexport interface AiGatewayResolvedEnv {\n apiKey: string | undefined;\n baseURL: string;\n chatModel: string;\n}\n\nexport interface AiGatewaySetupConfig {\n baseURL: string;\n chatModel: string;\n}\n\nexport interface AiGatewayContractSetupState<\n TConfig extends AiGatewaySetupConfig = AiGatewaySetupConfig,\n> {\n config: TConfig;\n isReady: boolean;\n issues: string[];\n nodeVersion: string;\n}\n\nexport interface AiGatewayContractOptions<\n TConfig extends AiGatewaySetupConfig = AiGatewaySetupConfig,\n> {\n buildConfig?: (\n resolvedEnv: AiGatewayResolvedEnv,\n env: AiGatewayEnvRecord\n ) => TConfig;\n defaultBaseURL?: string;\n defaultChatModel: string;\n getAdditionalIssues?: (\n resolvedEnv: AiGatewayResolvedEnv,\n env: AiGatewayEnvRecord\n ) => string[];\n missingApiKeyError: string;\n missingApiKeyIssue?: string;\n}\n\nconst genericMissingApiKeyIssue =\n \"AI_GATEWAY_API_KEY is missing. The demo can render, but chat requests will fail until it is configured.\";\n\nexport function parseNodeVersion(version: string): ParsedNodeVersion {\n const match = nodeVersionPattern.exec(version);\n const major = Number(match?.[1]);\n const minor = Number(match?.[2]);\n const patch = Number(match?.[3]);\n\n if (![major, minor, patch].every(Number.isInteger)) {\n throw new Error(`Unable to parse Node.js version: \"${version}\".`);\n }\n\n return { major, minor, patch };\n}\n\nexport function getNodeMajor(version: string): number {\n return parseNodeVersion(version).major;\n}\n\nfunction compareNodeVersions(\n left: ParsedNodeVersion,\n right: ParsedNodeVersion\n): number {\n if (left.major !== right.major) {\n return left.major - right.major;\n }\n\n if (left.minor !== right.minor) {\n return left.minor - right.minor;\n }\n\n return left.patch - right.patch;\n}\n\nexport function assertSupportedNodeRuntime(version = process.version): number {\n const parsedVersion = parseNodeVersion(version);\n const minimumVersion = parseNodeVersion(MINIMUM_NODE_VERSION);\n\n if (compareNodeVersions(parsedVersion, minimumVersion) < 0) {\n throw new Error(\n `Node.js ${version} is unsupported. This demo workspace requires Node.js >=${MINIMUM_NODE_VERSION}.`\n );\n }\n\n return parsedVersion.major;\n}\n\nexport function resolveAiGatewayContractEnv(\n env: AiGatewayEnvRecord,\n options: Pick\n): AiGatewayResolvedEnv {\n return {\n apiKey: env.AI_GATEWAY_API_KEY,\n baseURL:\n env.AI_GATEWAY_BASE_URL ||\n options.defaultBaseURL ||\n DEFAULT_GATEWAY_BASE_URL,\n chatModel: env.AI_GATEWAY_CHAT_MODEL || options.defaultChatModel,\n };\n}\n\nexport function readAiGatewayContractConfig(\n env: AiGatewayEnvRecord,\n options: AiGatewayContractOptions\n): AiGatewayContractConfig {\n assertSupportedNodeRuntime();\n const resolvedEnv = resolveAiGatewayContractEnv(env, options);\n\n if (!resolvedEnv.apiKey) {\n throw new Error(options.missingApiKeyError);\n }\n\n return {\n apiKey: resolvedEnv.apiKey,\n baseURL: resolvedEnv.baseURL,\n chatModel: resolvedEnv.chatModel,\n };\n}\n\nexport function buildAiGatewayContractSetupState<\n TConfig extends AiGatewaySetupConfig,\n>(\n env: AiGatewayEnvRecord,\n options: AiGatewayContractOptions\n): AiGatewayContractSetupState {\n const issues: string[] = [];\n const resolvedEnv = resolveAiGatewayContractEnv(env, options);\n\n try {\n assertSupportedNodeRuntime();\n } catch (error) {\n issues.push(\n error instanceof Error ? error.message : \"Unsupported Node.js runtime.\"\n );\n }\n\n if (!resolvedEnv.apiKey) {\n issues.push(options.missingApiKeyIssue || genericMissingApiKeyIssue);\n }\n\n issues.push(...(options.getAdditionalIssues?.(resolvedEnv, env) ?? []));\n\n return {\n config:\n options.buildConfig?.(resolvedEnv, env) ??\n ({\n baseURL: resolvedEnv.baseURL,\n chatModel: resolvedEnv.chatModel,\n } as TConfig),\n isReady: issues.length === 0,\n issues,\n nodeVersion: process.version,\n };\n}\n\nexport function createAiGatewayFromContract(\n env: AiGatewayEnvRecord,\n options: AiGatewayContractOptions\n): ReturnType {\n const { apiKey, baseURL } = readAiGatewayContractConfig(env, options);\n\n return createGateway({\n apiKey,\n baseURL,\n });\n}\n", "type": "registry:lib", "target": "@lib/ai-gateway/contract.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/server/env-source.ts", "content": "export function getTraceEvalAgentAppEnv() {\n // biome-ignore lint/style/noProcessEnv: Registry source installs into consumer apps without this repo's env wrapper.\n return process.env;\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/server/env-source.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/server/env.ts", "content": "import { getTraceEvalAgentAppEnv } from \"./env-source\";\nimport {\n buildAiGatewayContractSetupState,\n createAiGatewayFromContract,\n readAiGatewayContractConfig,\n type AiGatewayContractConfig,\n type AiGatewayContractSetupState,\n type AiGatewayEnvRecord,\n type AiGatewaySetupConfig,\n} from \"@/lib/ai-gateway/contract\";\nimport {\n DEFAULT_TRACE_EVAL_AGENT_CHAT_MODEL,\n resolveTraceEvalAgentChatModel,\n} from \"./model\";\n\nexport const DEFAULT_CHAT_MODEL = DEFAULT_TRACE_EVAL_AGENT_CHAT_MODEL;\n\nexport type TraceEvalAgentEnv = AiGatewayEnvRecord;\n\nexport type TraceEvalAgentConfig = AiGatewayContractConfig;\n\nexport type TraceEvalAgentSetupState = AiGatewayContractSetupState;\n\nexport type TraceEvalAgentGateway = ReturnType;\n\nconst traceEvalAgentContract = {\n defaultChatModel: DEFAULT_CHAT_MODEL,\n missingApiKeyError: \"Missing AI_GATEWAY_API_KEY. Add it to .env.local before using the trace eval agent.\",\n missingApiKeyIssue: \"AI_GATEWAY_API_KEY is missing. The demo can render, but trace and eval requests will fail until it is configured.\",\n} as const;\n\nexport function getTraceEvalAgentEnv(): TraceEvalAgentEnv {\n return getTraceEvalAgentAppEnv();\n}\n\nexport function getTraceEvalAgentConfig(\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n): TraceEvalAgentConfig {\n return {\n ...readAiGatewayContractConfig(env, traceEvalAgentContract),\n chatModel: resolveTraceEvalAgentChatModel(env),\n };\n}\n\nexport function getTraceEvalAgentSetupState(\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n): TraceEvalAgentSetupState {\n return buildAiGatewayContractSetupState(env, {\n ...traceEvalAgentContract,\n buildConfig: (resolvedEnv) => ({\n baseURL: resolvedEnv.baseURL,\n chatModel: resolveTraceEvalAgentChatModel(env),\n }),\n });\n}\n\nexport function createTraceEvalAgentGateway(\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n): TraceEvalAgentGateway {\n return createAiGatewayFromContract(env, traceEvalAgentContract);\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/server/env.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/server/model.ts", "content": "export const DEFAULT_TRACE_EVAL_AGENT_CHAT_MODEL = \"openai/gpt-5-mini\";\nexport const TRACE_EVAL_AGENT_CHAT_MODEL_ENV_KEY =\n \"TRACE_EVAL_AGENT_CHAT_MODEL\";\n\nexport const TRACE_EVAL_SEARCH_TOOL_NAME = \"web_search\";\n\nexport const TRACE_EVAL_AGENT_PROVIDER_OPTIONS = {\n openai: {\n textVerbosity: \"low\",\n },\n} as const;\n\ntype TraceEvalAgentEnv = Record;\n\nexport function resolveTraceEvalAgentChatModel(\n env: TraceEvalAgentEnv = {}\n): string {\n const featureChatModel =\n env[TRACE_EVAL_AGENT_CHAT_MODEL_ENV_KEY] ||\n DEFAULT_TRACE_EVAL_AGENT_CHAT_MODEL;\n\n return (\n featureChatModel ||\n env.AI_GATEWAY_CHAT_MODEL ||\n DEFAULT_TRACE_EVAL_AGENT_CHAT_MODEL\n );\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/server/model.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/server/runtime.ts", "content": "import { type UIMessage, validateUIMessages } from \"ai\";\n\nimport { streamTraceEvalAgent } from \"./chat\";\nimport {\n getTraceEvalAgentEnv,\n getTraceEvalAgentSetupState,\n type TraceEvalAgentEnv,\n} from \"./env\";\nimport { resolveTraceEvalAgentChatModel } from \"./model\";\n\ninterface TraceEvalAgentRequestBody {\n messages?: UIMessage[];\n}\n\ninterface TraceEvalAgentRequestDependencies {\n streamTraceEvalAgent: (\n messages: UIMessage[],\n env: TraceEvalAgentEnv\n ) => Promise | Response;\n}\n\nexport interface TraceEvalAgentRuntimeState {\n chatModel: string;\n isChatAvailable: boolean;\n nodeVersion: string;\n setupMessage: string | null;\n statusLabel: \"Ready\" | \"Setup required\";\n}\n\nconst invalidMessagesError = 'Expected a JSON body with a \"messages\" array.';\nconst invalidUiMessagesError =\n 'Expected each \"messages\" entry to match the UIMessage format.';\n\nasync function readTraceEvalAgentMessages(body: unknown): Promise {\n const { messages } = (body ?? {}) as TraceEvalAgentRequestBody;\n\n if (!Array.isArray(messages)) {\n throw new Error(invalidMessagesError);\n }\n\n try {\n return await validateUIMessages({ messages });\n } catch {\n throw new Error(invalidUiMessagesError);\n }\n}\n\nexport function getTraceEvalAgentRuntimeState(\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv()\n): TraceEvalAgentRuntimeState {\n const setup = getTraceEvalAgentSetupState(env);\n\n return {\n chatModel: resolveTraceEvalAgentChatModel(env),\n isChatAvailable: setup.isReady,\n nodeVersion: setup.nodeVersion,\n setupMessage: setup.issues.length > 0 ? setup.issues.join(\" \") : null,\n statusLabel: setup.isReady ? \"Ready\" : \"Setup required\",\n };\n}\n\nexport async function handleTraceEvalAgentRequest(\n request: Request,\n env: TraceEvalAgentEnv = getTraceEvalAgentEnv(),\n dependencies: TraceEvalAgentRequestDependencies = {\n streamTraceEvalAgent,\n }\n) {\n const runtimeState = getTraceEvalAgentRuntimeState(env);\n\n if (!runtimeState.isChatAvailable) {\n return Response.json(\n {\n error: runtimeState.setupMessage,\n },\n { status: 500 }\n );\n }\n\n try {\n const messages = await readTraceEvalAgentMessages(await request.json());\n\n return dependencies.streamTraceEvalAgent(messages, env);\n } catch (error) {\n if (\n error instanceof Error &&\n [invalidMessagesError, invalidUiMessagesError].includes(error.message)\n ) {\n return Response.json(\n {\n error: error.message,\n },\n { status: 400 }\n );\n }\n\n throw error;\n }\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/server/runtime.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-chat-history.ts", "content": "import { isFileUIPart, isTextUIPart, type UIMessage } from \"ai\";\n\nimport { isTraceEvalRefusalAnswer } from \"./trace-eval-run-state\";\n\nfunction getReplayableAssistantText(message: UIMessage) {\n return message.parts\n .filter(isTextUIPart)\n .map((part) => part.text)\n .join(\"\\n\")\n .trim();\n}\n\nexport function projectTraceEvalHistoryForModel(messages: UIMessage[]) {\n return messages.flatMap((message) => {\n if (message.role === \"assistant\") {\n const text = getReplayableAssistantText(message);\n\n if (!(text && !isTraceEvalRefusalAnswer(text))) {\n return [];\n }\n\n return [\n {\n id: message.id,\n parts: [\n {\n text,\n type: \"text\" as const,\n },\n ],\n role: \"assistant\" as const,\n },\n ];\n }\n\n if (message.role === \"user\") {\n const parts = message.parts.filter(\n (part) => isTextUIPart(part) || isFileUIPart(part)\n );\n\n return parts.length > 0\n ? [\n {\n ...message,\n parts,\n },\n ]\n : [];\n }\n\n if (message.role === \"system\") {\n const parts = message.parts.filter(isTextUIPart);\n\n return parts.length > 0\n ? [\n {\n ...message,\n parts,\n },\n ]\n : [];\n }\n\n return [];\n });\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-chat-history.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-judge-progress.ts", "content": "import type { DeepPartial } from \"ai\";\n\nimport type { TraceEvalJudgeStreamObject } from \"./trace-eval-judge\";\n\nexport type TraceEvalJudgeProgressStep =\n | \"starting\"\n | \"summary\"\n | \"dimensions\"\n | \"finalizing\";\n\nexport interface TraceEvalJudgeProgressState {\n label: string;\n message: string;\n progress: number;\n step: TraceEvalJudgeProgressStep;\n}\n\nconst startingProgressState: TraceEvalJudgeProgressState = {\n label: \"Starting\",\n message: \"Opening the structured judge stream.\",\n progress: 0.08,\n step: \"starting\",\n};\n\nexport function deriveTraceEvalJudgeProgress(\n partialJudge: DeepPartial | null\n): TraceEvalJudgeProgressState {\n if (!partialJudge) {\n return startingProgressState;\n }\n\n if (partialJudge.action) {\n return {\n label: \"Finalizing recommendation\",\n message: \"The judge is completing the final remediation recommendation.\",\n progress: 0.92,\n step: \"finalizing\",\n };\n }\n\n if ((partialJudge.dimensions?.length ?? 0) > 0) {\n return {\n label: \"Scoring dimensions\",\n message: \"The judge is filling the dimension-by-dimension scores.\",\n progress: 0.72,\n step: \"dimensions\",\n };\n }\n\n if (partialJudge.summary || partialJudge.rationale) {\n return {\n label: \"Streaming summary\",\n message: \"The judge is drafting the high-level summary and rationale.\",\n progress: 0.32,\n step: \"summary\",\n };\n }\n\n return startingProgressState;\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-judge-progress.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-judge.ts", "content": "import { z } from \"zod\";\nimport type {\n TraceEvalCheck,\n TraceEvalSnapshot,\n TraceEvalSource,\n TraceEvalTraceItem,\n} from \"./trace-eval-snapshot\";\n\nexport type TraceEvalJudgeAction =\n | \"ready\"\n | \"needs-revision\"\n | \"rerun-research\"\n | \"investigate-failure\";\n\nexport type TraceEvalJudgeDimensionId =\n | \"answer-usefulness\"\n | \"source-faithfulness\"\n | \"evidence-sufficiency\"\n | \"uncertainty-handling\"\n | \"run-discipline\";\n\nexport interface TraceEvalJudgeRubricDimension {\n description: string;\n id: TraceEvalJudgeDimensionId;\n title: string;\n}\n\nexport interface TraceEvalJudgeDimensionScore {\n id: TraceEvalJudgeDimensionId;\n rationale: string;\n score: number;\n title: string;\n}\n\nexport interface TraceEvalJudgeResult {\n action: TraceEvalJudgeAction;\n deterministicFailures: TraceEvalCheck[];\n dimensions: TraceEvalJudgeDimensionScore[];\n evaluatedAt: string;\n model: string;\n overallScore: number;\n rationale: string;\n summary: string;\n}\n\nexport interface TraceEvalJudgeContext {\n answer: string;\n deterministicChecks: TraceEvalCheck[];\n deterministicFailures: TraceEvalCheck[];\n prompt: string;\n rubric: TraceEvalJudgeRubricDimension[];\n sources: TraceEvalSource[];\n trace: TraceEvalTraceItem[];\n usage: {\n durationMs?: number;\n totalTokens?: number;\n };\n}\n\nexport const traceEvalJudgeDimensionSchema = z.object({\n id: z.enum([\n \"answer-usefulness\",\n \"source-faithfulness\",\n \"evidence-sufficiency\",\n \"uncertainty-handling\",\n \"run-discipline\",\n ]),\n rationale: z.string().min(1),\n score: z.number().min(0).max(1),\n title: z.string().min(1),\n});\n\nexport const traceEvalJudgeStreamSchema = z.object({\n action: z.enum([\n \"ready\",\n \"needs-revision\",\n \"rerun-research\",\n \"investigate-failure\",\n ]),\n dimensions: z.array(traceEvalJudgeDimensionSchema).min(1),\n overallScore: z.number().min(0).max(1),\n rationale: z.string().min(1),\n summary: z.string().min(1),\n});\n\nexport type TraceEvalJudgeStreamObject = z.infer<\n typeof traceEvalJudgeStreamSchema\n>;\n\nexport const traceEvalJudgeRubric: TraceEvalJudgeRubricDimension[] = [\n {\n description:\n \"Whether the final answer is directly useful for the user's research request.\",\n id: \"answer-usefulness\",\n title: \"Answer usefulness\",\n },\n {\n description:\n \"Whether claims in the answer stay faithful to the visible sources.\",\n id: \"source-faithfulness\",\n title: \"Source faithfulness\",\n },\n {\n description:\n \"Whether the run gathered enough evidence for the requested comparison or recommendation.\",\n id: \"evidence-sufficiency\",\n title: \"Evidence sufficiency\",\n },\n {\n description:\n \"Whether the answer names uncertainty, missing context, or conflicting evidence when needed.\",\n id: \"uncertainty-handling\",\n title: \"Uncertainty handling\",\n },\n {\n description:\n \"Whether the run followed the expected process: use search for current facts, expose sources, avoid bypassing required tools, and explain failures or weak evidence.\",\n id: \"run-discipline\",\n title: \"Run discipline\",\n },\n];\n\nexport function buildTraceEvalJudgeContext(\n snapshot: TraceEvalSnapshot\n): TraceEvalJudgeContext {\n return {\n answer: snapshot.latestAnswer,\n deterministicChecks: snapshot.checks,\n deterministicFailures: snapshot.checks.filter(\n (check) => check.status === \"failed\"\n ),\n prompt: snapshot.latestPrompt ?? \"\",\n rubric: traceEvalJudgeRubric,\n sources: snapshot.sources,\n trace: snapshot.trace,\n usage: {\n durationMs: snapshot.durationMs,\n totalTokens: snapshot.totalTokens,\n },\n };\n}\n\nconst investigateFailureIds = new Set([\"provider-error\", \"tool-error\"]);\n\nexport function resolveTraceEvalJudgeAction({\n deterministicFailures,\n overallScore,\n}: {\n deterministicFailures: TraceEvalCheck[];\n overallScore: number;\n}): TraceEvalJudgeAction {\n if (\n deterministicFailures.some((failure) =>\n investigateFailureIds.has(failure.id)\n )\n ) {\n return \"investigate-failure\";\n }\n\n if (deterministicFailures.length > 0) {\n return \"rerun-research\";\n }\n\n if (overallScore >= 0.8) {\n return \"ready\";\n }\n\n return \"needs-revision\";\n}\n\nexport function completeTraceEvalJudgeResult({\n deterministicFailures,\n evaluatedAt,\n judge,\n model,\n}: {\n deterministicFailures: TraceEvalCheck[];\n evaluatedAt: string;\n judge: TraceEvalJudgeStreamObject;\n model: string;\n}): TraceEvalJudgeResult {\n return {\n ...judge,\n action: resolveTraceEvalJudgeAction({\n deterministicFailures,\n overallScore: judge.overallScore,\n }),\n deterministicFailures,\n evaluatedAt,\n model,\n };\n}\n\nexport function formatTraceEvalJudgePrompt(\n context: TraceEvalJudgeContext\n): string {\n return [\n \"Evaluate this AI research-agent run.\",\n \"\",\n \"User prompt:\",\n context.prompt || \"No user prompt captured.\",\n \"\",\n \"Final answer:\",\n context.answer || \"No final answer captured.\",\n \"\",\n \"Deterministic checks:\",\n JSON.stringify(context.deterministicChecks, null, 2),\n \"\",\n \"Deterministic failures:\",\n JSON.stringify(context.deterministicFailures, null, 2),\n \"\",\n \"Execution trace:\",\n JSON.stringify(context.trace, null, 2),\n \"\",\n \"Sources:\",\n JSON.stringify(context.sources, null, 2),\n \"\",\n \"Usage:\",\n JSON.stringify(context.usage, null, 2),\n \"\",\n \"Rubric dimensions:\",\n JSON.stringify(context.rubric, null, 2),\n ].join(\"\\n\");\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-judge.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-prompt-intent.ts", "content": "const greetingPattern =\n /^(?:hi|hello|hey|yo|sup|hola|bonjour|你好|您好|嗨|哈喽|在吗)[!.?]*$/i;\n\nconst researchIntentPattern =\n /\\b(research|compare|comparison|evaluate|analysis|analyze|explain|latest|current|today|recommend|recommendation|pick|should|versus|vs\\.?|source|sources|cite|search)\\b|研究|调研|比较|分析|解释|最新|推荐|检索|搜索|资料|来源/u;\n\nconst minimumResearchPromptLength = 20;\n\nexport function hasTraceEvalResearchIntent(prompt: string | null) {\n const normalizedPrompt = prompt?.trim() ?? \"\";\n\n if (normalizedPrompt.length === 0) {\n return false;\n }\n\n if (greetingPattern.test(normalizedPrompt)) {\n return false;\n }\n\n return (\n normalizedPrompt.length >= minimumResearchPromptLength ||\n researchIntentPattern.test(normalizedPrompt)\n );\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-prompt-intent.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-run-outcome.ts", "content": "import type { TraceEvalRunRecord } from \"./trace-eval-run-record\";\nimport { isTraceEvalRefusalAnswer } from \"./trace-eval-run-state\";\n\nexport type TraceEvalRunOutcome =\n | {\n detail: string;\n kind: \"empty\";\n shouldJudge: false;\n title: string;\n }\n | {\n detail: string;\n kind: \"running\";\n shouldJudge: false;\n title: string;\n }\n | {\n detail: string;\n kind: \"skipped\";\n reason: \"non-research-turn\";\n shouldJudge: false;\n title: string;\n }\n | {\n detail: string;\n kind: \"failed-run\";\n reason: \"provider-error\" | \"refusal\" | \"missing-answer\";\n shouldJudge: false;\n title: string;\n }\n | {\n detail: string;\n kind: \"evaluated\";\n shouldJudge: true;\n title: string;\n };\n\nexport function classifyTraceEvalRunOutcome({\n error,\n record,\n}: {\n error?: Error | null;\n record: TraceEvalRunRecord;\n}): TraceEvalRunOutcome {\n if (error) {\n return {\n detail: error.message,\n kind: \"failed-run\",\n reason: \"provider-error\",\n shouldJudge: false,\n title: \"Run failed before evaluation\",\n };\n }\n\n if (record.status === \"empty\") {\n return {\n detail: \"Run a research question to populate trace and eval.\",\n kind: \"empty\",\n shouldJudge: false,\n title: \"No run yet\",\n };\n }\n\n if (record.status === \"running\") {\n return {\n detail: \"The latest run is still streaming.\",\n kind: \"running\",\n shouldJudge: false,\n title: \"Run in progress\",\n };\n }\n\n if (!record.hasResearchIntent) {\n return {\n detail:\n \"This turn looks like casual chat, so trace and eval are skipped without affecting the agent conversation.\",\n kind: \"skipped\",\n reason: \"non-research-turn\",\n shouldJudge: false,\n title: \"Skipped non-research turn\",\n };\n }\n\n if (isTraceEvalRefusalAnswer(record.latestAnswer)) {\n return {\n detail:\n \"The assistant refused the run. Retry with a research prompt that allows live search and source-grounded output.\",\n kind: \"failed-run\",\n reason: \"refusal\",\n shouldJudge: false,\n title: \"Run ended in refusal\",\n };\n }\n\n if (!record.latestAnswer.trim()) {\n return {\n detail:\n \"The run completed without a final answer. Retry the run before evaluating quality.\",\n kind: \"failed-run\",\n reason: \"missing-answer\",\n shouldJudge: false,\n title: \"Run completed without answer\",\n };\n }\n\n return {\n detail: \"The run produced a research answer and can be evaluated.\",\n kind: \"evaluated\",\n shouldJudge: true,\n title: \"Ready for evaluation\",\n };\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-run-outcome.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-run-record.ts", "content": "import {\n getToolName,\n isTextUIPart,\n isToolUIPart,\n type LanguageModelUsage,\n type SourceUrlUIPart,\n type UIMessage,\n} from \"ai\";\nimport { hasTraceEvalResearchIntent } from \"./trace-eval-prompt-intent\";\n\nexport interface TraceEvalAgentMessageMetadata {\n finishedAt?: number;\n finishReason?: string;\n model?: string;\n runId?: string;\n searchTool?: string;\n startedAt?: number;\n totalUsage?: LanguageModelUsage;\n}\n\nexport type TraceEvalAgentMessage = UIMessage;\n\nexport type TraceEvalStatus = \"empty\" | \"running\" | \"complete\";\n\nexport interface TraceEvalSource {\n id: string;\n origin: \"markdown-link\" | \"source-part\";\n title: string;\n url: string;\n}\n\nexport interface TraceEvalSearchCall {\n id: string;\n input?: unknown;\n name: string;\n status: \"running\" | \"passed\" | \"failed\";\n}\n\nexport interface TraceEvalRunRecord {\n durationMs?: number;\n finishReason?: string;\n hasResearchIntent: boolean;\n latestAnswer: string;\n latestPrompt: string | null;\n model?: string;\n runId?: string;\n searchCalls: TraceEvalSearchCall[];\n searchTool: string;\n sources: TraceEvalSource[];\n status: TraceEvalStatus;\n totalTokens?: number;\n usage?: LanguageModelUsage;\n}\n\nconst markdownLinkPattern = /\\[([^\\]]+)\\]\\((https?:\\/\\/[^\\s)]+)\\)/g;\n\nfunction getMessageText(message: UIMessage): string {\n return message.parts\n .filter(isTextUIPart)\n .map((part) => part.text)\n .join(\"\");\n}\n\nfunction getLatestUserPrompt(messages: UIMessage[]): string | null {\n for (let index = messages.length - 1; index >= 0; index -= 1) {\n const message = messages[index];\n\n if (message?.role === \"user\") {\n return getMessageText(message).trim() || null;\n }\n }\n\n return null;\n}\n\nfunction getAssistantMessages(messages: UIMessage[]) {\n return messages.filter(\n (message): message is TraceEvalAgentMessage => message.role === \"assistant\"\n );\n}\n\nfunction getLatestAssistant(messages: UIMessage[]) {\n return getAssistantMessages(messages).at(-1) ?? null;\n}\n\nfunction getSearchToolName(messages: UIMessage[]) {\n return getLatestAssistant(messages)?.metadata?.searchTool ?? \"web_search\";\n}\n\nfunction getToolTraceStatus(state: string): TraceEvalSearchCall[\"status\"] {\n if (state === \"output-error\") {\n return \"failed\";\n }\n\n if (state === \"output-available\") {\n return \"passed\";\n }\n\n return \"running\";\n}\n\nfunction getSearchCalls(messages: UIMessage[]): TraceEvalSearchCall[] {\n const searchToolName = getSearchToolName(messages);\n\n return getAssistantMessages(messages)\n .flatMap((message) => message.parts.filter(isToolUIPart))\n .filter((part) => getToolName(part) === searchToolName)\n .map((part, index) => ({\n id: `${part.toolCallId}-${index}`,\n input: part.input,\n name: searchToolName,\n status: getToolTraceStatus(part.state),\n }));\n}\n\nfunction toTraceEvalSource(part: SourceUrlUIPart): TraceEvalSource {\n return {\n id: part.sourceId,\n origin: \"source-part\",\n title: part.title || part.url,\n url: part.url,\n };\n}\n\nfunction getMarkdownLinkSources(text: string): TraceEvalSource[] {\n return Array.from(\n text.matchAll(markdownLinkPattern),\n ([match, rawTitle, rawUrl]) => {\n const url = rawUrl ?? match;\n const title = rawTitle?.trim() || url;\n\n return {\n id: match,\n origin: \"markdown-link\",\n title,\n url,\n };\n }\n );\n}\n\nfunction getSources(messages: UIMessage[]): TraceEvalSource[] {\n const latestAssistant = getLatestAssistant(messages);\n\n if (!latestAssistant) {\n return [];\n }\n\n const sources = [\n ...latestAssistant.parts.flatMap((part) =>\n part.type === \"source-url\" ? [toTraceEvalSource(part)] : []\n ),\n ...getMarkdownLinkSources(getMessageText(latestAssistant)),\n ];\n\n return Array.from(\n new Map(sources.map((source) => [source.url, source])).values()\n );\n}\n\nfunction getDurationMs(metadata: TraceEvalAgentMessageMetadata | undefined) {\n if (!(metadata?.startedAt && metadata.finishedAt)) {\n return;\n }\n\n return Math.max(0, metadata.finishedAt - metadata.startedAt);\n}\n\nexport function buildTraceEvalRunRecord(\n messages: UIMessage[],\n isBusy: boolean\n): TraceEvalRunRecord {\n const latestPrompt = getLatestUserPrompt(messages);\n const latestAssistant = getLatestAssistant(messages);\n const latestAnswer = latestAssistant\n ? getMessageText(latestAssistant).trim()\n : \"\";\n const metadata = latestAssistant?.metadata;\n let status: TraceEvalStatus = \"complete\";\n\n if (messages.length === 0) {\n status = \"empty\";\n } else if (isBusy) {\n status = \"running\";\n }\n\n return {\n durationMs: getDurationMs(metadata),\n finishReason: metadata?.finishReason,\n hasResearchIntent: hasTraceEvalResearchIntent(latestPrompt),\n latestAnswer,\n latestPrompt,\n model: metadata?.model,\n runId: metadata?.runId,\n searchCalls: getSearchCalls(messages),\n searchTool: getSearchToolName(messages),\n sources: getSources(messages),\n status,\n totalTokens: metadata?.totalUsage?.totalTokens,\n usage: metadata?.totalUsage,\n };\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-run-record.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-run-state.ts", "content": "import { hasTraceEvalResearchIntent } from \"./trace-eval-prompt-intent\";\nimport type { TraceEvalSnapshot } from \"./trace-eval-snapshot\";\n\nconst refusalPatterns = [\n /\\bi cannot assist with that request\\b/i,\n /\\bi can't assist with that request\\b/i,\n /\\bi cannot help with that request\\b/i,\n /\\bi can't help with that request\\b/i,\n /\\bi'm sorry, but i cannot assist\\b/i,\n /\\bi'm sorry, but i can't assist\\b/i,\n];\n\nexport function isTraceEvalRefusalAnswer(answer: string) {\n const normalizedAnswer = answer.trim();\n\n if (normalizedAnswer.length === 0) {\n return false;\n }\n\n return refusalPatterns.some((pattern) => pattern.test(normalizedAnswer));\n}\n\nexport function shouldEvaluateTraceEvalSnapshot(snapshot: TraceEvalSnapshot) {\n return Boolean(\n snapshot.status === \"complete\" &&\n hasTraceEvalResearchIntent(snapshot.latestPrompt) &&\n snapshot.latestAnswer.trim() &&\n !isTraceEvalRefusalAnswer(snapshot.latestAnswer)\n );\n}\n\nexport function hasTraceEvalFailedGeneration(snapshot: TraceEvalSnapshot) {\n return Boolean(\n snapshot.status === \"complete\" &&\n hasTraceEvalResearchIntent(snapshot.latestPrompt) &&\n !snapshot.latestAnswer.trim()\n );\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-run-state.ts" }, { "path": "registry/trace-eval-agent/lib/trace-eval-agent/model/trace-eval-snapshot.ts", "content": "import type { UIMessage } from \"ai\";\nimport {\n buildTraceEvalRunRecord,\n type TraceEvalRunRecord,\n type TraceEvalSource,\n type TraceEvalStatus,\n} from \"./trace-eval-run-record\";\n\nexport type {\n TraceEvalAgentMessage,\n TraceEvalSource,\n TraceEvalStatus,\n} from \"./trace-eval-run-record\";\n\nexport type TraceItemKind = \"user\" | \"model\" | \"tool\" | \"source\" | \"usage\";\n\nexport interface TraceEvalTraceItem {\n detail: string;\n id: string;\n kind: TraceItemKind;\n metric?: string;\n status: \"pending\" | \"running\" | \"passed\" | \"failed\";\n title: string;\n}\n\nexport interface TraceEvalCheck {\n detail: string;\n id: string;\n status: \"passed\" | \"failed\" | \"running\" | \"skipped\";\n title: string;\n}\n\nexport interface TraceEvalSummary {\n failed: number;\n passed: number;\n skipped: number;\n total: number;\n}\n\nexport interface TraceEvalSnapshot {\n checks: TraceEvalCheck[];\n durationMs?: number;\n latestAnswer: string;\n latestPrompt: string | null;\n runId?: string;\n score: number;\n sources: TraceEvalSource[];\n status: TraceEvalStatus;\n summary: TraceEvalSummary;\n totalTokens?: number;\n trace: TraceEvalTraceItem[];\n}\n\nconst minimumAnswerLength = 120;\n\nfunction getModelTraceStatus(\n record: TraceEvalRunRecord\n): TraceEvalTraceItem[\"status\"] {\n if (!record.model) {\n return \"pending\";\n }\n\n return record.status === \"running\" ? \"running\" : \"passed\";\n}\n\nfunction buildTraceItems({\n record,\n}: {\n record: TraceEvalRunRecord;\n}): TraceEvalTraceItem[] {\n const trace: TraceEvalTraceItem[] = [];\n\n trace.push({\n detail:\n record.latestPrompt ?? \"No research request has been submitted yet.\",\n id: \"request\",\n kind: \"user\",\n status: record.hasResearchIntent ? \"passed\" : \"pending\",\n title: \"Research request\",\n });\n\n trace.push({\n detail: record.model\n ? `${record.model} streaming through AI Gateway.`\n : \"The model run will appear after the first assistant response starts.\",\n id: \"model\",\n kind: \"model\",\n status: getModelTraceStatus(record),\n title: \"Gateway model run\",\n });\n\n trace.push(\n ...record.searchCalls.map((call) => ({\n detail: call.input\n ? JSON.stringify(call.input)\n : \"Search input is still streaming.\",\n id: call.id,\n kind: \"tool\" as const,\n status: call.status,\n title: call.name,\n }))\n );\n\n if (record.sources.length > 0) {\n trace.push({\n detail: record.sources.map((source) => source.title).join(\", \"),\n id: \"sources\",\n kind: \"source\",\n metric: `${record.sources.length} sources`,\n status: \"passed\",\n title: \"Grounding sources\",\n });\n }\n\n if (record.usage?.totalTokens !== undefined) {\n trace.push({\n detail: [\n `Input ${record.usage.inputTokens ?? \"unknown\"}`,\n `Output ${record.usage.outputTokens ?? \"unknown\"}`,\n ].join(\" / \"),\n id: \"usage\",\n kind: \"usage\",\n metric: `${record.usage.totalTokens} tokens`,\n status: \"passed\",\n title: \"Token usage\",\n });\n }\n\n return trace;\n}\n\nfunction buildChecks({\n record,\n}: {\n record: TraceEvalRunRecord;\n}): TraceEvalCheck[] {\n const completedSearches = record.searchCalls.filter(\n (call) => call.status === \"passed\"\n );\n\n return [\n buildResearchQueryCheck(record.hasResearchIntent),\n buildSearchCheck({\n completedSearchCount: completedSearches.length,\n hasPrompt: record.hasResearchIntent,\n hasSearchInFlight:\n record.status === \"running\" && record.searchCalls.length > 0,\n }),\n buildSourceCoverageCheck({\n hasPrompt: record.hasResearchIntent,\n sourceCount: record.sources.length,\n }),\n buildAnswerShapeCheck({\n answerLength: record.latestAnswer.length,\n hasPrompt: record.hasResearchIntent,\n isBusy: record.status === \"running\",\n }),\n ];\n}\n\nfunction buildResearchQueryCheck(hasPrompt: boolean): TraceEvalCheck {\n return {\n detail: hasPrompt\n ? \"A research question is present in the session.\"\n : \"Submit a research question instead of a casual chat turn to start the evaluation.\",\n id: \"research-query\",\n status: hasPrompt ? \"passed\" : \"skipped\",\n title: \"Research query present\",\n };\n}\n\nfunction buildSearchCheck({\n completedSearchCount,\n hasPrompt,\n hasSearchInFlight,\n}: {\n completedSearchCount: number;\n hasPrompt: boolean;\n hasSearchInFlight: boolean;\n}): TraceEvalCheck {\n let status: TraceEvalCheck[\"status\"] = \"skipped\";\n\n if (completedSearchCount > 0) {\n status = \"passed\";\n } else if (hasSearchInFlight) {\n status = \"running\";\n } else if (hasPrompt) {\n status = \"failed\";\n }\n\n return {\n detail:\n completedSearchCount > 0\n ? \"The agent used the configured Gateway web search tool.\"\n : \"The agent should call the configured web search tool before finalizing current facts.\",\n id: \"gateway-search\",\n status,\n title: \"Gateway web search executed\",\n };\n}\n\nfunction buildSourceCoverageCheck({\n hasPrompt,\n sourceCount,\n}: {\n hasPrompt: boolean;\n sourceCount: number;\n}): TraceEvalCheck {\n let status: TraceEvalCheck[\"status\"] = \"skipped\";\n\n if (sourceCount >= 2) {\n status = \"passed\";\n } else if (hasPrompt) {\n status = \"failed\";\n }\n\n return {\n detail:\n sourceCount >= 2\n ? `${sourceCount} source links are available on the latest assistant answer.`\n : \"The final answer should expose at least two source links.\",\n id: \"source-coverage\",\n status,\n title: \"Sources attached\",\n };\n}\n\nfunction buildAnswerShapeCheck({\n answerLength,\n hasPrompt,\n isBusy,\n}: {\n answerLength: number;\n hasPrompt: boolean;\n isBusy: boolean;\n}): TraceEvalCheck {\n let status: TraceEvalCheck[\"status\"] = \"skipped\";\n\n if (answerLength >= minimumAnswerLength) {\n status = \"passed\";\n } else if (isBusy) {\n status = \"running\";\n } else if (hasPrompt) {\n status = \"failed\";\n }\n\n return {\n detail:\n answerLength >= minimumAnswerLength\n ? \"The answer contains enough substance for review.\"\n : \"The response is still missing a substantive research summary.\",\n id: \"answer-shape\",\n status,\n title: \"Research answer shape\",\n };\n}\n\nfunction summarizeChecks(checks: TraceEvalCheck[]): TraceEvalSummary {\n return {\n failed: checks.filter((check) => check.status === \"failed\").length,\n passed: checks.filter((check) => check.status === \"passed\").length,\n skipped: checks.filter((check) => check.status === \"skipped\").length,\n total: checks.length,\n };\n}\n\nexport function buildTraceEvalSnapshotFromRunRecord(\n record: TraceEvalRunRecord\n): TraceEvalSnapshot {\n const checks = buildChecks({\n record,\n });\n const summary = summarizeChecks(checks);\n\n return {\n checks,\n durationMs: record.durationMs,\n latestAnswer: record.latestAnswer,\n latestPrompt: record.latestPrompt,\n runId: record.runId,\n score: summary.total === 0 ? 0 : summary.passed / summary.total,\n sources: record.sources,\n status: record.status,\n summary,\n totalTokens: record.totalTokens,\n trace: buildTraceItems({ record }),\n };\n}\n\nexport function buildTraceEvalSnapshot(\n messages: UIMessage[],\n isBusy: boolean\n): TraceEvalSnapshot {\n return buildTraceEvalSnapshotFromRunRecord(\n buildTraceEvalRunRecord(messages, isBusy)\n );\n}\n", "type": "registry:lib", "target": "@lib/trace-eval-agent/model/trace-eval-snapshot.ts" } ], "envVars": { "AI_GATEWAY_API_KEY": "", "AI_GATEWAY_BASE_URL": "https://ai-gateway.vercel.sh/v3/ai", "AI_GATEWAY_CHAT_MODEL": "openai/gpt-5-mini", "TRACE_EVAL_AGENT_CHAT_MODEL": "openai/gpt-5-mini" }, "docs": "Install into a Next.js App Router project initialized with shadcn/ui. This item adds the demo page, research chat route, deterministic eval route, streamed judge route, client workspace, and AI Gateway env vars required for live search plus trace and eval inspection.", "type": "registry:block" }