Saving Eval Results

import { MCPClientManager, reportEvalResults } from "@mcpjam/sdk";

reportEvalResults(input: ReportEvalResultsInput): Promise<ReportEvalResultsOutput>

const manager = new MCPClientManager({
  asana: {
    url: process.env.MCP_SERVER_URL!,
    refreshToken: process.env.MCP_REFRESH_TOKEN!,
    clientId: process.env.MCP_CLIENT_ID!,
    clientSecret: process.env.MCP_CLIENT_SECRET,
  },
});

await manager.connectToServer("asana");

const output = await reportEvalResults({
  suiteName: "Nightly",
  mcpClientManager: manager,
  results: [
    { caseTitle: "healthcheck", passed: true },
    { caseTitle: "tool-selection", passed: true, durationMs: 1200 },
    { caseTitle: "edge-case", passed: false, error: "Wrong tool called" },
  ],
  passCriteria: { minimumPassRate: 90 },
  ci: {
    branch: "main",
    commitSha: "abc123",
  },
});

console.log(`Run ${output.runId}: ${output.result}`);
// "Run abc123: passed"
console.log(`${output.summary.passed}/${output.summary.total} passed`);

import { MCPClientManager, reportEvalResultsSafely } from "@mcpjam/sdk";

reportEvalResultsSafely(input: ReportEvalResultsInput): Promise<ReportEvalResultsOutput | null>

const manager = new MCPClientManager({
  asana: {
    url: process.env.MCP_SERVER_URL!,
    refreshToken: process.env.MCP_REFRESH_TOKEN!,
    clientId: process.env.MCP_CLIENT_ID!,
  },
});

await manager.connectToServer("asana");

const output = await reportEvalResultsSafely({
  suiteName: "Nightly",
  mcpClientManager: manager,
  results: [{ caseTitle: "healthcheck", passed: true }],
});

if (output) {
  console.log(`Reported: ${output.summary.passRate * 100}% pass rate`);
} else {
  console.log("Reporting failed (non-blocking)");
}

import { createEvalRunReporter } from "@mcpjam/sdk";

createEvalRunReporter(input: CreateEvalRunReporterInput): EvalRunReporter

// Assumes `agent` was created with `mcpClientManager: manager`
const reporter = createEvalRunReporter({
  suiteName: "Integration Tests",
  passCriteria: { minimumPassRate: 85 },
  agent,
  ci: {
    branch: process.env.GITHUB_REF_NAME,
    commitSha: process.env.GITHUB_SHA,
  },
});

const reporter = createEvalRunReporter({
  suiteName: "Integration Tests",
  passCriteria: { minimumPassRate: 85 },
  mcpClientManager: manager,
});

// Add results as tests complete
await reporter.record({ caseTitle: "test-1", passed: true, durationMs: 500 });
await reporter.record({ caseTitle: "test-2", passed: false, error: "timeout" });
await reporter.record({ caseTitle: "test-3", passed: true });

// Finalize the run
const output = await reporter.finalize();
console.log(`${output.summary.passed}/${output.summary.total} passed`);

// Pass `agent` or `mcpClientManager` when you need server replay metadata in MCPJam
const reporter = createEvalRunReporter({ suiteName: "Prompt Tests", agent });

const result = await agent.run("Add 2 and 3");
reporter.addFromPrompt(result, {
  caseTitle: "addition",
  passed: result.hasToolCall("add"),
});

const output = await reporter.finalize();

const reporter = createEvalRunReporter({ suiteName: "Full Suite" });

const test = new EvalTest({
  name: "addition",
  test: async (agent) => (await agent.run("Add 2+3")).hasToolCall("add"),
});

const run = await test.run(agent, { iterations: 10 });
await reporter.recordFromRun(run, { casePrefix: "addition" });

const output = await reporter.finalize();

import { uploadEvalArtifact } from "@mcpjam/sdk";

uploadEvalArtifact(input: UploadEvalArtifactInput): Promise<ReportEvalResultsOutput>

import { readFileSync } from "fs";

// Upload JUnit XML
await uploadEvalArtifact({
  suiteName: "CI Results",
  format: "junit-xml",
  artifact: readFileSync("test-results.xml", "utf-8"),
});

// Upload Jest JSON
await uploadEvalArtifact({
  suiteName: "Jest Results",
  format: "jest-json",
  artifact: readFileSync("jest-results.json", "utf-8"),
});

// Custom parser
await uploadEvalArtifact({
  suiteName: "Custom",
  format: "custom",
  artifact: myData,
  customParser: (data) => [
    { caseTitle: "test-1", passed: true },
    { caseTitle: "test-2", passed: false, error: "failed" },
  ],
});

type ReportEvalResultsInput = MCPJamReportingConfig & {
  suiteName: string;
  results: EvalResultInput[];
  agent?: {
    getServerReplayConfigs?: () => MCPServerReplayConfig[] | undefined;
  };
  mcpClientManager?: MCPClientManager;
};

POST /api/v1/projects/{projectId}/eval-ingest/runs/start
{
  "suiteName": "...",
  "results": [...],
  "hostConfig":     {/* canonical HostConfigInputV2 with serverIds stripped */},
  "hostConfigHash": "sha256(canonical)"
}

import type { Predicate } from "@mcpjam/sdk/predicates";

const predicates: Predicate[] = [
  // Tool was called with the right airline argument
  {
    type: "toolCalledWith",
    toolName: "book_flight",
    args: { args: { airline: "DL" } },
  },
  // Response mentions the booking confirmation
  { type: "responseContains", needle: "confirmed" },
  // No tool errors occurred
  { type: "noToolErrors" },
  // Token budget respected
  { type: "tokenBudgetUnder", tokens: 2000 },
];

// Pass predicates via the per-run override in the Inspector corpus,
// or use evaluatePredicates() directly in custom pipelines:
import { evaluatePredicates, buildIterationTranscript } from "@mcpjam/sdk/predicates";

const transcript = buildIterationTranscript({
  trace: myTrace,
  toolCalls: myToolCalls,
  usage: { totalTokens: 1200 },
});

const results = evaluatePredicates(transcript, predicates);
const allPassed = results.every((r) => r.passed);

type PredicateResult = {
  predicate: Predicate;   // The authored predicate (sensitive arg keys are redacted)
  passed: boolean;
  reason: string;         // Structured explanation, e.g. "tool "book_flight" called with matching args"
};