Eval runs
Create an eval suite (author-only, does not run)
Creates a runnable eval suite — the suite record plus its test cases — and responds 201 synchronously, WITHOUT executing anything. Use this to author a suite, then run it later with POST /eval-runs (passing the returned suiteId).
This is distinct from POST /eval-runs, which creates a run and detaches execution, responding 202 with a runId. There is no concurrency cap here (no run is started).
The body uses an ergonomic authoring shape: a suite-level default model (and optional provider) applies to every test unless the test overrides it; provider is derived from a provider/model id when neither is supplied. Each test’s case body is an ordered steps array (prompt / toolCall / interact / assert).
Guest callers are denied (suite creation is a write).
POST
/
projects
/
{projectId}
/
eval-suites
Create an eval suite (author-only, does not run)
curl --request POST \
--url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "smoke",
"serverIds": [
"srv_abc123"
],
"model": "anthropic/claude-haiku-4.5",
"tests": [
{
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": {
"args": {}
}
}
}
]
}
]
}
'import requests
url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"
payload = {
"name": "smoke",
"serverIds": ["srv_abc123"],
"model": "anthropic/claude-haiku-4.5",
"tests": [
{
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": { "args": {} }
}
}
]
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: 'smoke',
serverIds: ['srv_abc123'],
model: 'anthropic/claude-haiku-4.5',
tests: [
{
title: 'lists files',
steps: [
{id: 's1', kind: 'prompt', prompt: 'List the files in the project root.'},
{
id: 's2',
kind: 'assert',
assertion: {type: 'toolCalledWith', toolName: 'list_files', args: {args: {}}}
}
]
}
]
})
};
fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => 'smoke',
'serverIds' => [
'srv_abc123'
],
'model' => 'anthropic/claude-haiku-4.5',
'tests' => [
[
'title' => 'lists files',
'steps' => [
[
'id' => 's1',
'kind' => 'prompt',
'prompt' => 'List the files in the project root.'
],
[
'id' => 's2',
'kind' => 'assert',
'assertion' => [
'type' => 'toolCalledWith',
'toolName' => 'list_files',
'args' => [
'args' => [
]
]
]
]
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"
payload := strings.NewReader("{\n \"name\": \"smoke\",\n \"serverIds\": [\n \"srv_abc123\"\n ],\n \"model\": \"anthropic/claude-haiku-4.5\",\n \"tests\": [\n {\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"smoke\",\n \"serverIds\": [\n \"srv_abc123\"\n ],\n \"model\": \"anthropic/claude-haiku-4.5\",\n \"tests\": [\n {\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"smoke\",\n \"serverIds\": [\n \"srv_abc123\"\n ],\n \"model\": \"anthropic/claude-haiku-4.5\",\n \"tests\": [\n {\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"suiteId": "<string>",
"name": "<string>",
"caseUpsert": {
"committed": [
{
"id": "<string>",
"name": "<string>"
}
],
"failed": [
{
"id": "<string>",
"name": "<string>",
"error": "<string>"
}
]
},
"servers": [
{
"id": "<string>",
"name": "<string>"
}
]
}
Authorizations
MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.
Path Parameters
ID of the hosted project that contains the server.
Body
application/json
Author-only suite-create body. A suite-level default model (and optional provider) applies to every test unless the test overrides it.
Suite name.
Minimum string length:
1Servers (by canonical project ID) the suite's cases run against.
Minimum array length:
1Suite-level default model id (e.g. anthropic/claude-haiku-4.5). Used for any test that omits model.
Minimum string length:
1Test cases to create in the suite.
Required array length:
1 - 100 elementsShow child attributes
Show child attributes
Optional display names, parallel to serverIds.
Optional suite-level default provider. When omitted, the provider is derived from a provider/model id.
Show child attributes
Show child attributes
Accepted for forward-compat; not persisted today (no-op).
Response
The suite was created.
Per-case create outcomes. Partial failures don't abort the suite; an all-failed new suite is rejected with VALIDATION_ERROR.
Show child attributes
Show child attributes
The servers attached to the suite. name is present when supplied.
Show child attributes
Show child attributes
Was this page helpful?
Previous
Import OAuth tokensStores OAuth tokens you obtained yourself (e.g. via the SDK's `runOAuthLogin` — interactive loopback, headless, or client-credentials) for this server, scoped to your user, project, and server. Subsequent API calls against the server inject the stored access token automatically, and `401`s from the server trigger a server-side refresh — no further caller involvement.
This closes the loop after an `OAUTH_REQUIRED` error.
Next
⌘I
Create an eval suite (author-only, does not run)
curl --request POST \
--url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "smoke",
"serverIds": [
"srv_abc123"
],
"model": "anthropic/claude-haiku-4.5",
"tests": [
{
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": {
"args": {}
}
}
}
]
}
]
}
'import requests
url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"
payload = {
"name": "smoke",
"serverIds": ["srv_abc123"],
"model": "anthropic/claude-haiku-4.5",
"tests": [
{
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": { "args": {} }
}
}
]
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: 'smoke',
serverIds: ['srv_abc123'],
model: 'anthropic/claude-haiku-4.5',
tests: [
{
title: 'lists files',
steps: [
{id: 's1', kind: 'prompt', prompt: 'List the files in the project root.'},
{
id: 's2',
kind: 'assert',
assertion: {type: 'toolCalledWith', toolName: 'list_files', args: {args: {}}}
}
]
}
]
})
};
fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => 'smoke',
'serverIds' => [
'srv_abc123'
],
'model' => 'anthropic/claude-haiku-4.5',
'tests' => [
[
'title' => 'lists files',
'steps' => [
[
'id' => 's1',
'kind' => 'prompt',
'prompt' => 'List the files in the project root.'
],
[
'id' => 's2',
'kind' => 'assert',
'assertion' => [
'type' => 'toolCalledWith',
'toolName' => 'list_files',
'args' => [
'args' => [
]
]
]
]
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"
payload := strings.NewReader("{\n \"name\": \"smoke\",\n \"serverIds\": [\n \"srv_abc123\"\n ],\n \"model\": \"anthropic/claude-haiku-4.5\",\n \"tests\": [\n {\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"smoke\",\n \"serverIds\": [\n \"srv_abc123\"\n ],\n \"model\": \"anthropic/claude-haiku-4.5\",\n \"tests\": [\n {\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"smoke\",\n \"serverIds\": [\n \"srv_abc123\"\n ],\n \"model\": \"anthropic/claude-haiku-4.5\",\n \"tests\": [\n {\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"suiteId": "<string>",
"name": "<string>",
"caseUpsert": {
"committed": [
{
"id": "<string>",
"name": "<string>"
}
],
"failed": [
{
"id": "<string>",
"name": "<string>",
"error": "<string>"
}
]
},
"servers": [
{
"id": "<string>",
"name": "<string>"
}
]
}