Queue an eval run

curl --request POST \
  --url https://gateway.errorbar.ai/v1/evals \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data @- <<EOF
{
  "name": "support-bot: 32B vs 235B",
  "baseline_model": "Qwen/Qwen3-32B",
  "candidate_models": [
    "Qwen/Qwen3-30B-A3B-Instruct-2507"
  ],
  "sample_count": 20,
  "screening": false,
  "rubric": "Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.",
  "rubric_type": "direct",
  "judge_model": "<string>",
  "sample_filters": {
    "tag": "support-bot",
    "model": "<string>",
    "segment": "<string>",
    "dataset_id": "<string>",
    "trace_replay": true
  },
  "eval_kind": "comparison",
  "criterion_id": "<string>",
  "assertions": [
    {
      "value": "<string>"
    }
  ],
  "max_output_tokens": 4096
}
EOF

import requests

url = "https://gateway.errorbar.ai/v1/evals"

payload = {
    "name": "support-bot: 32B vs 235B",
    "baseline_model": "Qwen/Qwen3-32B",
    "candidate_models": ["Qwen/Qwen3-30B-A3B-Instruct-2507"],
    "sample_count": 20,
    "screening": False,
    "rubric": "Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.",
    "rubric_type": "direct",
    "judge_model": "<string>",
    "sample_filters": {
        "tag": "support-bot",
        "model": "<string>",
        "segment": "<string>",
        "dataset_id": "<string>",
        "trace_replay": True
    },
    "eval_kind": "comparison",
    "criterion_id": "<string>",
    "assertions": [{ "value": "<string>" }],
    "max_output_tokens": 4096
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: 'support-bot: 32B vs 235B',
    baseline_model: 'Qwen/Qwen3-32B',
    candidate_models: ['Qwen/Qwen3-30B-A3B-Instruct-2507'],
    sample_count: 20,
    screening: false,
    rubric: 'Answers must be accurate, concise, and follow the system prompt\'s tone. Penalize hallucinated product features.',
    rubric_type: 'direct',
    judge_model: '<string>',
    sample_filters: {
      tag: 'support-bot',
      model: '<string>',
      segment: '<string>',
      dataset_id: '<string>',
      trace_replay: true
    },
    eval_kind: 'comparison',
    criterion_id: '<string>',
    assertions: [{value: '<string>'}],
    max_output_tokens: 4096
  })
};

fetch('https://gateway.errorbar.ai/v1/evals', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://gateway.errorbar.ai/v1/evals",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => 'support-bot: 32B vs 235B',
    'baseline_model' => 'Qwen/Qwen3-32B',
    'candidate_models' => [
        'Qwen/Qwen3-30B-A3B-Instruct-2507'
    ],
    'sample_count' => 20,
    'screening' => false,
    'rubric' => 'Answers must be accurate, concise, and follow the system prompt\'s tone. Penalize hallucinated product features.',
    'rubric_type' => 'direct',
    'judge_model' => '<string>',
    'sample_filters' => [
        'tag' => 'support-bot',
        'model' => '<string>',
        'segment' => '<string>',
        'dataset_id' => '<string>',
        'trace_replay' => true
    ],
    'eval_kind' => 'comparison',
    'criterion_id' => '<string>',
    'assertions' => [
        [
                'value' => '<string>'
        ]
    ],
    'max_output_tokens' => 4096
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://gateway.errorbar.ai/v1/evals"

	payload := strings.NewReader("{\n  \"name\": \"support-bot: 32B vs 235B\",\n  \"baseline_model\": \"Qwen/Qwen3-32B\",\n  \"candidate_models\": [\n    \"Qwen/Qwen3-30B-A3B-Instruct-2507\"\n  ],\n  \"sample_count\": 20,\n  \"screening\": false,\n  \"rubric\": \"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.\",\n  \"rubric_type\": \"direct\",\n  \"judge_model\": \"<string>\",\n  \"sample_filters\": {\n    \"tag\": \"support-bot\",\n    \"model\": \"<string>\",\n    \"segment\": \"<string>\",\n    \"dataset_id\": \"<string>\",\n    \"trace_replay\": true\n  },\n  \"eval_kind\": \"comparison\",\n  \"criterion_id\": \"<string>\",\n  \"assertions\": [\n    {\n      \"value\": \"<string>\"\n    }\n  ],\n  \"max_output_tokens\": 4096\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://gateway.errorbar.ai/v1/evals")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"support-bot: 32B vs 235B\",\n  \"baseline_model\": \"Qwen/Qwen3-32B\",\n  \"candidate_models\": [\n    \"Qwen/Qwen3-30B-A3B-Instruct-2507\"\n  ],\n  \"sample_count\": 20,\n  \"screening\": false,\n  \"rubric\": \"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.\",\n  \"rubric_type\": \"direct\",\n  \"judge_model\": \"<string>\",\n  \"sample_filters\": {\n    \"tag\": \"support-bot\",\n    \"model\": \"<string>\",\n    \"segment\": \"<string>\",\n    \"dataset_id\": \"<string>\",\n    \"trace_replay\": true\n  },\n  \"eval_kind\": \"comparison\",\n  \"criterion_id\": \"<string>\",\n  \"assertions\": [\n    {\n      \"value\": \"<string>\"\n    }\n  ],\n  \"max_output_tokens\": 4096\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://gateway.errorbar.ai/v1/evals")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"support-bot: 32B vs 235B\",\n  \"baseline_model\": \"Qwen/Qwen3-32B\",\n  \"candidate_models\": [\n    \"Qwen/Qwen3-30B-A3B-Instruct-2507\"\n  ],\n  \"sample_count\": 20,\n  \"screening\": false,\n  \"rubric\": \"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.\",\n  \"rubric_type\": \"direct\",\n  \"judge_model\": \"<string>\",\n  \"sample_filters\": {\n    \"tag\": \"support-bot\",\n    \"model\": \"<string>\",\n    \"segment\": \"<string>\",\n    \"dataset_id\": \"<string>\",\n    \"trace_replay\": true\n  },\n  \"eval_kind\": \"comparison\",\n  \"criterion_id\": \"<string>\",\n  \"assertions\": [\n    {\n      \"value\": \"<string>\"\n    }\n  ],\n  \"max_output_tokens\": 4096\n}"

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "name": "<string>",
  "rubric": "<string>",
  "rubric_type": "direct",
  "eval_kind": "comparison",
  "criterion_snapshot": {},
  "baseline_model": "<string>",
  "candidate_models": [
    "<string>"
  ],
  "judge_model": "<string>",
  "sample_count": 123,
  "gen_max_output_tokens": 123,
  "sample_filters": {},
  "status": "PENDING",
  "error": "<string>",
  "assertions": "<array>",
  "progress_ratio": 123,
  "results": {},
  "created_at": "2023-11-07T05:31:56Z"
}

{
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>"
  }
}

{
  "error": {
    "message": "Invalid API key",
    "type": "invalid_request_error",
    "code": "invalid_api_key"
  }
}

{
  "error": {
    "message": "Insufficient balance: deploying this endpoint requires at least 1h of runway. Top up and try again.",
    "type": "insufficient_quota",
    "code": "insufficient_balance"
  }
}

{
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>"
  }
}

{
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>"
  }
}

Evals API

Queue an eval run

Compare a baseline model against up to 3 candidates on your own prompts, scored by an LLM judge.

Prompts are sampled from your logged traffic (or a managed dataset). Every model generates a fresh answer to each prompt — the baseline included — so you’re comparing models, not comparing a new answer against a stale logged one. Each pair is judged twice with the answers swapped; a candidate must win both orderings to score a win (position bias is the classic LLM-judge failure).

Requires an owner/admin key: generations and judging bill your wallet as normal usage. The run is queued and processed in the background — poll GET /v1/evals/{id} until status is DONE or ERROR.

Requires request logging to be on, with at least 5 distinct captured prompts matching your filter.

POST

evals

Queue an eval run

curl --request POST \
  --url https://gateway.errorbar.ai/v1/evals \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data @- <<EOF
{
  "name": "support-bot: 32B vs 235B",
  "baseline_model": "Qwen/Qwen3-32B",
  "candidate_models": [
    "Qwen/Qwen3-30B-A3B-Instruct-2507"
  ],
  "sample_count": 20,
  "screening": false,
  "rubric": "Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.",
  "rubric_type": "direct",
  "judge_model": "<string>",
  "sample_filters": {
    "tag": "support-bot",
    "model": "<string>",
    "segment": "<string>",
    "dataset_id": "<string>",
    "trace_replay": true
  },
  "eval_kind": "comparison",
  "criterion_id": "<string>",
  "assertions": [
    {
      "value": "<string>"
    }
  ],
  "max_output_tokens": 4096
}
EOF

import requests

url = "https://gateway.errorbar.ai/v1/evals"

payload = {
    "name": "support-bot: 32B vs 235B",
    "baseline_model": "Qwen/Qwen3-32B",
    "candidate_models": ["Qwen/Qwen3-30B-A3B-Instruct-2507"],
    "sample_count": 20,
    "screening": False,
    "rubric": "Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.",
    "rubric_type": "direct",
    "judge_model": "<string>",
    "sample_filters": {
        "tag": "support-bot",
        "model": "<string>",
        "segment": "<string>",
        "dataset_id": "<string>",
        "trace_replay": True
    },
    "eval_kind": "comparison",
    "criterion_id": "<string>",
    "assertions": [{ "value": "<string>" }],
    "max_output_tokens": 4096
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: 'support-bot: 32B vs 235B',
    baseline_model: 'Qwen/Qwen3-32B',
    candidate_models: ['Qwen/Qwen3-30B-A3B-Instruct-2507'],
    sample_count: 20,
    screening: false,
    rubric: 'Answers must be accurate, concise, and follow the system prompt\'s tone. Penalize hallucinated product features.',
    rubric_type: 'direct',
    judge_model: '<string>',
    sample_filters: {
      tag: 'support-bot',
      model: '<string>',
      segment: '<string>',
      dataset_id: '<string>',
      trace_replay: true
    },
    eval_kind: 'comparison',
    criterion_id: '<string>',
    assertions: [{value: '<string>'}],
    max_output_tokens: 4096
  })
};

fetch('https://gateway.errorbar.ai/v1/evals', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://gateway.errorbar.ai/v1/evals",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => 'support-bot: 32B vs 235B',
    'baseline_model' => 'Qwen/Qwen3-32B',
    'candidate_models' => [
        'Qwen/Qwen3-30B-A3B-Instruct-2507'
    ],
    'sample_count' => 20,
    'screening' => false,
    'rubric' => 'Answers must be accurate, concise, and follow the system prompt\'s tone. Penalize hallucinated product features.',
    'rubric_type' => 'direct',
    'judge_model' => '<string>',
    'sample_filters' => [
        'tag' => 'support-bot',
        'model' => '<string>',
        'segment' => '<string>',
        'dataset_id' => '<string>',
        'trace_replay' => true
    ],
    'eval_kind' => 'comparison',
    'criterion_id' => '<string>',
    'assertions' => [
        [
                'value' => '<string>'
        ]
    ],
    'max_output_tokens' => 4096
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://gateway.errorbar.ai/v1/evals"

	payload := strings.NewReader("{\n  \"name\": \"support-bot: 32B vs 235B\",\n  \"baseline_model\": \"Qwen/Qwen3-32B\",\n  \"candidate_models\": [\n    \"Qwen/Qwen3-30B-A3B-Instruct-2507\"\n  ],\n  \"sample_count\": 20,\n  \"screening\": false,\n  \"rubric\": \"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.\",\n  \"rubric_type\": \"direct\",\n  \"judge_model\": \"<string>\",\n  \"sample_filters\": {\n    \"tag\": \"support-bot\",\n    \"model\": \"<string>\",\n    \"segment\": \"<string>\",\n    \"dataset_id\": \"<string>\",\n    \"trace_replay\": true\n  },\n  \"eval_kind\": \"comparison\",\n  \"criterion_id\": \"<string>\",\n  \"assertions\": [\n    {\n      \"value\": \"<string>\"\n    }\n  ],\n  \"max_output_tokens\": 4096\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://gateway.errorbar.ai/v1/evals")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"support-bot: 32B vs 235B\",\n  \"baseline_model\": \"Qwen/Qwen3-32B\",\n  \"candidate_models\": [\n    \"Qwen/Qwen3-30B-A3B-Instruct-2507\"\n  ],\n  \"sample_count\": 20,\n  \"screening\": false,\n  \"rubric\": \"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.\",\n  \"rubric_type\": \"direct\",\n  \"judge_model\": \"<string>\",\n  \"sample_filters\": {\n    \"tag\": \"support-bot\",\n    \"model\": \"<string>\",\n    \"segment\": \"<string>\",\n    \"dataset_id\": \"<string>\",\n    \"trace_replay\": true\n  },\n  \"eval_kind\": \"comparison\",\n  \"criterion_id\": \"<string>\",\n  \"assertions\": [\n    {\n      \"value\": \"<string>\"\n    }\n  ],\n  \"max_output_tokens\": 4096\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://gateway.errorbar.ai/v1/evals")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"support-bot: 32B vs 235B\",\n  \"baseline_model\": \"Qwen/Qwen3-32B\",\n  \"candidate_models\": [\n    \"Qwen/Qwen3-30B-A3B-Instruct-2507\"\n  ],\n  \"sample_count\": 20,\n  \"screening\": false,\n  \"rubric\": \"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features.\",\n  \"rubric_type\": \"direct\",\n  \"judge_model\": \"<string>\",\n  \"sample_filters\": {\n    \"tag\": \"support-bot\",\n    \"model\": \"<string>\",\n    \"segment\": \"<string>\",\n    \"dataset_id\": \"<string>\",\n    \"trace_replay\": true\n  },\n  \"eval_kind\": \"comparison\",\n  \"criterion_id\": \"<string>\",\n  \"assertions\": [\n    {\n      \"value\": \"<string>\"\n    }\n  ],\n  \"max_output_tokens\": 4096\n}"

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "name": "<string>",
  "rubric": "<string>",
  "rubric_type": "direct",
  "eval_kind": "comparison",
  "criterion_snapshot": {},
  "baseline_model": "<string>",
  "candidate_models": [
    "<string>"
  ],
  "judge_model": "<string>",
  "sample_count": 123,
  "gen_max_output_tokens": 123,
  "sample_filters": {},
  "status": "PENDING",
  "error": "<string>",
  "assertions": "<array>",
  "progress_ratio": 123,
  "results": {},
  "created_at": "2023-11-07T05:31:56Z"
}

{
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>"
  }
}

{
  "error": {
    "message": "Invalid API key",
    "type": "invalid_request_error",
    "code": "invalid_api_key"
  }
}

{
  "error": {
    "message": "Insufficient balance: deploying this endpoint requires at least 1h of runway. Top up and try again.",
    "type": "insufficient_quota",
    "code": "insufficient_balance"
  }
}

{
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>"
  }
}

{
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>"
  }
}

Authorizations

Authorization

string

header

required

Your workspace API key, e.g. sk_sovereign_..., sent as Authorization: Bearer <key>.

Body

application/json

Snake_case body. Comparison runs also require rubric. Criterion runs require criterion_id. With screening: true, no field is required at all.

name

string

required

Maximum string length: 80

Example:

"support-bot: 32B vs 235B"

baseline_model

string

required

The model you run today — or the literal __stored__ to use your logged answers as the baseline arm (the certified-switch shape): candidates are judged against what you actually shipped, a calibrated judge criterion is required, population matching is strict (a mismatched judge/sample population refuses instead of warning), and the gate applies a noninferiority margin scaled by the judge's measured error. An underpowered run answers 412 rather than concluding.

Example:

"Qwen/Qwen3-32B"

candidate_models

string[]

required

Up to 6. Every candidate answers every sample and is judged, so cost is linear in this number. Comparison runs need at least 1; a criterion run may grade the baseline alone. A registered customer endpoint is a candidate too: endpoint/<name> (see POST /v1/endpoints) — it generates through your URL at $0 and is judged exactly like a catalog model.

Required array length: 1 - 6 elements

Example:

["Qwen/Qwen3-30B-A3B-Instruct-2507"]

sample_count

integer

required

Required range: 5 <= x <= 500

Example:

20

screening

boolean

default:false

Screening mode: the server auto-fills the run like the dashboard's one-click screening (incumbent = your dominant logged model, baseline = its stored answers, candidates = the cheapest model of each distinct family, fixed quality rubric, sample_count = min(40, population)). With screening: true every other field becomes an optional override; name, sample_count, max_output_tokens, candidate_models (1-6, validated against the catalog), and sample_filters.dataset_id are honored.

rubric

string

What "good" means for this task. The judge uses it verbatim. Required for comparison runs; ignored for criterion runs (the criterion's judge prompt is the rubric).

Required string length: 10 - 2000

Example:

"Answers must be accurate, concise, and follow the system prompt's tone. Penalize hallucinated product features."

rubric_type

enum<string>

default:direct

direct: which answer better satisfies the rubric. adherence: which answer is closer to the stored reference (the logged reply, or the dataset's assistant turn) — use this to check a cheaper or fine-tuned model against answers you already trust.

Available options:

direct,

adherence

judge_model

string

Defaults to a strong instruct model. Prefer a judge from a different family than your candidates — LLM judges measurably favour their own family.

sample_filters

object

Which prompts to run. Omit to sample recent logged traffic. Keys: tag (X-Errorbar-Tag), model (sample only that model's traffic), segment (an auto-detected traffic segment, a segment value from GET /v1/logs), dataset_id (sample a managed dataset, e.g. a holdout split, instead of live logs), trace_replay (replay bake-off: sample whole completed agent runs, one teacher-forced sample per step; requires baseline_model "stored").

Show child attributes

eval_kind

enum<string>

default:comparison

comparison: pairwise judging of candidates against the baseline. criterion: every model is graded pass/fail against a validated criterion, reported with the Rogan–Gladen correction from your alignment run.

Available options:

comparison,

criterion

criterion_id

string

Required when eval_kind is criterion. The run freezes the criterion's judge prompt, judge model, and alignment metrics at creation — a later re-alignment never rewrites a finished run's corrected rates.

assertions

object[]

Deterministic checks run against every generated response at finalize (final answer, reasoning stripped). Free, exact, and CI-gateable via GET /v1/evals/{id}/gate. Configs are validated at create time: an uncompilable regex or unparseable schema is a 400, never a queued-broken run.

Maximum array length: 10

Show child attributes

max_output_tokens

integer

default:4096

Per-answer generation output cap. Sized by default so thinking models can finish reasoning AND answer — a candidate cut mid-reasoning loses comparisons for reasons unrelated to quality. The cost estimate shown before a run uses your traffic's MEASURED median tokens; this cap bounds the worst case.

Required range: 256 <= x <= 16384

Response

The queued run.

string

name

string

rubric

string

What "good" means, in plain English. The judge scores against this.

rubric_type

enum<string>

Available options:

direct,

adherence

eval_kind

enum<string>

comparison: pairwise judging against the baseline. criterion: a saved judge grades each model pass/fail.

Available options:

comparison,

criterion

criterion_snapshot

object | null

Criterion runs only: the judge frozen at creation (name, prompt, and its TPR/TNR/kappa at that moment), so the run's corrected rates stay reproducible after the judge changes.

baseline_model

string

candidate_models

string[]

judge_model

string

sample_count

integer

gen_max_output_tokens

integer

Per-answer generation cap the run was created with (see max_output_tokens on create).

sample_filters

object

Where the prompts came from: tag, model, segment, dataset_id, trace_replay, and screening (set on screening runs). Echoes what the run was created with.

status

enum<string>

Available options:

PENDING,

RUNNING,

DONE,

ERROR,

CANCELLED

error

string | null

assertions

array | null

The deterministic assertion configs the run was created with, when any.

progress_ratio

number

0–1. Completed inference calls over total.

results

object | null

Present when status is DONE. Comparison runs: sample_count, clipped_samples, baseline {model, stored_answers, truncated, avg_latency_ms, eval_cost_micros}, per_candidate [{model, win_rate, wins, ties, losses, ci95, failed, attempted, unreportable, truncated, avg_latency_ms, eval_cost_micros, savings_pct, replay?}], judge_cost_micros, and screening (screening runs only: incumbent, token_shape, per_candidate similarity + estimated cost placement, recommendation). Criterion runs: per_model [{model, observed_pass_rate, observed_ci, corrected_pass_rate, corrected_ci, ...}] using the frozen judge's measured error rates. Assertion runs add assertion_results.

created_at

string<date-time>

Overview

Inference API

Capture & setup API

Logs & datasets API

Grades (labels) API

Judges (criteria) API

Evals API

Alerts API

Fine-tuning API

Certified reward API

Reinforcement learning API

Dedicated endpoints API

Batches API

Model aliases API

Model versions API

Audit & proving API

Queue an eval run

Authorizations

Body

Response