{
"operations": [
{
"auth": "account",
"description": "Get setup status. Does not start an evaluation. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_setup_status",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/getting-started",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"environment": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "public",
"description": "List operations. Does not start an evaluation.",
"id": "list_operations",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/headless/operations",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Read credential authority, plan, model-selection entitlement, repository scope and pricing and upgrade links, and links to the current operation catalog and evaluation allowance. Does not grant permissions. Does not start an evaluation.",
"id": "capabilities",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/headless/capabilities",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Evaluation allowance. Does not start an evaluation.",
"id": "evaluation_allowance",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-allowance",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "public",
"description": "Get processing notice. Does not start an evaluation.",
"id": "get_processing_notice",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/legal/processing-notice",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"accepted": {
"type": "boolean"
},
"source": {
"type": "string"
},
"version": {
"type": "string"
}
},
"type": "object"
},
"description": "Record the user's explicit authorization for the current processing notice and selected source. Read get_processing_notice and show the notice first; never accept it on the user's behalf without permission. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "acknowledge_processing",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/legal/processing-acknowledgements",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"description": "Get login url. Does not start an evaluation.",
"id": "get_login_url",
"mcp": false,
"mcp_legacy": false,
"method": "GET",
"path": "/api/auth/login-url",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"challenge": {
"type": "string"
},
"provider": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"code": {
"type": "string"
},
"newsletter_consent": {
"type": "boolean"
},
"state": {
"type": "string"
},
"verifier": {
"type": "string"
}
},
"type": "object"
},
"description": "Complete login. Uses the same validation and plan limits as Bench.",
"id": "complete_login",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/callback",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"code": {
"type": "string"
},
"newsletter_consent": {
"type": "boolean"
},
"pending_authentication_token": {
"type": "string"
},
"provider": {
"type": "string"
}
},
"type": "object"
},
"description": "Verify email. Uses the same validation and plan limits as Bench.",
"id": "verify_email",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/verify-email",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"email": {
"type": "string"
},
"newsletter_consent": {
"type": "boolean"
},
"password": {
"type": "string"
}
},
"type": "object"
},
"description": "Login with password. Uses the same validation and plan limits as Bench.",
"id": "login_with_password",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/password",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"description": "Logout. Uses the same validation and plan limits as Bench.",
"id": "logout",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/logout",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"token": {
"type": "string"
}
},
"required": [
"token"
],
"type": "object"
},
"description": "Exchange oauth token. Uses the same validation and plan limits as Bench.",
"id": "exchange_oauth_token",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/mcp-token",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Whoami. Does not start an evaluation.",
"id": "whoami",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/auth/me",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"generation": {
"type": "integer"
}
},
"type": "object"
},
"description": "Record setup completion for the current account. Use onboarding_generation from whoami; a stale generation is rejected. No browser wizard is required. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "complete_onboarding",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/onboarding/complete",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Update profile. Uses the same validation and plan limits as Bench.",
"id": "update_profile",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/auth/me",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"providers": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"description": "Choose allowed model providers for future Bench evaluations. Requires an active Growth or Enterprise plan. Free and Builder use Bench defaults and cannot change this preference.",
"id": "set_model_providers",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/auth/me/model-providers",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"installation_id": {
"type": "integer"
},
"state": {
"type": "string"
}
},
"type": "object"
},
"description": "Receive github callback. Uses the same validation and plan limits as Bench.",
"id": "receive_github_callback",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/github/headless-callback",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Start GitHub repository authorization. Set headless=true to return to the coding agent after the user approves access on GitHub. Keep the returned state and poll finish_github_connection. GitHub installation approval is required; no Bench onboarding is needed.",
"id": "connect_github",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/install-url",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"headless": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"installation_id": {
"type": "integer"
},
"state": {
"type": "string"
}
},
"type": "object"
},
"description": "Complete a manual GitHub callback using the original account-bound state and installation_id. Prefer connect_github with headless=true and finish_github_connection. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "complete_github_callback",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/github/callback",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"state": {
"type": "string"
}
},
"type": "object"
},
"description": "Finish the connection started with connect_github(headless=true). Pass its state. A pending response means wait the returned poll_interval before calling again. On success, list_repos and choose a branch. This binds the installation to the signed-in user. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "finish_github_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/github/connection",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Connection status. Does not start an evaluation.",
"id": "connection_status",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/status",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"description": "Activate installation. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "activate_installation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/github/installations/{installationID}/activate",
"path_parameters": {
"installationID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List repos. Does not start an evaluation.",
"id": "list_repos",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/repos",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"all_accounts": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List branches. Does not start an evaluation.",
"id": "list_branches",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/repos/{owner}/{repo}/branches",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Upload prompts. Uses the same validation and plan limits as Bench.",
"file_field": "files",
"form": {
"additionalProperties": false,
"properties": {
"model": {
"type": "string"
},
"pasted_text": {
"type": "string"
}
},
"type": "object"
},
"id": "upload_prompts",
"max_file_bytes": 2097152,
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"multipart": true,
"path": "/api/prompt-scans",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Scan repo. Uses the same validation and plan limits as Bench.",
"id": "scan_repo",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/scan",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get scan. Does not start an evaluation.",
"id": "get_scan",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/scan/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get repository refresh. Does not start an evaluation.",
"id": "get_repository_refresh",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/scan/refresh",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Set repository refresh. Uses the same validation and plan limits as Bench.",
"id": "set_repository_refresh",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/repos/{owner}/{repo}/scan/refresh",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"call_site_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"model": {
"type": "string"
}
},
"type": "object"
},
"description": "Record the model used by uploaded prompt call sites. This changes source metadata, not the allowed-provider setting. Requires model-selection entitlement.",
"id": "set_prompt_models",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/scan/call-sites/model",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"base_branch": {
"type": "string"
},
"body": {
"type": "string"
},
"branch": {
"type": "string"
},
"changes": {
"items": {
"additionalProperties": false,
"properties": {
"content": {
"type": "string"
},
"path": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"title": {
"type": "string"
}
},
"type": "object"
},
"description": "Open prompt pr. Uses the same validation and plan limits as Bench.",
"id": "open_prompt_pr",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/prompt-pr",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"business_doc_text": {
"type": "string"
}
},
"type": "object"
},
"description": "Generate business context. Uses the same validation and plan limits as Bench.",
"id": "generate_business_context",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/business-context",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get business context. Does not start an evaluation.",
"id": "get_business_context",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/business-context/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"previous_routing_json": {
"type": "string"
},
"reference_today": {
"type": "string"
},
"suite_name": {
"type": "string"
}
},
"type": "object"
},
"description": "Generate eval benchmark. Uses the same validation and plan limits as Bench.",
"id": "generate_eval_benchmark",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/eval-benchmark",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false,
"streaming": true
},
{
"auth": "credential",
"description": "Get eval benchmark. Does not start an evaluation.",
"id": "get_eval_benchmark",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/eval-benchmark/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"call_site_id": {
"type": "string"
},
"test_cases": {}
},
"type": "object"
},
"description": "Review eval benchmark. Uses the same validation and plan limits as Bench.",
"id": "review_eval_benchmark",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/eval-benchmark/review",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"max_concurrency": {
"type": "integer"
},
"model": {
"type": "string"
},
"suite_name": {
"type": "string"
}
},
"type": "object"
},
"description": "Run baseline scoring for an existing reviewed benchmark. This can consume an evaluation credit; confirm spending first. Streamed events preserve progress and terminal errors. A model override requires Growth or Enterprise.",
"id": "run_baseline",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/baseline",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false,
"streaming": true
},
{
"auth": "credential",
"description": "Get baseline. Does not start an evaluation.",
"id": "get_baseline",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/baseline/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"candidate_models": {
"items": {
"additionalProperties": false,
"properties": {
"model_id": {
"type": "string"
},
"provider": {
"type": "string"
},
"tier": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"max_case_concurrency": {
"type": "integer"
},
"max_combo_concurrency": {
"type": "integer"
},
"max_rounds": {
"type": "integer"
},
"min_quality_score": {
"type": "number"
},
"suite_name": {
"type": "string"
},
"variant_count": {
"type": "integer"
}
},
"type": "object"
},
"description": "Evaluate prompt improvements after a baseline. Confirm evaluation spending first. Candidate lists are server controlled on all plans; use set_model_providers for eligible provider preferences.",
"id": "run_search_optimize",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/search-optimize",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false,
"streaming": true
},
{
"auth": "credential",
"description": "Get search optimize. Does not start an evaluation.",
"id": "get_search_optimize",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/search-optimize/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get recommendation. Does not start an evaluation.",
"id": "get_recommendation",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/recommend/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List golden cases. Does not start an evaluation.",
"id": "list_golden_cases",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/golden-cases",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"label": {
"type": "string"
},
"note": {
"type": "string"
}
},
"type": "object"
},
"description": "Label golden case. Uses the same validation and plan limits as Bench.",
"id": "label_golden_case",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/golden-cases/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"url": {
"type": "string"
}
},
"type": "object"
},
"description": "Fetch website text. Uses the same validation and plan limits as Bench.",
"id": "fetch_website_text",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/website-text",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"url": {
"type": "string"
}
},
"type": "object"
},
"description": "Get link metadata. Uses the same validation and plan limits as Bench.",
"id": "get_link_metadata",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/link-metadata",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_id": {
"type": "integer"
},
"branch": {
"type": "string"
},
"context_doc": {
"type": "string"
},
"generate_context": {
"type": "boolean"
},
"max_rounds": {
"type": "integer"
},
"min_quality_score": {
"type": "number"
},
"prompts": {
"items": {
"additionalProperties": false,
"properties": {
"call_site_id": {
"type": "string"
},
"workflow_name": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"description": "Start evaluation. Uses the same validation and plan limits as Bench.",
"id": "start_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Start system evaluation. Uses the same validation and plan limits as Bench.",
"id": "start_system_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/system",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List evaluations. Does not start an evaluation.",
"id": "list_evaluations",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"include_active": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"context_doc": {
"type": "string"
},
"generate_context": {
"type": "boolean"
},
"single_prompt": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Rerun evaluation. Uses the same validation and plan limits as Bench.",
"id": "rerun_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/{id}/rerun",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get evaluation. Does not start an evaluation.",
"id": "get_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get evaluation artifacts. Does not start an evaluation.",
"id": "get_evaluation_artifacts",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs/{id}/artifacts",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get fix brief. Does not start an evaluation.",
"id": "get_fix_brief",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs/{id}/fix-brief",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Cancel evaluation. Uses the same validation and plan limits as Bench.",
"id": "cancel_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/{id}/cancel",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Cancel bench. Uses the same validation and plan limits as Bench.",
"id": "cancel_bench",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/evaluation-runs/{id}/cancel-bench",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"test_cases": {}
},
"type": "object"
},
"description": "Submit run review. Uses the same validation and plan limits as Bench.",
"id": "submit_run_review",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/{id}/review",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Archive evaluation. Uses the same validation and plan limits as Bench.",
"id": "archive_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/evaluation-runs/{id}/archive",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Unarchive evaluation. Uses the same validation and plan limits as Bench.",
"id": "unarchive_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/evaluation-runs/{id}/unarchive",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"description": {
"type": "string"
},
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Create an AI system with a name and optional description. Add prompt, tool or harness components after scanning a repository or uploading prompts. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "create_system",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List systems. Does not start an evaluation.",
"id": "list_systems",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get system. Does not start an evaluation.",
"id": "get_system",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List runtime evaluations. Does not start an evaluation.",
"id": "list_runtime_evaluations",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runtime-evaluations",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"environment": {
"type": "string"
},
"offset": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": true,
"properties": {},
"type": "object"
},
"description": "Publish an SDK real app test report (maximum 500 KB). Execute the actual application with the SDK evaluateSystem/evaluate_system API first. Uploaded results remain client-reported evidence, not hosted verification.",
"id": "publish_runtime_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runtime-evaluations",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": {},
"type": "object"
},
"description": "Validate and persist an immutable application evaluation plan without paid execution. Pins versions, grouped cases, configurations, runtime and explicit budgets. Confirmation inputs and expected outcomes remain private.",
"id": "runtime_eval_plan",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-plans",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Read public plan counts and budget estimates. Confirmation content is never returned.",
"id": "runtime_eval_get_plan",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Read only development cases for human review; confirmation and calibration cases are excluded.",
"id": "runtime_eval_review_cases",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}/review",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"case_id": {
"type": "string"
},
"note": {
"type": "string"
},
"verdict": {
"type": "string"
}
},
"type": "object"
},
"description": "Record good, bad or skip feedback for a development case. Applying feedback requires a new immutable dataset and plan.",
"id": "runtime_eval_review",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}/review",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"allowed_providers": {
"items": {
"type": "string"
},
"type": "array"
},
"domains": {
"items": {
"type": "string"
},
"type": "array"
},
"run_id": {
"type": "string"
}
},
"type": "object"
},
"description": "Propose candidate configurations for a new plan from development trials and published model evidence across allowed providers. Reads no confirmation trials, runs no model, and never edits the saved plan; every proposal is unverified until measured.",
"id": "runtime_eval_candidates",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}/candidates",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"plan_id": {
"type": "string"
}
},
"type": "object"
},
"description": "Start a saved plan. Requires Idempotency-Key header (8\u2013200 characters), GitHub authorization, and evaluation allowance. Duplicate starts return the original run without repeating execution.",
"id": "runtime_eval_run",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-runs",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Read job status separately from evaluation decision, uncertainty, resource metrics and development evidence. No confirmation content is returned.",
"id": "runtime_eval_results",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-runs/{runID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"after": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Read durable events with action=events and optional after cursor. Other actions return the protected result.",
"id": "runtime_eval_events",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-runs/{runID}/{action}",
"path_parameters": {
"action": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"after": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"extend_budget": {
"additionalProperties": {},
"type": "object"
},
"retry_ambiguous_attempt": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Cancel a run, resume a safely checkpointed canceled run, or continue a paused run with an explicit extend_budget or an acknowledged retry_ambiguous_attempt. Used holdouts and plans needing more independent data are never automatically retried.",
"id": "runtime_eval_control",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-runs/{runID}/{action}",
"path_parameters": {
"action": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"after": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": null,
"description": "Read hosted runtime availability, settings and jobs. For local real app tests, use the SDK even when hosted execution is unavailable. Does not start an evaluation.",
"id": "get_runtime_settings",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runtime-settings",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"config": {
"additionalProperties": true,
"properties": {},
"type": "object"
},
"daily_limit": {
"type": "integer"
},
"enabled": {
"type": "boolean"
},
"repository": {
"type": "string"
}
},
"required": [
"repository",
"branch",
"enabled",
"daily_limit",
"config"
],
"type": "object"
},
"description": "Configure hosted real app execution for a connected system: repository, branch, enabled, daily_limit and config. Check get_runtime_settings for environment availability. Enabling it can execute application code and consume evaluation allowance; obtain user approval first. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_runtime_settings",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/runtime-settings",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Publish runtime fix. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "publish_runtime_fix",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runtime-jobs/{jobID}/publish",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Retry runtime job. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "retry_runtime_job",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runtime-jobs/{jobID}/retry",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Preview CSV, XLSX, JSON or JSONL and suggested column mapping without storing it. Upload the file bytes, optional sheet, mode and mapping. Inspect mapping_error before import. No model runs.",
"file_field": "file",
"form": {
"additionalProperties": false,
"properties": {
"content_consent": {
"type": "boolean"
},
"mapping": {
"additionalProperties": false,
"properties": {
"actual": {
"type": "string"
},
"expected": {
"type": "string"
},
"history": {
"type": "string"
},
"input": {
"type": "string"
},
"notes": {
"type": "string"
},
"resolved_prompt": {
"type": "string"
},
"template": {
"type": "string"
},
"variables": {
"type": "string"
}
},
"type": "object"
},
"mode": {
"enum": [
"classifier",
"recorded_outputs",
"replay"
],
"type": "string"
},
"name": {
"type": "string"
},
"sheet": {
"type": "string"
}
},
"type": "object"
},
"id": "preview_dataset",
"max_file_bytes": 4194304,
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"multipart": true,
"path": "/api/ai-systems/{id}/datasets/preview",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List test library. Does not start an evaluation.",
"id": "list_test_library",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/test-library",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"kind": {
"type": "string"
},
"offset": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"component_id": {
"type": "integer"
},
"dataset_id": {
"type": "string"
},
"dataset_version": {
"type": "integer"
},
"expected_version": {
"type": "integer"
}
},
"type": "object"
},
"description": "Import dataset cases. Uses the same validation and plan limits as Bench.",
"id": "import_dataset_cases",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/test-library/import",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Store a dataset up to 4 MB after the user approves content_consent=true and the column mapping. Preview first. A stored dataset becomes a test suite only when import_dataset_cases is called.",
"file_field": "file",
"form": {
"additionalProperties": false,
"properties": {
"content_consent": {
"type": "boolean"
},
"mapping": {
"additionalProperties": false,
"properties": {
"actual": {
"type": "string"
},
"expected": {
"type": "string"
},
"history": {
"type": "string"
},
"input": {
"type": "string"
},
"notes": {
"type": "string"
},
"resolved_prompt": {
"type": "string"
},
"template": {
"type": "string"
},
"variables": {
"type": "string"
}
},
"type": "object"
},
"mode": {
"enum": [
"classifier",
"recorded_outputs",
"replay"
],
"type": "string"
},
"name": {
"type": "string"
},
"sheet": {
"type": "string"
}
},
"type": "object"
},
"id": "upload_dataset",
"max_file_bytes": 4194304,
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"multipart": true,
"path": "/api/ai-systems/{id}/datasets",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List datasets. Does not start an evaluation.",
"id": "list_datasets",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/datasets",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get dataset. Does not start an evaluation.",
"id": "get_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/datasets/{datasetID}",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"version": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"expected_version": {
"type": "integer"
},
"mapping": {
"additionalProperties": false,
"properties": {
"actual": {
"type": "string"
},
"expected": {
"type": "string"
},
"history": {
"type": "string"
},
"input": {
"type": "string"
},
"notes": {
"type": "string"
},
"resolved_prompt": {
"type": "string"
},
"template": {
"type": "string"
},
"variables": {
"type": "string"
}
},
"type": "object"
},
"mode": {
"type": "string"
}
},
"required": [
"expected_version",
"mapping",
"mode"
],
"type": "object"
},
"description": "Create a new dataset version with a corrected column mapping. Supply expected_version from get_dataset. Old reports retain their original mapping.",
"id": "remap_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/datasets/{datasetID}/mapping",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"version": {
"type": "integer"
}
},
"type": "object"
},
"description": "Compare recorded outputs with expected values for an explicit dataset version. No model or live app is called. Live replay is not supported by this operation.",
"id": "compare_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/datasets/{datasetID}/compare",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List dataset reports. Does not start an evaluation.",
"id": "list_dataset_reports",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/datasets/{datasetID}/reports",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Delete dataset. Uses the same validation and plan limits as Bench.",
"id": "delete_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/datasets/{datasetID}",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get system context. Does not start an evaluation.",
"id": "get_system_context",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/context",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"version": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List context sources. Does not start an evaluation.",
"id": "list_context_sources",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/context/sources",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"limit": {
"type": "string"
},
"offset": {
"type": "string"
},
"q": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"expected_version": {
"type": "integer"
},
"source": {
"additionalProperties": false,
"properties": {
"case_id": {
"type": "string"
},
"category": {
"type": "string"
},
"id": {
"description": "Starts with manual- (for example manual-purpose). Reuse the same id to update the source. Other ids belong to connector imports and are refused.",
"type": "string"
},
"kind": {
"type": "string"
},
"scope": {
"type": "string"
},
"text": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"id",
"kind",
"category",
"scope",
"text"
],
"type": "object"
}
},
"required": [
"expected_version",
"source"
],
"type": "object"
},
"description": "Create or update versioned system understanding, feedback, a golden case or criterion. Read the current context version first and provide expected_version. Give each source an id that starts with manual- (for example manual-purpose) and reuse it to update that source; other ids belong to connector imports and are refused. Preserve user-supplied expected values; do not infer business policy from observed output.",
"id": "put_context_source",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/context/sources",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Upload a text, Markdown, JSON or JSONL document up to 60 KB to system understanding. Provide category and expected_version from get_system_context. Export other document formats to text first.",
"file_field": "file",
"form": {
"additionalProperties": false,
"properties": {
"category": {
"type": "string"
},
"expected_version": {
"type": "integer"
}
},
"type": "object"
},
"id": "upload_context_document",
"max_file_bytes": 60000,
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"multipart": true,
"path": "/api/ai-systems/{id}/context/upload",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Seed system context. Uses the same validation and plan limits as Bench.",
"id": "seed_system_context",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/seed",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Queue a summary of the system's saved understanding. Poll get_system_context for completion. Does not start a Bench evaluation.",
"id": "summarize_system_context",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/summarize",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Delete context source. Uses the same validation and plan limits as Bench.",
"id": "delete_context_source",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/context/sources/{sourceID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"sourceID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List context connections. Does not start an evaluation.",
"id": "list_context_connections",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/context/connections",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"config": {
"additionalProperties": false,
"properties": {
"base_url": {
"type": "string"
},
"dataset_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"environment": {
"type": "string"
},
"project": {
"type": "string"
},
"prompt_names": {
"items": {
"type": "string"
},
"type": "array"
},
"provider": {
"enum": [
"langfuse",
"langsmith"
],
"type": "string"
},
"since": {
"type": "string"
}
},
"required": [
"provider",
"base_url",
"project",
"since"
],
"type": "object"
},
"content_consent": {
"type": "boolean"
},
"credentials": {
"additionalProperties": false,
"properties": {
"api_key": {
"type": "string"
},
"public_key": {
"type": "string"
},
"secret_key": {
"type": "string"
}
},
"type": "object"
},
"retention_days": {
"type": "integer"
}
},
"required": [
"config",
"credentials",
"content_consent"
],
"type": "object"
},
"description": "Connect a supported evidence provider using explicit content consent and selected project configuration. Store provider credentials only in secrets. Synchronization imports redacted evidence; it does not make that evidence business policy.",
"id": "create_context_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/connections",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Sync context connection. Uses the same validation and plan limits as Bench.",
"id": "sync_context_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/connections/{connectionID}/sync",
"path_parameters": {
"connectionID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Delete context connection. Uses the same validation and plan limits as Bench.",
"id": "delete_context_connection",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/context/connections/{connectionID}",
"path_parameters": {
"connectionID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"description": {
"type": "string"
},
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Update system. Uses the same validation and plan limits as Bench.",
"id": "update_system",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/ai-systems/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"key": {
"type": "string"
},
"status": {
"type": "string"
}
},
"type": "object"
},
"description": "Resolve finding. Uses the same validation and plan limits as Bench.",
"id": "resolve_finding",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/finding-resolution",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Delete system. Uses the same validation and plan limits as Bench.",
"id": "delete_system",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"call_site_id": {
"type": "string"
},
"repo_full_name": {
"type": "string"
},
"role": {
"type": "string"
}
},
"type": "object"
},
"description": "Attach a scanned prompt, tool, model configuration or harness to a system. Use the exact repo_full_name, branch and call_site_id returned by a scan. This does not edit source code.",
"id": "add_system_component",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/components",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"commit_sha": {
"description": "Optional 40-hex commit the paths and lines refer to",
"type": "string"
},
"model": {
"description": "Model these prompts run on; requires model-selection entitlement, otherwise omit",
"type": "string"
},
"prompts": {
"items": {
"additionalProperties": false,
"properties": {
"agent": {
"description": "Name of the agent that sends this prompt, matching its runtime span name when instrumented",
"type": "string"
},
"content": {
"description": "Exact prompt template text as written in the source, variables unfilled",
"type": "string"
},
"end_line": {
"minimum": 1,
"type": "integer"
},
"line": {
"minimum": 1,
"type": "integer"
},
"name": {
"description": "Short stable name, e.g. analyzer system prompt",
"type": "string"
},
"path": {
"description": "Repository-relative file path",
"type": "string"
},
"role": {
"enum": [
"system",
"user",
"assistant",
"developer"
],
"type": "string"
},
"variables": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"type": "array"
},
"repo_full_name": {
"description": "owner/name of the repository the prompts live in",
"type": "string"
}
},
"type": "object"
},
"description": "Deprecated: use register_calls, which registers one entry per model call. Register prompts your coding agent found in the repository as evaluable prompt components of a system: file path, line, exact text, optional model and owning agent. Stored under the real repository and branch and merged with any existing scan; no GitHub connection or Bench scan is needed. Returns component ids to reference as bench.component_id on the model-call spans that send each prompt. Does not edit source code and does not start an evaluation.",
"id": "register_prompts",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/prompts",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"calls": {
"items": {
"additionalProperties": false,
"properties": {
"conditions": {
"items": {
"additionalProperties": false,
"properties": {
"code_ref": {
"description": "The code that decides it, e.g. \"slot\" in present_types",
"type": "string"
},
"description": {
"description": "Plain words: when is this fragment sent",
"type": "string"
},
"id": {
"type": "string"
}
},
"required": [
"id",
"description",
"code_ref"
],
"type": "object"
},
"type": "array"
},
"configurations": {
"items": {
"additionalProperties": false,
"properties": {
"fallback": {
"type": "string"
},
"id": {
"type": "string"
},
"label": {
"type": "string"
},
"model": {
"type": "string"
},
"provenance": {
"enum": [
"declared",
"observed"
],
"type": "string"
},
"provider": {
"type": "string"
},
"settings": {
"additionalProperties": false,
"properties": {
"declared": {
"description": "Values written in the code",
"type": "object"
},
"effective": {
"description": "Values the provider call really gets, e.g. after a wrapper drops or adds options",
"type": "object"
},
"effective_provenance": {
"enum": [
"declared",
"observed",
"unknown"
],
"type": "string"
}
},
"type": "object"
},
"source": {
"description": "Exactly one configuration has source code: what the code does today",
"enum": [
"code",
"suggested"
],
"type": "string"
},
"tool_choice": {
"type": "string"
},
"tools": {
"items": {
"type": "object"
},
"type": "array"
}
},
"required": [
"source",
"model"
],
"type": "object"
},
"type": "array"
},
"constraints": {
"items": {
"additionalProperties": false,
"properties": {
"conditions": {
"items": {
"type": "string"
},
"type": "array"
},
"source": {
"enum": [
"code",
"user",
"observed"
],
"type": "string"
},
"type": {
"enum": [
"exactly_one",
"not_both",
"implies"
],
"type": "string"
}
},
"required": [
"type",
"conditions",
"source"
],
"type": "object"
},
"type": "array"
},
"fragments": {
"items": {
"additionalProperties": false,
"properties": {
"group": {
"description": "Id of a group whose header and separator the code emits around this fragment",
"type": "string"
},
"id": {
"description": "Identifier, unique in the call; used by `when` and by bound variables",
"type": "string"
},
"kind": {
"description": "text = prompt text; runtime_value = a message with no source text that the test case supplies; constant = a value bound to a variable of another fragment, never sent on its own",
"enum": [
"text",
"runtime_value",
"constant"
],
"type": "string"
},
"message_group": {
"description": "Fragments with the same message_group form one message; system fragments without one form the single system message",
"type": "string"
},
"role": {
"enum": [
"system",
"developer",
"user",
"assistant"
],
"type": "string"
},
"source": {
"additionalProperties": false,
"properties": {
"file": {
"type": "string"
},
"line": {
"minimum": 1,
"type": "integer"
}
},
"type": "object"
},
"text": {
"description": "The string the model receives with every variable written as {name}. Supply the resolved string (what the code produces), not the source code that builds it. Omit for runtime_value.",
"type": "string"
},
"text_provenance": {
"description": "resolved_string = you ran or evaluated the code to get the text; source_text = read from the source without running it",
"enum": [
"resolved_string",
"source_text"
],
"type": "string"
},
"variables": {
"items": {
"additionalProperties": false,
"properties": {
"bound_to_fragment": {
"description": "Id of a constant fragment whose text is this variable's value",
"type": "string"
},
"expr": {
"description": "The expression exactly as written in the code, e.g. item.name or row[0]",
"type": "string"
},
"format_hint": {
"description": "What the value looks like, from running the code",
"type": "string"
},
"kind": {
"enum": [
"simple",
"attribute",
"index",
"expression",
"converted"
],
"type": "string"
},
"name": {
"description": "Placeholder name used in the text, an identifier",
"type": "string"
},
"source": {
"enum": [
"literal",
"code",
"command_output",
"runtime_input"
],
"type": "string"
}
},
"required": [
"name",
"expr"
],
"type": "object"
},
"type": "array"
},
"when": {
"description": "Inclusion rule: condition ids joined with and, or, not and parentheses. Omit when the fragment is always sent.",
"type": "string"
}
},
"required": [
"id",
"role",
"kind"
],
"type": "object"
},
"type": "array"
},
"groups": {
"items": {
"additionalProperties": false,
"properties": {
"header": {
"description": "Text emitted before the first included fragment of the group",
"type": "string"
},
"id": {
"type": "string"
},
"separator": {
"description": "Text emitted between included fragments of the group",
"type": "string"
}
},
"required": [
"id"
],
"type": "object"
},
"type": "array"
},
"key": {
"description": "Code-derived identity: <repo-relative file>::<qualified function>#<ordinal of the model call in that function>. Never a line number or a name you chose.",
"type": "string"
},
"key_source": {
"description": "ast = derived by parsing the code; agent = derived by hand",
"enum": [
"ast",
"agent"
],
"type": "string"
},
"location": {
"additionalProperties": false,
"properties": {
"file": {
"type": "string"
},
"function": {
"type": "string"
},
"line": {
"minimum": 1,
"type": "integer"
}
},
"type": "object"
},
"name": {
"description": "Short display name, e.g. recipe extraction",
"type": "string"
},
"notes": {
"items": {
"type": "string"
},
"type": "array"
},
"output_format": {
"type": "object"
},
"scope": {
"enum": [
"production",
"offline_script"
],
"type": "string"
},
"shape": {
"description": "Fingerprint of the call (argument names, model expression, enclosing function parameters), required when key_source is ast; lets Bench recognize the call after a rename or move",
"type": "string"
},
"slots": {
"items": {
"additionalProperties": false,
"properties": {
"after_fragment": {
"type": "string"
},
"description": {
"type": "string"
},
"kind": {
"enum": [
"history",
"injected_context"
],
"type": "string"
},
"max_messages": {
"type": "integer"
},
"role": {
"type": "string"
}
},
"required": [
"kind"
],
"type": "object"
},
"type": "array"
},
"verification": {
"additionalProperties": false,
"properties": {
"evidence": {
"additionalProperties": false,
"properties": {
"method": {
"type": "string"
},
"notes": {
"type": "string"
},
"states_checked": {
"type": "integer"
},
"states_matched": {
"type": "integer"
}
},
"type": "object"
},
"level": {
"description": "agent_verified = you ran the code that builds the prompt for sample inputs and every checked state matched the registered fragments; declared_only = read from code, not run; evidence_lacking = part of the text could not be resolved",
"enum": [
"agent_verified",
"declared_only",
"evidence_lacking"
],
"type": "string"
}
},
"required": [
"level"
],
"type": "object"
}
},
"required": [
"key",
"name",
"scope",
"configurations",
"fragments",
"verification"
],
"type": "object"
},
"type": "array"
},
"commit_sha": {
"description": "Optional 40-hex commit the calls were read at",
"type": "string"
},
"repo_full_name": {
"description": "owner/name of the repository the calls live in",
"type": "string"
}
},
"required": [
"repo_full_name",
"branch",
"calls"
],
"type": "object"
},
"description": "Register the model calls your coding agent found in the repository, one entry per call (one request sent to a model), not per prompt. A call lists its messages as fragments (system text, user templates, constants, runtime values), the model configuration the code uses, the conditions that decide which fragments are sent, and how well you verified the text. Prefer the resolved string over source code: run or evaluate the code that builds the prompt for sample inputs and register what it produces with variables written as {name}; mark verification agent_verified only when every checked state matched. Calls are identified by a code-derived key, so registering again keeps the same call after a rename or move; a doubtful match gets a new id; calls missing from a later registration are kept and labelled not seen. Returns one component id per call (shown as Call #id) to reference as bench.component_id on the spans of that call. Does not edit source code and does not start an evaluation. Replaces register_prompts, which is deprecated.",
"id": "register_calls",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/calls",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Remove system component. Uses the same validation and plan limits as Bench.",
"id": "remove_system_component",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/components/{componentID}",
"path_parameters": {
"componentID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Set continuous evaluation. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_continuous_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/ai-systems/{id}/components/{componentID}/continuous-eval",
"path_parameters": {
"componentID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"edge_type": {
"type": "string"
},
"from_component_id": {
"type": "integer"
},
"to_component_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Add system connection. Uses the same validation and plan limits as Bench.",
"id": "add_system_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/edges",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Remove system connection. Uses the same validation and plan limits as Bench.",
"id": "remove_system_connection",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/edges/{edgeID}",
"path_parameters": {
"edgeID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Create system run. Uses the same validation and plan limits as Bench.",
"id": "create_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runs",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List system runs. Does not start an evaluation.",
"id": "list_system_runs",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runs",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get system run. Does not start an evaluation.",
"id": "get_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runs/{runID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_component_id": {
"type": "integer"
},
"evaluation_run_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Link system run. Uses the same validation and plan limits as Bench.",
"id": "link_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runs/{runID}/components",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"outcome": {
"type": "string"
},
"outcome_reason": {
"type": "string"
},
"status": {
"type": "string"
}
},
"type": "object"
},
"description": "Complete system run. Uses the same validation and plan limits as Bench.",
"id": "complete_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runs/{runID}/complete",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get production health. Does not start an evaluation.",
"id": "get_production_health",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/production-health",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get production feedback. Does not start an evaluation.",
"id": "get_production_feedback",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/production-feedback",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"auto_bench": {
"type": "boolean"
},
"enabled": {
"type": "boolean"
},
"environment": {
"type": "string"
}
},
"required": [
"enabled",
"environment"
],
"type": "object"
},
"description": "Configure production checks and optional automatic investigations for one environment. Enabling automatic evaluations can spend the account allowance; confirm with the user first. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_production_feedback",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/production-feedback",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Production capabilities. Does not start an evaluation.",
"id": "production_capabilities",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/capabilities",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Sdk status. Does not start an evaluation.",
"id": "sdk_status",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/sdk-status",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"environment": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"description": "Get trace privacy. Does not start an evaluation. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_trace_privacy",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/privacy",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"fields": {
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"enabled",
"fields"
],
"type": "object"
},
"description": "Set trace privacy. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_trace_privacy",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/traces/privacy",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get production check. Does not start an evaluation.",
"id": "get_production_check",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/traces/checks/{jobID}",
"path_parameters": {
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Retry production check. Uses the same validation and plan limits as Bench.",
"id": "retry_production_check",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/traces/checks/{jobID}/retry",
"path_parameters": {
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"capture_content": {
"type": "boolean"
},
"repo_full_name": {
"type": "string"
},
"system_name": {
"type": "string"
},
"traces": {
"items": {
"additionalProperties": false,
"properties": {
"source": {
"type": "string"
},
"spans": {
"items": {},
"type": "array"
},
"trace_id": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
}
},
"type": "object"
},
"description": "Upload explicit SDK trace data. Prefer SDK instrumentation for automatic bounds and redaction. Capture content only when the application owner has opted in.",
"id": "ingest_traces",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/traces",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List traces. Does not start an evaluation.",
"id": "list_traces",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"environment": {
"type": "string"
},
"limit": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get trace. Does not start an evaluation.",
"id": "get_trace",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Delete trace. Uses the same validation and plan limits as Bench.",
"id": "delete_trace",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/traces/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_component_id": {
"type": "integer"
},
"ai_system_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Link span. Uses the same validation and plan limits as Bench.",
"id": "link_span",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/traces/{id}/spans/{spanID}/link",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"spanID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Unlink span. Uses the same validation and plan limits as Bench.",
"id": "unlink_span",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/traces/{id}/links/{linkID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"linkID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"component_id": {
"type": "integer"
},
"evidence_scope": {
"type": "string"
},
"no": {
"type": "string"
},
"provider": {
"type": "string"
},
"question": {
"type": "string"
},
"share_with_typesafe": {
"type": "boolean"
},
"yes": {
"type": "string"
}
},
"type": "object"
},
"description": "Evaluate span. Uses the same validation and plan limits as Bench.",
"id": "evaluate_span",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/traces/{id}/spans/{spanID}/evaluate",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"spanID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get workspace. Does not start an evaluation.",
"id": "get_workspace",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/org",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Create workspace. Uses the same validation and plan limits as Bench.",
"id": "create_workspace",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/org",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Rename workspace. Uses the same validation and plan limits as Bench.",
"id": "rename_workspace",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/org",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"email": {
"type": "string"
},
"role": {
"type": "string"
}
},
"type": "object"
},
"description": "Invite workspace member. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "invite_workspace_member",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/org/invites",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"token": {
"type": "string"
}
},
"type": "object"
},
"description": "Accept workspace invite. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "accept_workspace_invite",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/org/invites/accept",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"role": {
"type": "string"
}
},
"type": "object"
},
"description": "Set workspace member role. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_workspace_member_role",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/org/members/{userID}",
"path_parameters": {
"userID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Remove workspace member. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "remove_workspace_member",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/org/members/{userID}",
"path_parameters": {
"userID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"max_evaluations_per_month": {
"type": "integer"
},
"name": {
"type": "string"
},
"org_read": {
"type": "boolean"
},
"repo_allowlist": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"description": "Create a scoped automation credential. Returns plaintext only once. Store it in the application's secret environment, never in source, logs or chat replies. OAuth account authorization is required; API keys cannot mint other keys. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "create_api_key",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/account/api-keys",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"replace_revoked": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Retrieve the account's reusable SDK setup credential. Never print or commit the returned secret. replace_revoked=true explicitly replaces a revoked setup key. OAuth account authorization is required. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_setup_key",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/account/api-keys/setup",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "List api keys. Does not start an evaluation. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "list_api_keys",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/account/api-keys",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"description": "Revoke api key. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "revoke_api_key",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/account/api-keys/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"description": "List plans. Does not start an evaluation.",
"id": "list_plans",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/plans",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get payment method. Does not start an evaluation.",
"id": "get_payment_method",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/payment-method",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List invoices. Does not start an evaluation.",
"id": "list_invoices",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/invoices",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"price_id": {
"type": "string"
},
"return_path": {
"type": "string"
}
},
"type": "object"
},
"description": "Create a Stripe checkout link for a selected plan price. Obtain the user's approval before starting payment. Return the URL for payment and coupon entry; do not handle card details in the agent. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "create_checkout",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/checkout-session",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"price_id": {
"type": "string"
}
},
"type": "object"
},
"description": "Change the subscription to the chosen configured price. This can bill immediately. Obtain explicit approval for the price and interval before calling; use get_billing_portal for coupon-aware upgrades. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "change_plan",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/change-plan",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Create a Stripe billing or upgrade link. An optional target price_id opens immediate upgrade confirmation with promotion-code entry. Return the URL to the user to review payment. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_billing_portal",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/portal-session",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"price_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Cancel renewal of the subscription. This changes billing; require explicit user approval. Organization billing requires an administrator. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "cancel_subscription",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/cancel",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Reactivate subscription renewal. This changes billing; require explicit user approval. Organization billing requires an administrator. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "reactivate_subscription",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/reactivate",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
}
],
"version": 1
}