{
"operations": [
{
"auth": "account",
"description": "Get setup status. Does not start an evaluation. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_setup_status",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/getting-started",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"environment": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "public",
"description": "List operations. Does not start an evaluation.",
"id": "list_operations",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/headless/operations",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Read credential authority, plan, model-selection entitlement, repository scope and pricing and upgrade links, and links to the current operation catalog and evaluation allowance. Does not grant permissions. Does not start an evaluation.",
"id": "capabilities",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/headless/capabilities",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Evaluation allowance. Does not start an evaluation.",
"id": "evaluation_allowance",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-allowance",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "public",
"description": "Get processing notice. Does not start an evaluation.",
"id": "get_processing_notice",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/legal/processing-notice",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"accepted": {
"type": "boolean"
},
"source": {
"type": "string"
},
"version": {
"type": "string"
}
},
"type": "object"
},
"description": "Record the user's explicit authorization for the current processing notice and selected source. Read get_processing_notice and show the notice first; never accept it on the user's behalf without permission. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "acknowledge_processing",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/legal/processing-acknowledgements",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"description": "Get login url. Does not start an evaluation.",
"id": "get_login_url",
"mcp": false,
"mcp_legacy": false,
"method": "GET",
"path": "/api/auth/login-url",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"challenge": {
"type": "string"
},
"provider": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"code": {
"type": "string"
},
"newsletter_consent": {
"type": "boolean"
},
"state": {
"type": "string"
},
"verifier": {
"type": "string"
}
},
"type": "object"
},
"description": "Complete login. Uses the same validation and plan limits as Bench.",
"id": "complete_login",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/callback",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"code": {
"type": "string"
},
"newsletter_consent": {
"type": "boolean"
},
"pending_authentication_token": {
"type": "string"
},
"provider": {
"type": "string"
}
},
"type": "object"
},
"description": "Verify email. Uses the same validation and plan limits as Bench.",
"id": "verify_email",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/verify-email",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"email": {
"type": "string"
},
"newsletter_consent": {
"type": "boolean"
},
"password": {
"type": "string"
}
},
"type": "object"
},
"description": "Login with password. Uses the same validation and plan limits as Bench.",
"id": "login_with_password",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/password",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"description": "Logout. Uses the same validation and plan limits as Bench.",
"id": "logout",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/logout",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"token": {
"type": "string"
}
},
"required": [
"token"
],
"type": "object"
},
"description": "Exchange oauth token. Uses the same validation and plan limits as Bench.",
"id": "exchange_oauth_token",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/mcp-token",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Whoami. Does not start an evaluation.",
"id": "whoami",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/auth/me",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"generation": {
"type": "integer"
}
},
"type": "object"
},
"description": "Record setup completion for the current account. Use onboarding_generation from whoami; a stale generation is rejected. No browser wizard is required. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "complete_onboarding",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/auth/onboarding/complete",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Update profile. Uses the same validation and plan limits as Bench.",
"id": "update_profile",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/auth/me",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"providers": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"description": "Choose allowed model providers for future Bench evaluations. Requires an active Growth or Enterprise plan. Free and Builder use Bench defaults and cannot change this preference.",
"id": "set_model_providers",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/auth/me/model-providers",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"body": {
"additionalProperties": false,
"properties": {
"installation_id": {
"type": "integer"
},
"state": {
"type": "string"
}
},
"type": "object"
},
"description": "Receive github callback. Uses the same validation and plan limits as Bench.",
"id": "receive_github_callback",
"mcp": false,
"mcp_legacy": false,
"method": "POST",
"path": "/api/github/headless-callback",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Start GitHub repository authorization. Set headless=true to return to the coding agent after the user approves access on GitHub. Keep the returned state and poll finish_github_connection. GitHub installation approval is required; no Bench onboarding is needed.",
"id": "connect_github",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/install-url",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"headless": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"installation_id": {
"type": "integer"
},
"state": {
"type": "string"
}
},
"type": "object"
},
"description": "Complete a manual GitHub callback using the original account-bound state and installation_id. Prefer connect_github with headless=true and finish_github_connection. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "complete_github_callback",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/github/callback",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"state": {
"type": "string"
}
},
"type": "object"
},
"description": "Finish the connection started with connect_github(headless=true). Pass its state. A pending response means wait the returned poll_interval before calling again. On success, list_repos and choose a branch. This binds the installation to the signed-in user. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "finish_github_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/github/connection",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Connection status. Does not start an evaluation.",
"id": "connection_status",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/status",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"description": "Activate installation. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "activate_installation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/github/installations/{installationID}/activate",
"path_parameters": {
"installationID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List repos. Does not start an evaluation.",
"id": "list_repos",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/repos",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"all_accounts": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List branches. Does not start an evaluation.",
"id": "list_branches",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/github/repos/{owner}/{repo}/branches",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Upload prompts. Uses the same validation and plan limits as Bench.",
"file_field": "files",
"form": {
"additionalProperties": false,
"properties": {
"model": {
"type": "string"
},
"pasted_text": {
"type": "string"
}
},
"type": "object"
},
"id": "upload_prompts",
"max_file_bytes": 2097152,
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"multipart": true,
"path": "/api/prompt-scans",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Scan repo. Uses the same validation and plan limits as Bench.",
"id": "scan_repo",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/scan",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get scan. Does not start an evaluation.",
"id": "get_scan",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/scan/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get repository refresh. Does not start an evaluation.",
"id": "get_repository_refresh",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/scan/refresh",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Set repository refresh. Uses the same validation and plan limits as Bench.",
"id": "set_repository_refresh",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/repos/{owner}/{repo}/scan/refresh",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"call_site_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"model": {
"type": "string"
}
},
"type": "object"
},
"description": "Record the model used by uploaded prompt call sites. This changes source metadata, not the allowed-provider setting. Requires model-selection entitlement.",
"id": "set_prompt_models",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/scan/call-sites/model",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"base_branch": {
"type": "string"
},
"body": {
"type": "string"
},
"branch": {
"type": "string"
},
"changes": {
"items": {
"additionalProperties": false,
"properties": {
"content": {
"type": "string"
},
"path": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"title": {
"type": "string"
}
},
"type": "object"
},
"description": "Open prompt pr. Uses the same validation and plan limits as Bench.",
"id": "open_prompt_pr",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/prompt-pr",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"business_doc_text": {
"type": "string"
}
},
"type": "object"
},
"description": "Generate business context. Uses the same validation and plan limits as Bench.",
"id": "generate_business_context",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/business-context",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get business context. Does not start an evaluation.",
"id": "get_business_context",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/business-context/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"previous_routing_json": {
"type": "string"
},
"reference_today": {
"type": "string"
},
"suite_name": {
"type": "string"
}
},
"type": "object"
},
"description": "Generate eval benchmark. Uses the same validation and plan limits as Bench.",
"id": "generate_eval_benchmark",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/eval-benchmark",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false,
"streaming": true
},
{
"auth": "credential",
"description": "Get eval benchmark. Does not start an evaluation.",
"id": "get_eval_benchmark",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/eval-benchmark/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"call_site_id": {
"type": "string"
},
"test_cases": {}
},
"type": "object"
},
"description": "Review eval benchmark. Uses the same validation and plan limits as Bench.",
"id": "review_eval_benchmark",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/eval-benchmark/review",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"max_concurrency": {
"type": "integer"
},
"model": {
"type": "string"
},
"suite_name": {
"type": "string"
}
},
"type": "object"
},
"description": "Run baseline scoring for an existing reviewed benchmark. This can consume an evaluation credit; confirm spending first. Streamed events preserve progress and terminal errors. A model override requires Growth or Enterprise.",
"id": "run_baseline",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/baseline",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false,
"streaming": true
},
{
"auth": "credential",
"description": "Get baseline. Does not start an evaluation.",
"id": "get_baseline",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/baseline/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"candidate_models": {
"items": {
"additionalProperties": false,
"properties": {
"model_id": {
"type": "string"
},
"provider": {
"type": "string"
},
"tier": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"max_case_concurrency": {
"type": "integer"
},
"max_combo_concurrency": {
"type": "integer"
},
"max_rounds": {
"type": "integer"
},
"min_quality_score": {
"type": "number"
},
"suite_name": {
"type": "string"
},
"variant_count": {
"type": "integer"
}
},
"type": "object"
},
"description": "Evaluate prompt improvements after a baseline. Confirm evaluation spending first. Candidate lists are server controlled on all plans; use set_model_providers for eligible provider preferences.",
"id": "run_search_optimize",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/repos/{owner}/{repo}/search-optimize",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false,
"streaming": true
},
{
"auth": "credential",
"description": "Get search optimize. Does not start an evaluation.",
"id": "get_search_optimize",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/search-optimize/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get recommendation. Does not start an evaluation.",
"id": "get_recommendation",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/recommend/latest",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List golden cases. Does not start an evaluation.",
"id": "list_golden_cases",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/repos/{owner}/{repo}/golden-cases",
"path_parameters": {
"owner": {
"minLength": 1,
"type": "string"
},
"repo": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"description": "Repository branch, required for branch-scoped operations.",
"type": "string"
},
"call_site_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"label": {
"type": "string"
},
"note": {
"type": "string"
}
},
"type": "object"
},
"description": "Label golden case. Uses the same validation and plan limits as Bench.",
"id": "label_golden_case",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/golden-cases/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"url": {
"type": "string"
}
},
"type": "object"
},
"description": "Fetch website text. Uses the same validation and plan limits as Bench.",
"id": "fetch_website_text",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/website-text",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"url": {
"type": "string"
}
},
"type": "object"
},
"description": "Get link metadata. Uses the same validation and plan limits as Bench.",
"id": "get_link_metadata",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/link-metadata",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_id": {
"type": "integer"
},
"branch": {
"type": "string"
},
"context_doc": {
"type": "string"
},
"generate_context": {
"type": "boolean"
},
"max_rounds": {
"type": "integer"
},
"min_quality_score": {
"type": "number"
},
"prompts": {
"items": {
"additionalProperties": false,
"properties": {
"call_site_id": {
"type": "string"
},
"workflow_name": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"description": "Start evaluation. Uses the same validation and plan limits as Bench.",
"id": "start_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Start system evaluation. Uses the same validation and plan limits as Bench.",
"id": "start_system_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/system",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List evaluations. Does not start an evaluation.",
"id": "list_evaluations",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"include_active": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"context_doc": {
"type": "string"
},
"generate_context": {
"type": "boolean"
},
"single_prompt": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Rerun evaluation. Uses the same validation and plan limits as Bench.",
"id": "rerun_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/{id}/rerun",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get evaluation. Does not start an evaluation.",
"id": "get_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get evaluation artifacts. Does not start an evaluation.",
"id": "get_evaluation_artifacts",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs/{id}/artifacts",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get fix brief. Does not start an evaluation.",
"id": "get_fix_brief",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/evaluation-runs/{id}/fix-brief",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Cancel evaluation. Uses the same validation and plan limits as Bench.",
"id": "cancel_evaluation",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/{id}/cancel",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Cancel bench. Uses the same validation and plan limits as Bench.",
"id": "cancel_bench",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/evaluation-runs/{id}/cancel-bench",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"test_cases": {}
},
"type": "object"
},
"description": "Submit run review. Uses the same validation and plan limits as Bench.",
"id": "submit_run_review",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/evaluation-runs/{id}/review",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Archive evaluation. Uses the same validation and plan limits as Bench.",
"id": "archive_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/evaluation-runs/{id}/archive",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Unarchive evaluation. Uses the same validation and plan limits as Bench.",
"id": "unarchive_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/evaluation-runs/{id}/unarchive",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"description": {
"type": "string"
},
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Create an AI system with a name and optional description. Add prompt, tool or harness components after scanning a repository or uploading prompts. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "create_system",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List systems. Does not start an evaluation.",
"id": "list_systems",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get system. Does not start an evaluation.",
"id": "get_system",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List runtime evaluations. Does not start an evaluation.",
"id": "list_runtime_evaluations",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runtime-evaluations",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"environment": {
"type": "string"
},
"offset": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": true,
"properties": {},
"type": "object"
},
"description": "Publish an SDK real app test report (maximum 500 KB). Execute the actual application with the SDK evaluateSystem/evaluate_system API first. Uploaded results remain client-reported evidence, not hosted verification.",
"id": "publish_runtime_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runtime-evaluations",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": {},
"type": "object"
},
"description": "Validate and persist an immutable application evaluation plan without paid execution. Pins versions, grouped cases, configurations, runtime and explicit budgets. Confirmation inputs and expected outcomes remain private.",
"id": "runtime_eval_plan",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-plans",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Read public plan counts and budget estimates. Confirmation content is never returned.",
"id": "runtime_eval_get_plan",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Read only development cases for human review; confirmation and calibration cases are excluded.",
"id": "runtime_eval_review_cases",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}/review",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"case_id": {
"type": "string"
},
"note": {
"type": "string"
},
"verdict": {
"type": "string"
}
},
"type": "object"
},
"description": "Record good, bad or skip feedback for a development case. Applying feedback requires a new immutable dataset and plan.",
"id": "runtime_eval_review",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}/review",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"allowed_providers": {
"items": {
"type": "string"
},
"type": "array"
},
"domains": {
"items": {
"type": "string"
},
"type": "array"
},
"run_id": {
"type": "string"
}
},
"type": "object"
},
"description": "Propose candidate configurations for a new plan from development trials and published model evidence across allowed providers. Reads no confirmation trials, runs no model, and never edits the saved plan; every proposal is unverified until measured.",
"id": "runtime_eval_candidates",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-plans/{planID}/candidates",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"planID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"plan_id": {
"type": "string"
}
},
"type": "object"
},
"description": "Start a saved plan. Requires Idempotency-Key header (8\u2013200 characters), GitHub authorization, and evaluation allowance. Duplicate starts return the original run without repeating execution.",
"id": "runtime_eval_run",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-runs",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Read job status separately from evaluation decision, uncertainty, resource metrics and development evidence. No confirmation content is returned.",
"id": "runtime_eval_results",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-runs/{runID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"after": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Read durable events with action=events and optional after cursor. Other actions return the protected result.",
"id": "runtime_eval_events",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/evaluation-runs/{runID}/{action}",
"path_parameters": {
"action": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"after": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"extend_budget": {
"additionalProperties": {},
"type": "object"
},
"retry_ambiguous_attempt": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Cancel a run, resume a safely checkpointed canceled run, or continue a paused run with an explicit extend_budget or an acknowledged retry_ambiguous_attempt. Used holdouts and plans needing more independent data are never automatically retried.",
"id": "runtime_eval_control",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/evaluation-runs/{runID}/{action}",
"path_parameters": {
"action": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"after": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": null,
"description": "Read hosted runtime availability, settings and jobs. For local real app tests, use the SDK even when hosted execution is unavailable. Does not start an evaluation.",
"id": "get_runtime_settings",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runtime-settings",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"config": {
"additionalProperties": true,
"properties": {},
"type": "object"
},
"daily_limit": {
"type": "integer"
},
"enabled": {
"type": "boolean"
},
"repository": {
"type": "string"
}
},
"required": [
"repository",
"branch",
"enabled",
"daily_limit",
"config"
],
"type": "object"
},
"description": "Configure hosted real app execution for a connected system: repository, branch, enabled, daily_limit and config. Check get_runtime_settings for environment availability. Enabling it can execute application code and consume evaluation allowance; obtain user approval first. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_runtime_settings",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/runtime-settings",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Publish runtime fix. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "publish_runtime_fix",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runtime-jobs/{jobID}/publish",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Retry runtime job. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "retry_runtime_job",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runtime-jobs/{jobID}/retry",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Preview CSV, XLSX, JSON or JSONL and suggested column mapping without storing it. Upload the file bytes, optional sheet, mode and mapping. Inspect mapping_error before import. No model runs.",
"file_field": "file",
"form": {
"additionalProperties": false,
"properties": {
"content_consent": {
"type": "boolean"
},
"mapping": {
"additionalProperties": false,
"properties": {
"actual": {
"type": "string"
},
"expected": {
"type": "string"
},
"history": {
"type": "string"
},
"input": {
"type": "string"
},
"notes": {
"type": "string"
},
"resolved_prompt": {
"type": "string"
},
"template": {
"type": "string"
},
"variables": {
"type": "string"
}
},
"type": "object"
},
"mode": {
"enum": [
"classifier",
"recorded_outputs",
"replay"
],
"type": "string"
},
"name": {
"type": "string"
},
"sheet": {
"type": "string"
}
},
"type": "object"
},
"id": "preview_dataset",
"max_file_bytes": 4194304,
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"multipart": true,
"path": "/api/ai-systems/{id}/datasets/preview",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List test library. Does not start an evaluation.",
"id": "list_test_library",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/test-library",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"kind": {
"type": "string"
},
"offset": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"component_id": {
"type": "integer"
},
"dataset_id": {
"type": "string"
},
"dataset_version": {
"type": "integer"
},
"expected_version": {
"type": "integer"
}
},
"type": "object"
},
"description": "Import dataset cases. Uses the same validation and plan limits as Bench.",
"id": "import_dataset_cases",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/ai-systems/{id}/test-library/import",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Store a dataset up to 4 MB after the user approves content_consent=true and the column mapping. Preview first. A stored dataset becomes a test suite only when import_dataset_cases is called.",
"file_field": "file",
"form": {
"additionalProperties": false,
"properties": {
"content_consent": {
"type": "boolean"
},
"mapping": {
"additionalProperties": false,
"properties": {
"actual": {
"type": "string"
},
"expected": {
"type": "string"
},
"history": {
"type": "string"
},
"input": {
"type": "string"
},
"notes": {
"type": "string"
},
"resolved_prompt": {
"type": "string"
},
"template": {
"type": "string"
},
"variables": {
"type": "string"
}
},
"type": "object"
},
"mode": {
"enum": [
"classifier",
"recorded_outputs",
"replay"
],
"type": "string"
},
"name": {
"type": "string"
},
"sheet": {
"type": "string"
}
},
"type": "object"
},
"id": "upload_dataset",
"max_file_bytes": 4194304,
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"multipart": true,
"path": "/api/ai-systems/{id}/datasets",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List datasets. Does not start an evaluation.",
"id": "list_datasets",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/datasets",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get dataset. Does not start an evaluation.",
"id": "get_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/datasets/{datasetID}",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"version": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"expected_version": {
"type": "integer"
},
"mapping": {
"additionalProperties": false,
"properties": {
"actual": {
"type": "string"
},
"expected": {
"type": "string"
},
"history": {
"type": "string"
},
"input": {
"type": "string"
},
"notes": {
"type": "string"
},
"resolved_prompt": {
"type": "string"
},
"template": {
"type": "string"
},
"variables": {
"type": "string"
}
},
"type": "object"
},
"mode": {
"type": "string"
}
},
"required": [
"expected_version",
"mapping",
"mode"
],
"type": "object"
},
"description": "Create a new dataset version with a corrected column mapping. Supply expected_version from get_dataset. Old reports retain their original mapping.",
"id": "remap_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/datasets/{datasetID}/mapping",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"version": {
"type": "integer"
}
},
"type": "object"
},
"description": "Compare recorded outputs with expected values for an explicit dataset version. No model or live app is called. Live replay is not supported by this operation.",
"id": "compare_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/datasets/{datasetID}/compare",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List dataset reports. Does not start an evaluation.",
"id": "list_dataset_reports",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/datasets/{datasetID}/reports",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Delete dataset. Uses the same validation and plan limits as Bench.",
"id": "delete_dataset",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/datasets/{datasetID}",
"path_parameters": {
"datasetID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get system context. Does not start an evaluation.",
"id": "get_system_context",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/context",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"version": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List context sources. Does not start an evaluation.",
"id": "list_context_sources",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/ai-systems/{id}/context/sources",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {
"limit": {
"type": "string"
},
"offset": {
"type": "string"
},
"q": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"expected_version": {
"type": "integer"
},
"source": {
"additionalProperties": false,
"properties": {
"case_id": {
"type": "string"
},
"category": {
"type": "string"
},
"id": {
"type": "string"
},
"kind": {
"type": "string"
},
"scope": {
"type": "string"
},
"text": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"id",
"kind",
"category",
"scope",
"text"
],
"type": "object"
}
},
"required": [
"expected_version",
"source"
],
"type": "object"
},
"description": "Create or update versioned system understanding, feedback, a golden case or criterion. Read the current context version first and provide expected_version. Preserve user-supplied expected values; do not infer business policy from observed output.",
"id": "put_context_source",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/context/sources",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Upload a text, Markdown, JSON or JSONL document up to 60 KB to system understanding. Provide category and expected_version from get_system_context. Export other document formats to text first.",
"file_field": "file",
"form": {
"additionalProperties": false,
"properties": {
"category": {
"type": "string"
},
"expected_version": {
"type": "integer"
}
},
"type": "object"
},
"id": "upload_context_document",
"max_file_bytes": 60000,
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"multipart": true,
"path": "/api/ai-systems/{id}/context/upload",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Seed system context. Uses the same validation and plan limits as Bench.",
"id": "seed_system_context",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/seed",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Queue a summary of the system's saved understanding. Poll get_system_context for completion. Does not start a Bench evaluation.",
"id": "summarize_system_context",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/summarize",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Delete context source. Uses the same validation and plan limits as Bench.",
"id": "delete_context_source",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/context/sources/{sourceID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"sourceID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List context connections. Does not start an evaluation.",
"id": "list_context_connections",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/context/connections",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"config": {
"additionalProperties": false,
"properties": {
"base_url": {
"type": "string"
},
"dataset_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"environment": {
"type": "string"
},
"project": {
"type": "string"
},
"prompt_names": {
"items": {
"type": "string"
},
"type": "array"
},
"provider": {
"enum": [
"langfuse",
"langsmith"
],
"type": "string"
},
"since": {
"type": "string"
}
},
"required": [
"provider",
"base_url",
"project",
"since"
],
"type": "object"
},
"content_consent": {
"type": "boolean"
},
"credentials": {
"additionalProperties": false,
"properties": {
"api_key": {
"type": "string"
},
"public_key": {
"type": "string"
},
"secret_key": {
"type": "string"
}
},
"type": "object"
},
"retention_days": {
"type": "integer"
}
},
"required": [
"config",
"credentials",
"content_consent"
],
"type": "object"
},
"description": "Connect a supported evidence provider using explicit content consent and selected project configuration. Store provider credentials only in secrets. Synchronization imports redacted evidence; it does not make that evidence business policy.",
"id": "create_context_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/connections",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Sync context connection. Uses the same validation and plan limits as Bench.",
"id": "sync_context_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/context/connections/{connectionID}/sync",
"path_parameters": {
"connectionID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Delete context connection. Uses the same validation and plan limits as Bench.",
"id": "delete_context_connection",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/context/connections/{connectionID}",
"path_parameters": {
"connectionID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"description": {
"type": "string"
},
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Update system. Uses the same validation and plan limits as Bench.",
"id": "update_system",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/ai-systems/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"key": {
"type": "string"
},
"status": {
"type": "string"
}
},
"type": "object"
},
"description": "Resolve finding. Uses the same validation and plan limits as Bench.",
"id": "resolve_finding",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/finding-resolution",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Delete system. Uses the same validation and plan limits as Bench.",
"id": "delete_system",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"call_site_id": {
"type": "string"
},
"repo_full_name": {
"type": "string"
},
"role": {
"type": "string"
}
},
"type": "object"
},
"description": "Attach a scanned prompt, tool, model configuration or harness to a system. Use the exact repo_full_name, branch and call_site_id returned by a scan. This does not edit source code.",
"id": "add_system_component",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/components",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"commit_sha": {
"description": "Optional 40-hex commit the paths and lines refer to",
"type": "string"
},
"model": {
"description": "Model these prompts run on; requires model-selection entitlement, otherwise omit",
"type": "string"
},
"prompts": {
"items": {
"additionalProperties": false,
"properties": {
"agent": {
"description": "Name of the agent that sends this prompt, matching its runtime span name when instrumented",
"type": "string"
},
"content": {
"description": "Exact prompt template text as written in the source, variables unfilled",
"type": "string"
},
"end_line": {
"minimum": 1,
"type": "integer"
},
"line": {
"minimum": 1,
"type": "integer"
},
"name": {
"description": "Short stable name, e.g. analyzer system prompt",
"type": "string"
},
"path": {
"description": "Repository-relative file path",
"type": "string"
},
"role": {
"enum": [
"system",
"user",
"assistant",
"developer"
],
"type": "string"
},
"variables": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"type": "array"
},
"repo_full_name": {
"description": "owner/name of the repository the prompts live in",
"type": "string"
}
},
"type": "object"
},
"description": "Register prompts your coding agent found in the repository as evaluable prompt components of a system: file path, line, exact text, optional model and owning agent. Stored under the real repository and branch and merged with any existing scan; no GitHub connection or Bench scan is needed. Returns component ids to reference as bench.component_id on the model-call spans that send each prompt. Does not edit source code and does not start an evaluation.",
"id": "register_prompts",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/prompts",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Remove system component. Uses the same validation and plan limits as Bench.",
"id": "remove_system_component",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/components/{componentID}",
"path_parameters": {
"componentID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Set continuous evaluation. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_continuous_evaluation",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/ai-systems/{id}/components/{componentID}/continuous-eval",
"path_parameters": {
"componentID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"edge_type": {
"type": "string"
},
"from_component_id": {
"type": "integer"
},
"to_component_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Add system connection. Uses the same validation and plan limits as Bench.",
"id": "add_system_connection",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/edges",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Remove system connection. Uses the same validation and plan limits as Bench.",
"id": "remove_system_connection",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/ai-systems/{id}/edges/{edgeID}",
"path_parameters": {
"edgeID": {
"minLength": 1,
"type": "string"
},
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Create system run. Uses the same validation and plan limits as Bench.",
"id": "create_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runs",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List system runs. Does not start an evaluation.",
"id": "list_system_runs",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runs",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get system run. Does not start an evaluation.",
"id": "get_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/runs/{runID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_component_id": {
"type": "integer"
},
"evaluation_run_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Link system run. Uses the same validation and plan limits as Bench.",
"id": "link_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runs/{runID}/components",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"outcome": {
"type": "string"
},
"outcome_reason": {
"type": "string"
},
"status": {
"type": "string"
}
},
"type": "object"
},
"description": "Complete system run. Uses the same validation and plan limits as Bench.",
"id": "complete_system_run",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/ai-systems/{id}/runs/{runID}/complete",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"runID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get production health. Does not start an evaluation.",
"id": "get_production_health",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/production-health",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get production feedback. Does not start an evaluation.",
"id": "get_production_feedback",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/ai-systems/{id}/production-feedback",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"auto_bench": {
"type": "boolean"
},
"enabled": {
"type": "boolean"
},
"environment": {
"type": "string"
}
},
"required": [
"enabled",
"environment"
],
"type": "object"
},
"description": "Configure production checks and optional automatic investigations for one environment. Enabling automatic evaluations can spend the account allowance; confirm with the user first. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_production_feedback",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/ai-systems/{id}/production-feedback",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Production capabilities. Does not start an evaluation.",
"id": "production_capabilities",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/capabilities",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Sdk status. Does not start an evaluation.",
"id": "sdk_status",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/sdk-status",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"environment": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"description": "Get trace privacy. Does not start an evaluation. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_trace_privacy",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/privacy",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"fields": {
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"enabled",
"fields"
],
"type": "object"
},
"description": "Set trace privacy. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_trace_privacy",
"mcp": true,
"mcp_legacy": false,
"method": "PUT",
"path": "/api/traces/privacy",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get production check. Does not start an evaluation.",
"id": "get_production_check",
"mcp": true,
"mcp_legacy": true,
"method": "GET",
"path": "/api/traces/checks/{jobID}",
"path_parameters": {
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Retry production check. Uses the same validation and plan limits as Bench.",
"id": "retry_production_check",
"mcp": true,
"mcp_legacy": true,
"method": "POST",
"path": "/api/traces/checks/{jobID}/retry",
"path_parameters": {
"jobID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"capture_content": {
"type": "boolean"
},
"repo_full_name": {
"type": "string"
},
"system_name": {
"type": "string"
},
"traces": {
"items": {
"additionalProperties": false,
"properties": {
"source": {
"type": "string"
},
"spans": {
"items": {},
"type": "array"
},
"trace_id": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
}
},
"type": "object"
},
"description": "Upload explicit SDK trace data. Prefer SDK instrumentation for automatic bounds and redaction. Capture content only when the application owner has opted in.",
"id": "ingest_traces",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/traces",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "List traces. Does not start an evaluation.",
"id": "list_traces",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"branch": {
"type": "string"
},
"environment": {
"type": "string"
},
"limit": {
"type": "string"
},
"repo_full_name": {
"type": "string"
}
},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get trace. Does not start an evaluation.",
"id": "get_trace",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/traces/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Delete trace. Uses the same validation and plan limits as Bench.",
"id": "delete_trace",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/traces/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"ai_system_component_id": {
"type": "integer"
},
"ai_system_id": {
"type": "integer"
}
},
"type": "object"
},
"description": "Link span. Uses the same validation and plan limits as Bench.",
"id": "link_span",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/traces/{id}/spans/{spanID}/link",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"spanID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Unlink span. Uses the same validation and plan limits as Bench.",
"id": "unlink_span",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/traces/{id}/links/{linkID}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"linkID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"component_id": {
"type": "integer"
},
"evidence_scope": {
"type": "string"
},
"no": {
"type": "string"
},
"provider": {
"type": "string"
},
"question": {
"type": "string"
},
"share_with_typesafe": {
"type": "boolean"
},
"yes": {
"type": "string"
}
},
"type": "object"
},
"description": "Evaluate span. Uses the same validation and plan limits as Bench.",
"id": "evaluate_span",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/traces/{id}/spans/{spanID}/evaluate",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
},
"spanID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"description": "Get workspace. Does not start an evaluation.",
"id": "get_workspace",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/org",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Create workspace. Uses the same validation and plan limits as Bench.",
"id": "create_workspace",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/org",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "credential",
"body": {
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"description": "Rename workspace. Uses the same validation and plan limits as Bench.",
"id": "rename_workspace",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/org",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"email": {
"type": "string"
},
"role": {
"type": "string"
}
},
"type": "object"
},
"description": "Invite workspace member. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "invite_workspace_member",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/org/invites",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"token": {
"type": "string"
}
},
"type": "object"
},
"description": "Accept workspace invite. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "accept_workspace_invite",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/org/invites/accept",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"role": {
"type": "string"
}
},
"type": "object"
},
"description": "Set workspace member role. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "set_workspace_member_role",
"mcp": true,
"mcp_legacy": false,
"method": "PATCH",
"path": "/api/org/members/{userID}",
"path_parameters": {
"userID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Remove workspace member. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "remove_workspace_member",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/org/members/{userID}",
"path_parameters": {
"userID": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"max_evaluations_per_month": {
"type": "integer"
},
"name": {
"type": "string"
},
"org_read": {
"type": "boolean"
},
"repo_allowlist": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"description": "Create a scoped automation credential. Returns plaintext only once. Store it in the application's secret environment, never in source, logs or chat replies. OAuth account authorization is required; API keys cannot mint other keys. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "create_api_key",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/account/api-keys",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"replace_revoked": {
"type": "boolean"
}
},
"type": "object"
},
"description": "Retrieve the account's reusable SDK setup credential. Never print or commit the returned secret. replace_revoked=true explicitly replaces a revoked setup key. OAuth account authorization is required. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_setup_key",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/account/api-keys/setup",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "List api keys. Does not start an evaluation. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "list_api_keys",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/account/api-keys",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"description": "Revoke api key. Uses the same validation and plan limits as Bench. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "revoke_api_key",
"mcp": true,
"mcp_legacy": false,
"method": "DELETE",
"path": "/api/account/api-keys/{id}",
"path_parameters": {
"id": {
"minLength": 1,
"type": "string"
}
},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "public",
"description": "List plans. Does not start an evaluation.",
"id": "list_plans",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/plans",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "Get payment method. Does not start an evaluation.",
"id": "get_payment_method",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/payment-method",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "credential",
"description": "List invoices. Does not start an evaluation.",
"id": "list_invoices",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/invoices",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": true
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"price_id": {
"type": "string"
},
"return_path": {
"type": "string"
}
},
"type": "object"
},
"description": "Create a Stripe checkout link for a selected plan price. Obtain the user's approval before starting payment. Return the URL for payment and coupon entry; do not handle card details in the agent. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "create_checkout",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/checkout-session",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"body": {
"additionalProperties": false,
"properties": {
"price_id": {
"type": "string"
}
},
"type": "object"
},
"description": "Change the subscription to the chosen configured price. This can bill immediately. Obtain explicit approval for the price and interval before calling; use get_billing_portal for coupon-aware upgrades. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "change_plan",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/change-plan",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Create a Stripe billing or upgrade link. An optional target price_id opens immediate upgrade confirmation with promotion-code entry. Return the URL to the user to review payment. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "get_billing_portal",
"mcp": true,
"mcp_legacy": false,
"method": "GET",
"path": "/api/billing/portal-session",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {
"price_id": {
"type": "string"
}
},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Cancel renewal of the subscription. This changes billing; require explicit user approval. Organization billing requires an administrator. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "cancel_subscription",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/cancel",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
},
{
"auth": "account",
"description": "Reactivate subscription renewal. This changes billing; require explicit user approval. Organization billing requires an administrator. Requires Bench OAuth account authorization; a scoped API key is insufficient.",
"id": "reactivate_subscription",
"mcp": true,
"mcp_legacy": false,
"method": "POST",
"path": "/api/billing/reactivate",
"path_parameters": {},
"query": {
"additionalProperties": false,
"properties": {},
"type": "object"
},
"read_only": false
}
],
"version": 1
}