use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::Duration;
use auth_cloudflare::auth::{AuthProvider, ACCOUNT_ENV, TOKEN_ENV};
use auth_cloudflare::cache::{
cache_dir_for_account, cache_is_stale, read_catalog_cache, write_catalog_cache, CatalogCacheMeta,
};
use auth_cloudflare::catalog::{ModelRecord, FALLBACK_MODELS};
use auth_cloudflare::config::{Config, SecretString};
use auth_cloudflare::error::CloudflareError;
use auth_cloudflare::fetch::{fetch_catalog_from_api, FETCH_TIMEOUT};
use auth_cloudflare::policy::ModelPolicy;
use auth_cloudflare::schema::{VersionInfo, CATALOG_SCHEMA_VERSION};
use auth_cloudflare::health;
use auth_cloudflare::verify::{self, SuiteKind};
use auth_cloudflare::{DEFAULT_MODEL, VERSION};
type FetchCatalog =
fn(account_id: &str, token: &SecretString, timeout: Duration) -> Result<serde_json::Value, CloudflareError>;
const EXIT_OK: i32 = 0;
const EXIT_OPERATIONAL: i32 = 1;
const EXIT_CREDENTIALS: i32 = 2;
const EXIT_REMOTE_API: i32 = 3;
const EXIT_STALE_CACHE: i32 = 4;
const EXIT_NO_ELIGIBLE_MODEL: i32 = 5;
const EXIT_CONFORMANCE: i32 = 6;
const EXIT_UNSAFE_CONFIG: i32 = 7;
const CACHE_MAX_AGE: Duration = Duration::from_secs(6 * 3600);
const TOOL_LOOP_TIMEOUT: Duration = Duration::from_secs(90);
const ACCOUNT_ID_ENV: &str = "AUTH_CLOUDFLARE_ACCOUNT_ID";
const API_TOKEN_ENV: &str = "AUTH_CLOUDFLARE_API_TOKEN";
const BASE_URL_ENV: &str = "AUTH_CLOUDFLARE_WORKERS_AI_BASE_URL";
const CACHE_DIR_ENV: &str = "AUTH_CLOUDFLARE_CACHE_DIR";
const CONFIG_ENV: &str = "AUTH_CLOUDFLARE_CONFIG";
const LEGACY_HERMES_TOKEN_ENV: &str = "HERMES_CUSTOM_API_CLOUDFLARE_COM_API_KEY";
const ACCOUNT_ID_LEN: usize = 32;
const CONFIG_FILE_NAME: &str = "config.json";
const EXPORT_DIR_ENV: &str = "AUTH_CLOUDFLARE_EXPORT_DIR";
const USAGE: &str = "\
auth-cloudflare - Cloudflare Workers AI auth provider CLI (v{version})
Usage:
auth-cloudflare version [--format json]
auth-cloudflare doctor [--format json]
auth-cloudflare catalog get [--format json]
auth-cloudflare catalog list [--format json]
auth-cloudflare catalog refresh [--format json]
auth-cloudflare catalog export yaml|markdown
auth-cloudflare catalog diff [--format json]
auth-cloudflare model inspect <model-id> [--format json]
auth-cloudflare model verify <model-id> [--suite smoke] [--format json]
auth-cloudflare model health [--format json]
auth-cloudflare policy get [--format json]
auth-cloudflare help
Exit codes (feedback 02, binding):
0 success
1 operational error (malformed args, I/O, unexpected failure)
2 credentials missing or invalid
3 remote Cloudflare API failure
4 live API unavailable; stale cache successfully used
5 requested model not found / no eligible model
6 conformance suite ran but failed acceptance criteria
7 unsafe configuration / secret-leak risk detected
Environment (core precedence, feedback 03/06):
AUTH_CLOUDFLARE_ACCOUNT_ID, AUTH_CLOUDFLARE_API_TOKEN,
AUTH_CLOUDFLARE_WORKERS_AI_BASE_URL, AUTH_CLOUDFLARE_CACHE_DIR,
AUTH_CLOUDFLARE_CONFIG, AUTH_CLOUDFLARE_EXPORT_DIR (export target dir)
AUTH_CLOUDFLARE_LIVE_TESTS=1 (opens the live gate for 'model verify';
paid inference is refused unless the value is exactly the digit 1)
AUTH_CLOUDFLARE_MAX_COST_USD=<budget> (optional conformance budget;
reported as cost_estimate_usd in the run report, documented but never
enforced)
Legacy aliases: CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN,
HERMES_CUSTOM_API_CLOUDFLARE_COM_API_KEY
";
#[derive(Debug, Clone, PartialEq, Eq)]
enum Command {
Version,
Doctor,
CatalogGet,
CatalogList,
CatalogRefresh,
CatalogExport { format: ExportFormat },
CatalogDiff,
ModelInspect { model_id: String },
ModelVerify { model_id: String, suite: SuiteKind, recommended: bool },
ModelHealth,
PolicyGet,
Help,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ExportFormat {
Yaml,
Markdown,
}
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
let code = run(&args, &mut std::io::stdout(), &mut std::io::stderr());
std::process::exit(code);
}
fn run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> i32 {
run_with_fetch(args, out, err, fetch_catalog_from_api)
}
fn run_with_fetch(args: &[String], out: &mut dyn Write, err: &mut dyn Write, fetch: FetchCatalog) -> i32 {
let (command, format) = match parse_args(args) {
Ok(parsed) => parsed,
Err(message) => {
let _ = writeln!(err, "auth-cloudflare: {message}");
let _ = writeln!(err, "{}", USAGE.replace("{version}", VERSION));
return EXIT_OPERATIONAL;
},
};
if let Some(format) = format {
if format != "json" {
let _ = writeln!(
err,
"auth-cloudflare: unsupported --format {format}: this command only supports --format json"
);
return EXIT_OPERATIONAL;
}
}
execute(command, out, err, fetch)
}
fn parse_args(args: &[String]) -> Result<(Command, Option<String>), String> {
let mut format: Option<String> = None;
let mut suite: Option<String> = None;
let mut recommended = false;
let mut positional: Vec<String> = Vec::new();
let mut iter = args.iter();
while let Some(arg) = iter.next() {
if arg == "--format" {
let Some(value) = iter.next() else {
return Err("--format requires a value (e.g. --format json)".to_string());
};
format = Some(value.clone());
} else if let Some(value) = arg.strip_prefix("--format=") {
format = Some(value.to_string());
} else if arg == "--suite" {
let Some(value) = iter.next() else {
return Err("--suite requires a value (e.g. --suite smoke)".to_string());
};
suite = Some(value.clone());
} else if let Some(value) = arg.strip_prefix("--suite=") {
suite = Some(value.to_string());
} else if arg == "--recommended" {
recommended = true;
} else {
positional.push(arg.clone());
}
}
let command = match positional.as_slice() {
[] => return Err("no command given".to_string()),
[word] if word == "help" || word == "--help" || word == "-h" => Command::Help,
[word] if word == "version" => Command::Version,
[word] if word == "doctor" => Command::Doctor,
[first, rest @ ..] if first == "catalog" => match rest {
[sub] if sub == "get" => Command::CatalogGet,
[sub] if sub == "list" => Command::CatalogList,
[sub] if sub == "refresh" => Command::CatalogRefresh,
[sub] if sub == "diff" => Command::CatalogDiff,
[sub, export_format] if sub == "export" => {
Command::CatalogExport { format: parse_export_format(export_format)? }
},
[sub] if sub == "export" => return Err("catalog export requires a format: yaml|markdown".to_string()),
_ => return Err(format!("unknown catalog subcommand: {}", positional.join(" "))),
},
[first, rest @ ..] if first == "model" => match rest {
[sub, model_id] if sub == "inspect" => Command::ModelInspect { model_id: model_id.clone() },
[sub] if sub == "inspect" => {
return Err(
"model inspect requires a model id, e.g. model inspect @cf/deepseek-ai/deepseek-v4-flash-0731"
.to_string(),
);
},
[sub, model_id] if sub == "verify" => Command::ModelVerify {
model_id: model_id.clone(),
suite: SuiteKind::parse(suite.as_deref())?,
recommended,
},
[sub] if sub == "verify" && recommended => Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::parse(suite.as_deref())?,
recommended,
},
[sub] if sub == "verify" => {
return Err(
"model verify requires a model id (or --recommended), e.g. model verify @cf/deepseek-ai/deepseek-v4-flash-0731 [--suite smoke]"
.to_string(),
);
},
[sub] if sub == "health" => Command::ModelHealth,
_ => return Err(format!("unknown model subcommand: {}", positional.join(" "))),
},
[first, rest @ ..] if first == "policy" => match rest {
[sub] if sub == "get" => Command::PolicyGet,
_ => return Err(format!("unknown policy subcommand: {}", positional.join(" "))),
},
_ => return Err(format!("unknown command: {}", positional.join(" "))),
};
if suite.is_some() && !matches!(command, Command::ModelVerify { .. }) {
return Err("--suite is only valid with 'model verify'".to_string());
}
if recommended && !matches!(command, Command::ModelVerify { .. }) {
return Err("--recommended is only valid with 'model verify'".to_string());
}
Ok((command, format))
}
fn parse_export_format(value: &str) -> Result<ExportFormat, String> {
match value {
"yaml" | "yml" => Ok(ExportFormat::Yaml),
"markdown" | "md" => Ok(ExportFormat::Markdown),
other => Err(format!("unsupported export format {other}: use yaml or markdown")),
}
}
fn execute(command: Command, out: &mut dyn Write, err: &mut dyn Write, fetch: FetchCatalog) -> i32 {
match command {
Command::Help => {
let _ = writeln!(out, "{}", USAGE.replace("{version}", VERSION));
EXIT_OK
},
Command::Version => {
let (code, value) = version_json();
emit_json(out, err, &value, code)
},
Command::Doctor => {
let (code, value) = doctor_json();
emit_json(out, err, &value, code)
},
Command::CatalogGet => {
let (code, value) = catalog_get_json(fetch);
emit_json(out, err, &value, code)
},
Command::CatalogList => {
let (code, value) = catalog_list_json(fetch);
emit_json(out, err, &value, code)
},
Command::CatalogRefresh => {
let (code, value) = catalog_refresh_json(fetch, err);
emit_json(out, err, &value, code)
},
Command::CatalogExport { format } => catalog_export(format, fetch, out, err),
Command::CatalogDiff => {
let (code, value) = catalog_diff_json();
emit_json(out, err, &value, code)
},
Command::ModelInspect { model_id } => {
let (code, value) = model_inspect_json(fetch, &model_id);
emit_json(out, err, &value, code)
},
Command::ModelVerify { model_id, suite, recommended } => {
let (code, value) = model_verify_json(&model_id, suite, recommended, err);
emit_json(out, err, &value, code)
},
Command::ModelHealth => {
let (code, value) = model_health_json();
emit_json(out, err, &value, code)
},
Command::PolicyGet => {
let (code, value) = policy_get_json();
emit_json(out, err, &value, code)
},
}
}
fn emit_json(out: &mut dyn Write, err: &mut dyn Write, value: &serde_json::Value, code: i32) -> i32 {
match serde_json::to_string_pretty(value) {
Ok(json) => {
let _ = writeln!(out, "{json}");
if code != EXIT_OK {
if let Some(message) = value.get("error").and_then(|e| e.as_str()) {
let _ = writeln!(err, "auth-cloudflare: {message} (exit {code})");
}
}
},
Err(error) => {
let _ = writeln!(err, "auth-cloudflare: failed to serialize JSON output: {error}");
return EXIT_OPERATIONAL;
},
}
code
}
fn version_json() -> (i32, serde_json::Value) {
(
EXIT_OK,
serde_json::to_value(VersionInfo::current()).expect("VersionInfo serializes"),
)
}
fn doctor_json() -> (i32, serde_json::Value) {
let resolved = resolve_config();
let account_id = resolved.account_id.clone();
let account_configured = account_id.is_some();
let token_configured = resolved.token_configured;
let unsafe_config = detect_unsafe_config(&resolved.config_path);
let (redacted, base_url, cache_dir) = match &account_id {
Some(id) => {
let redacted = redact_account_id(id);
let base_url = match &resolved.base_url_override {
Some(url) => Some(url.clone()),
None => Some(AuthProvider::new(id).base_url().replace(id.as_str(), "<redacted>")),
};
(Some(redacted), base_url, resolved.cache_dir.clone())
},
None => (None, None, None),
};
let (cache_present, cache_age) = match &cache_dir {
Some(dir) => match read_catalog_cache(dir) {
Ok(Some((meta, _))) => (true, cache_age_seconds(&meta)),
_ => (false, 0),
},
None => (false, 0),
};
let status = if account_configured && token_configured && !unsafe_config {
"ok"
} else {
"error"
};
let exit = if unsafe_config {
EXIT_UNSAFE_CONFIG
} else if !account_configured || !token_configured {
EXIT_CREDENTIALS
} else {
EXIT_OK
};
let value = serde_json::json!({
"status": status,
"account_id": {
"configured": account_configured,
"redacted": redacted,
},
"api_token": {
"configured": token_configured,
"value_redacted": true,
},
"endpoint": {
"base_url": base_url,
},
"catalog_cache": {
"present": cache_present,
"age_seconds": cache_age,
},
});
(exit, value)
}
struct ResolvedConfig {
account_id: Option<String>,
token_configured: bool,
cache_dir: Option<PathBuf>,
base_url_override: Option<String>,
config_path: PathBuf,
}
fn resolve_config() -> ResolvedConfig {
let config_path = env_nonempty(CONFIG_ENV)
.map(PathBuf::from)
.unwrap_or_else(|| hermes_home().join("auth-cloudflare").join(CONFIG_FILE_NAME));
let file = read_config_file(&config_path);
let account_id = env_nonempty(ACCOUNT_ID_ENV)
.or_else(|| env_nonempty(ACCOUNT_ENV))
.or_else(|| file.account_id.clone())
.filter(|value| is_valid_account_id(value));
let token_configured = env_nonempty(API_TOKEN_ENV)
.or_else(|| env_nonempty(TOKEN_ENV))
.or_else(|| env_nonempty(LEGACY_HERMES_TOKEN_ENV))
.or_else(|| file.api_token_env.as_deref().and_then(env_nonempty))
.is_some();
let base_url_override = env_nonempty(BASE_URL_ENV).or_else(|| file.base_url.clone());
let cache_dir = env_nonempty(CACHE_DIR_ENV)
.map(PathBuf::from)
.or_else(|| file.cache_dir.map(PathBuf::from))
.or_else(|| account_id.as_ref().map(|id| cache_dir_for_account(&AuthProvider::new(id))));
ResolvedConfig { account_id, token_configured, cache_dir, base_url_override, config_path }
}
#[derive(Default, serde::Deserialize)]
struct ConfigFile {
account_id: Option<String>,
base_url: Option<String>,
cache_dir: Option<String>,
api_token_env: Option<String>,
}
fn read_config_file(path: &Path) -> ConfigFile {
let Ok(raw) = std::fs::read_to_string(path) else {
return ConfigFile::default();
};
serde_json::from_str(&raw).unwrap_or_default()
}
fn env_nonempty(name: &str) -> Option<String> {
std::env::var(name).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty())
}
fn is_valid_account_id(value: &str) -> bool {
value.len() == ACCOUNT_ID_LEN && value.bytes().all(|b| b.is_ascii_hexdigit())
}
fn hermes_home() -> PathBuf {
env_nonempty("HERMES_HOME")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(std::env::var("HOME").unwrap_or_else(|_| "~".to_string())).join(".hermes"))
}
fn detect_unsafe_config(path: &Path) -> bool {
let Ok(raw) = std::fs::read_to_string(path) else {
return false;
};
let Ok(value) = serde_json::from_str::<serde_json::Value>(&raw) else {
return false;
};
match value.get("api_token") {
Some(serde_json::Value::String(token)) => !token.trim().is_empty(),
_ => false,
}
}
fn redact_account_id(id: &str) -> String {
let chars: Vec<char> = id.chars().collect();
let n = chars.len();
if n >= 10 {
let head: String = chars[..6].iter().collect();
let tail: String = chars[n - 4..].iter().collect();
format!("{head}…{tail}")
} else if n >= 4 {
let head: String = chars[..2].iter().collect();
let tail: String = chars[n - 2..].iter().collect();
format!("{head}…{tail}")
} else if n >= 2 {
let head: String = chars[..1].iter().collect();
format!("{head}…")
} else {
"…".to_string()
}
}
fn cache_age_seconds(meta: &CatalogCacheMeta) -> u64 {
match chrono::DateTime::parse_from_rfc3339(&meta.fetched_at) {
Ok(fetched_at) => chrono::Utc::now().signed_duration_since(fetched_at).num_seconds().max(0) as u64,
Err(_) => 0,
}
}
struct ResolvedCatalog {
source: &'static str,
cache_status: &'static str,
fetched_at: String,
records: Vec<ModelRecord>,
}
fn resolve_catalog_data(fetch: FetchCatalog) -> ResolvedCatalog {
let resolved_config = resolve_config();
let cache_dir = resolved_config.cache_dir.filter(|_| resolved_config.account_id.is_some());
if let Some(dir) = &cache_dir {
if let Ok(Some((meta, payload))) = read_catalog_cache(dir) {
let records = records_from_payload(&payload);
if !records.is_empty() {
let stale = cache_is_stale(&meta, CACHE_MAX_AGE);
return ResolvedCatalog {
source: "cache",
cache_status: if stale { "stale" } else { "fresh" },
fetched_at: meta.fetched_at,
records,
};
}
}
}
if let Some((account_id, token)) = live_credentials() {
if let Ok(payload) = fetch(&account_id, &token, FETCH_TIMEOUT) {
let records = records_from_payload(&payload);
if !records.is_empty() {
let fetched_at = chrono::Utc::now().to_rfc3339();
if let Some(dir) = &cache_dir {
let meta = CatalogCacheMeta {
schema_version: CATALOG_SCHEMA_VERSION,
fetched_at: fetched_at.clone(),
source: "cloudflare-workers-ai".to_string(),
model_count: records.len(),
account_fingerprint: AuthProvider::new(&account_id).cache_slug(),
};
let _ = write_catalog_cache(dir, &meta, &payload);
}
return ResolvedCatalog { source: "live", cache_status: "fresh", fetched_at, records };
}
}
}
ResolvedCatalog {
source: "fallback",
cache_status: "none",
fetched_at: chrono::Utc::now().to_rfc3339(),
records: fallback_records(),
}
}
fn live_credentials() -> Option<(String, SecretString)> {
let config = Config::from_env().ok()?;
Some((config.account_id().to_string(), config.api_token().clone()))
}
fn records_from_payload(payload: &serde_json::Value) -> Vec<ModelRecord> {
match payload.get("data").and_then(|data| data.as_array()) {
Some(items) => items.iter().filter_map(ModelRecord::from_openrouter).collect(),
None => Vec::new(),
}
}
fn fallback_records() -> Vec<ModelRecord> {
let data: Vec<serde_json::Value> = FALLBACK_MODELS
.iter()
.map(|id| serde_json::json!({ "id": id, "name": display_name_from_id(id) }))
.collect();
records_from_payload(&serde_json::json!({ "data": data }))
}
fn display_name_from_id(id: &str) -> String {
let rest = id.strip_prefix("@cf/").unwrap_or(id);
let model = rest.split('/').next_back().unwrap_or(rest);
let mut out = String::new();
let mut capitalize_next = true;
let mut previous_was_digit = false;
for ch in model.chars() {
if ch == '-' || ch == '_' {
out.push(' ');
capitalize_next = true;
previous_was_digit = false;
} else if capitalize_next || previous_was_digit {
out.extend(ch.to_uppercase());
capitalize_next = false;
previous_was_digit = ch.is_ascii_digit();
} else {
out.push(ch);
previous_was_digit = ch.is_ascii_digit();
}
}
out.split_whitespace()
.map(|word| match word {
"Deepseek" => "DeepSeek".to_string(),
"Openai" => "OpenAI".to_string(),
"Glm" => "GLM".to_string(),
other => other.to_string(),
})
.collect::<Vec<_>>()
.join(" ")
}
fn model_json(record: &ModelRecord, policy: &ModelPolicy) -> serde_json::Value {
serde_json::json!({
"id": record.id,
"display_name": record.display_name,
"status": policy.status_for(&record.id),
"primary_agent_eligible": policy.is_primary_agent_eligible(&record.id),
"context_tokens": record.limits.context_tokens,
"pricing_per_million": {
"input": record.pricing.input,
"cached_input": record.pricing.cached_input,
"output": record.pricing.output,
},
"capabilities": {
"chat": record.capabilities.chat,
"tools": record.capabilities.tools,
"reasoning": record.capabilities.reasoning,
},
})
}
fn catalog_document(
source: &str,
cache_status: &str,
fetched_at: String,
records: &[ModelRecord],
) -> serde_json::Value {
let policy = ModelPolicy::default_policy();
let models: Vec<serde_json::Value> = records.iter().map(|record| model_json(record, &policy)).collect();
serde_json::json!({
"schema_version": CATALOG_SCHEMA_VERSION,
"source": source,
"fetched_at": fetched_at,
"cache_status": cache_status,
"default_model": DEFAULT_MODEL,
"model_count": records.len(),
"experimental_included": true,
"deprecated_included": false,
"models": models,
})
}
fn catalog_get_json(fetch: FetchCatalog) -> (i32, serde_json::Value) {
let resolved = resolve_catalog_data(fetch);
let exit = match resolved.cache_status {
"stale" => EXIT_STALE_CACHE,
_ => EXIT_OK,
};
let value = catalog_document(resolved.source, resolved.cache_status, resolved.fetched_at, &resolved.records);
(exit, value)
}
fn catalog_list_json(fetch: FetchCatalog) -> (i32, serde_json::Value) {
let resolved = resolve_catalog_data(fetch);
let models: Vec<String> = resolved.records.iter().map(|record| record.id.clone()).collect();
let exit = match resolved.cache_status {
"stale" => EXIT_STALE_CACHE,
_ => EXIT_OK,
};
let value = serde_json::json!({
"schema_version": CATALOG_SCHEMA_VERSION,
"source": resolved.source,
"cache_status": resolved.cache_status,
"default_model": DEFAULT_MODEL,
"model_count": models.len(),
"experimental_included": true,
"deprecated_included": false,
"models": models,
});
(exit, value)
}
fn catalog_refresh_json(fetch: FetchCatalog, err: &mut dyn Write) -> (i32, serde_json::Value) {
let config = match Config::from_env() {
Ok(config) => config,
Err(error) => {
return (
EXIT_CREDENTIALS,
serde_json::json!({
"status": "error",
"error": error.to_string(),
"exit_code": EXIT_CREDENTIALS,
}),
);
},
};
let account_id = config.account_id().to_string();
let cache_dir = config.cache_dir();
match fetch(&account_id, config.api_token(), FETCH_TIMEOUT) {
Ok(payload) => {
let records = records_from_payload(&payload);
if records.is_empty() {
return refresh_failure(&CloudflareError::NoDataArray, &cache_dir, err);
}
let fetched_at = chrono::Utc::now().to_rfc3339();
let meta = CatalogCacheMeta {
schema_version: CATALOG_SCHEMA_VERSION,
fetched_at: fetched_at.clone(),
source: "cloudflare-workers-ai".to_string(),
model_count: records.len(),
account_fingerprint: AuthProvider::new(&account_id).cache_slug(),
};
if let Err(write_error) = write_catalog_cache(&cache_dir, &meta, &payload) {
let _ = writeln!(
err,
"auth-cloudflare: warning: catalog refreshed but cache write failed: {write_error}"
);
}
(
EXIT_OK,
catalog_document("cloudflare-workers-ai", "fresh", fetched_at, &records),
)
},
Err(error) => refresh_failure(&error, &cache_dir, err),
}
}
fn refresh_failure(error: &CloudflareError, cache_dir: &Path, err: &mut dyn Write) -> (i32, serde_json::Value) {
if let Ok(Some((meta, payload))) = read_catalog_cache(cache_dir) {
let records = records_from_payload(&payload);
if !records.is_empty() {
let stale = cache_is_stale(&meta, CACHE_MAX_AGE);
let cache_status = if stale { "stale" } else { "fresh" };
let exit = if stale { EXIT_STALE_CACHE } else { EXIT_OK };
let _ = writeln!(
err,
"auth-cloudflare: live catalog fetch failed ({error}); serving {cache_status} cache (exit {exit})"
);
return (
exit,
catalog_document("cloudflare-workers-ai", cache_status, meta.fetched_at, &records),
);
}
}
(
EXIT_REMOTE_API,
serde_json::json!({
"status": "error",
"error": error.to_string(),
"exit_code": EXIT_REMOTE_API,
}),
)
}
fn catalog_diff_json() -> (i32, serde_json::Value) {
let resolved = resolve_config();
let cache_dir = resolved.cache_dir.filter(|_| resolved.account_id.is_some());
let Some(cache_dir) = cache_dir else {
return (
EXIT_OPERATIONAL,
serde_json::json!({
"status": "error",
"error": "no account credentials resolved - cannot locate an account-scoped catalog cache to diff",
"exit_code": EXIT_OPERATIONAL,
}),
);
};
let (meta, payload) = match read_catalog_cache(&cache_dir) {
Ok(Some(found)) => found,
Ok(None) => {
return (
EXIT_OPERATIONAL,
serde_json::json!({
"status": "error",
"error": "no catalog cache present to diff against; run 'catalog refresh' to seed the cache",
"exit_code": EXIT_OPERATIONAL,
}),
);
},
Err(error) => {
return (
EXIT_OPERATIONAL,
serde_json::json!({
"status": "error",
"error": format!("catalog cache unreadable: {error}"),
"exit_code": EXIT_OPERATIONAL,
}),
);
},
};
let cache_ids: std::collections::BTreeSet<String> =
records_from_payload(&payload).iter().map(|record| record.id.clone()).collect();
let fallback_ids: std::collections::BTreeSet<String> = FALLBACK_MODELS.iter().map(|id| (*id).to_string()).collect();
let added: Vec<String> = fallback_ids.difference(&cache_ids).cloned().collect();
let removed: Vec<String> = cache_ids.difference(&fallback_ids).cloned().collect();
let common = cache_ids.intersection(&fallback_ids).count();
(
EXIT_OK,
serde_json::json!({
"schema_version": CATALOG_SCHEMA_VERSION,
"baseline": {
"source": "cache",
"fetched_at": meta.fetched_at,
"model_count": cache_ids.len(),
},
"comparison": {
"source": "fallback",
"model_count": fallback_ids.len(),
},
"added": added,
"removed": removed,
"common_count": common,
}),
)
}
fn catalog_export(format: ExportFormat, fetch: FetchCatalog, out: &mut dyn Write, err: &mut dyn Write) -> i32 {
let resolved = resolve_catalog_data(fetch);
let policy = ModelPolicy::default_policy();
let export_dir = std::env::var(EXPORT_DIR_ENV)
.ok()
.filter(|v| !v.trim().is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("."));
let (file_name, contents) = match format {
ExportFormat::Yaml => (
"catalog.generated.yaml",
export_yaml(&resolved.records, resolved.source, &resolved.fetched_at, &policy),
),
ExportFormat::Markdown => (
"catalog.generated.md",
export_markdown(&resolved.records, resolved.source, &resolved.fetched_at, &policy),
),
};
let path = export_dir.join(file_name);
if let Err(error) = std::fs::write(&path, contents) {
let _ = writeln!(err, "auth-cloudflare: failed to write {}: {error}", path.display());
return EXIT_OPERATIONAL;
}
let _ = writeln!(
out,
"wrote {} ({} models, source: {}, cache_status: {})",
path.display(),
resolved.records.len(),
resolved.source,
resolved.cache_status
);
EXIT_OK
}
fn model_inspect_json(fetch: FetchCatalog, model_id: &str) -> (i32, serde_json::Value) {
let resolved = resolve_catalog_data(fetch);
let policy = ModelPolicy::default_policy();
let Some(record) = resolved.records.iter().find(|record| record.id == model_id) else {
return (
EXIT_NO_ELIGIBLE_MODEL,
serde_json::json!({
"status": "error",
"error": format!("model '{model_id}' not found in catalog"),
"model_id": model_id,
"exit_code": EXIT_NO_ELIGIBLE_MODEL,
}),
);
};
let mut value = model_json(record, &policy);
value["source"] = serde_json::Value::String(resolved.source.to_string());
if let Some(warning) = degraded_model_warning(model_id) {
value["warning"] = serde_json::Value::String(warning);
}
(EXIT_OK, value)
}
enum RunOutcome {
Smoke(Box<verify::SmokeRunReport>),
ToolLoop(Box<auth_cloudflare::tool_loop::ToolLoopOutcome>),
}
fn model_verify_json(
model_id: &str,
suite: SuiteKind,
recommended: bool,
err: &mut dyn Write,
) -> (i32, serde_json::Value) {
let model_id = if recommended { DEFAULT_MODEL } else { model_id };
let suite_name = suite.as_str();
let gate = if verify::live_tests_enabled() { "open" } else { "closed" };
if gate == "closed" {
return (
EXIT_OPERATIONAL,
serde_json::json!({
"model_id": model_id,
"suite": suite_name,
"gate": "closed",
"status": "error",
"error": "live tests disabled (set AUTH_CLOUDFLARE_LIVE_TESTS=1 to allow paid inference)",
"passed": false,
"exit_code": EXIT_OPERATIONAL,
}),
);
}
let config = match Config::from_env() {
Ok(config) => config,
Err(error) => {
return (
EXIT_CREDENTIALS,
serde_json::json!({
"model_id": model_id,
"suite": suite_name,
"gate": "open",
"status": "error",
"error": error.to_string(),
"passed": false,
"exit_code": EXIT_CREDENTIALS,
}),
);
},
};
let result: Result<RunOutcome, CloudflareError> = match suite {
SuiteKind::Smoke => {
verify::run_smoke_suite(&config, model_id).map(|report| RunOutcome::Smoke(Box::new(report)))
},
SuiteKind::ToolLoop => {
let base_url = match config.base_url() {
Ok(url) => url,
Err(error) => {
return (
EXIT_CREDENTIALS,
serde_json::json!({
"model_id": model_id,
"suite": suite_name,
"gate": gate,
"status": "error",
"error": error.to_string(),
"passed": false,
"exit_code": EXIT_CREDENTIALS,
}),
);
},
};
auth_cloudflare::tool_loop::run_tool_loop(
config.account_id(),
config.api_token(),
&base_url,
model_id,
TOOL_LOOP_TIMEOUT,
)
.map(|outcome| RunOutcome::ToolLoop(Box::new(outcome)))
},
};
match result {
Ok(RunOutcome::Smoke(report)) => {
let dir = config.cache_dir();
if let Err(error) = verify::save_verification(&dir, &report.verification) {
let _ = writeln!(err, "auth-cloudflare: warning: model-health.json persistence failed: {error}");
}
let exit = if report.passed { EXIT_OK } else { EXIT_CONFORMANCE };
let mut value = serde_json::to_value(&report).expect("SmokeRunReport serializes");
value["gate"] = serde_json::Value::String(gate.to_string());
value["exit_code"] = serde_json::json!(exit);
(exit, value)
},
Ok(RunOutcome::ToolLoop(report)) => {
let verification = verify::verification_from_tool_loop(model_id, &report);
let dir = config.cache_dir();
if let Err(error) = verify::save_verification(&dir, &verification) {
let _ = writeln!(err, "auth-cloudflare: warning: model-health.json persistence failed: {error}");
}
let exit = if report.converged { EXIT_OK } else { EXIT_CONFORMANCE };
let mut value = serde_json::to_value(&report).expect("ToolLoopOutcome serializes");
value["verification"] = serde_json::to_value(&verification).expect("verification serializes");
value["gate"] = serde_json::Value::String(gate.to_string());
value["exit_code"] = serde_json::json!(exit);
(exit, value)
},
Err(CloudflareError::MissingEnv { env_var, .. }) if env_var == verify::LIVE_TESTS_ENV => {
(
EXIT_OPERATIONAL,
serde_json::json!({
"model_id": model_id,
"suite": suite_name,
"gate": gate,
"status": "error",
"error": "live tests disabled (set AUTH_CLOUDFLARE_LIVE_TESTS=1 to allow paid inference)",
"passed": false,
"exit_code": EXIT_OPERATIONAL,
}),
)
},
Err(error) => {
(
EXIT_REMOTE_API,
serde_json::json!({
"model_id": model_id,
"suite": suite_name,
"gate": gate,
"status": "error",
"error": error.to_string(),
"passed": false,
"exit_code": EXIT_REMOTE_API,
}),
)
},
}
}
fn model_health_json() -> (i32, serde_json::Value) {
let resolved = resolve_config();
if resolved.account_id.is_none() || !resolved.token_configured {
return (
EXIT_CREDENTIALS,
serde_json::json!({
"status": "error",
"error": "credentials missing: export AUTH_CLOUDFLARE_ACCOUNT_ID and AUTH_CLOUDFLARE_API_TOKEN (or the legacy CLOUDFLARE_* aliases)",
"exit_code": EXIT_CREDENTIALS,
}),
);
}
let Some(cache_dir) = resolved.cache_dir else {
return (
EXIT_OPERATIONAL,
serde_json::json!({
"status": "error",
"error": "no account-scoped cache directory resolved",
"exit_code": EXIT_OPERATIONAL,
}),
);
};
match verify::load_health_store(&cache_dir) {
Ok(store) => (
EXIT_OK,
serde_json::json!({
"status": "ok",
"version": store.version,
"updated_at": store.updated_at.to_rfc3339(),
"records": serde_json::to_value(&store.records).expect("records serialize"),
"exit_code": EXIT_OK,
}),
),
Err(error) => (
EXIT_OPERATIONAL,
serde_json::json!({
"status": "error",
"error": error.to_string(),
"exit_code": EXIT_OPERATIONAL,
}),
),
}
}
fn policy_get_json() -> (i32, serde_json::Value) {
let policy = ModelPolicy::default_policy();
let mut value = serde_json::to_value(policy).expect("ModelPolicy serializes");
if let Some(store) = load_health_store_best_effort() {
let warnings: Vec<serde_json::Value> = store
.records
.values()
.filter_map(|verification| {
degraded_warning(verification).map(|message| {
serde_json::json!({
"model_id": verification.model_id,
"message": message,
})
})
})
.collect();
if !warnings.is_empty() {
value["warnings"] = serde_json::Value::Array(warnings);
}
}
(EXIT_OK, value)
}
fn load_health_store_best_effort() -> Option<verify::HealthStore> {
let cache_dir = resolve_config().cache_dir?;
verify::load_health_store(&cache_dir).ok()
}
fn degraded_warning(verification: &health::ModelVerification) -> Option<String> {
let degraded = matches!(
verification.status,
health::VerificationStatus::Degraded | health::VerificationStatus::Failing
);
if !degraded {
return None;
}
let alternative = health::recommended_stable_alternative(&verification.model_id);
let alternative = if alternative.is_empty() { DEFAULT_MODEL } else { alternative };
Some(format!(
"model {} delivery is degraded (verification status: {}); recommended stable alternative: {}",
verification.model_id,
enum_str(&verification.status),
alternative
))
}
fn degraded_model_warning(model_id: &str) -> Option<String> {
let store = load_health_store_best_effort()?;
let verification = store.get(model_id)?;
degraded_warning(verification)
}
fn export_yaml(records: &[ModelRecord], source: &str, fetched_at: &str, policy: &ModelPolicy) -> String {
let mut s = String::new();
s.push_str("# GENERATED FILE - do not edit by hand.\n");
s.push_str(&format!("# Source: Cloudflare Workers AI catalog ({source})\n"));
s.push_str(&format!("# Refreshed: {fetched_at}\n"));
s.push_str("# Plugin policy: model-policy.yaml\n");
s.push_str("# Verification: model-health.json\n");
s.push_str("\nmodels:\n");
for record in records {
let status = policy.status_for(&record.id);
let status_str = enum_str(&status);
let marker: String = if record.id == DEFAULT_MODEL {
"DEFAULT".to_string()
} else {
status_str.clone()
};
s.push_str(&format!(" # {marker} | {}\n", record.display_name));
s.push_str(&format!(" # Status: {status_str}\n"));
match record.limits.context_tokens {
Some(tokens) => s.push_str(&format!(" # Context: {tokens} tokens\n")),
None => s.push_str(" # Context: unknown\n"),
}
let price = |value: Option<f64>| match value {
Some(v) => format!("${v:.2}/M"),
None => "unknown".to_string(),
};
s.push_str(&format!(
" # Price: {} input; {} cached input; {} output\n",
price(record.pricing.input),
price(record.pricing.cached_input),
price(record.pricing.output)
));
s.push_str(&format!(" # Tools: {}\n", enum_str(&record.capabilities.tools)));
s.push_str(&format!(" # Reasoning: {}\n", enum_str(&record.capabilities.reasoning)));
s.push_str(&format!(" - \"{}\"\n", record.id));
}
s
}
fn export_markdown(records: &[ModelRecord], source: &str, fetched_at: &str, policy: &ModelPolicy) -> String {
let eligible = records
.iter()
.filter(|record| policy.is_primary_agent_eligible(&record.id))
.count();
let mut s = String::new();
s.push_str("# Cloudflare Workers AI catalog\n\n");
s.push_str(&format!("Generated: {fetched_at}\n"));
s.push_str(&format!("Plugin: {VERSION}\n"));
s.push_str(&format!("Catalog: {source}\n"));
s.push_str(&format!("Cache age: {} seconds\n", doc_cache_age_seconds(fetched_at)));
s.push_str(&format!("Models found: {}\n", records.len()));
s.push_str(&format!("Primary-agent eligible: {eligible}\n\n"));
s.push_str("| Model | Status | Context | Input/M | Cached/M | Output/M | Tools | Reasoning |\n");
s.push_str("| --- | --- | ---: | ---: | ---: | ---: | --- | --- |\n");
for record in records {
let status_str = enum_str(&policy.status_for(&record.id));
let context = record
.limits
.context_tokens
.map(|tokens| format!("{tokens}"))
.unwrap_or_else(|| "-".to_string());
let price = |value: Option<f64>| match value {
Some(v) => format!("${v:.2}"),
None => "-".to_string(),
};
s.push_str(&format!(
"| {} | {} | {} | {} | {} | {} | {} | {} |\n",
record.display_name,
status_str,
context,
price(record.pricing.input),
price(record.pricing.cached_input),
price(record.pricing.output),
enum_str(&record.capabilities.tools),
enum_str(&record.capabilities.reasoning)
));
}
s
}
fn enum_str<T: serde::Serialize>(value: &T) -> String {
serde_json::to_value(value)
.ok()
.and_then(|value| value.as_str().map(str::to_string))
.unwrap_or_else(|| "unknown".to_string())
}
fn doc_cache_age_seconds(fetched_at: &str) -> u64 {
match chrono::DateTime::parse_from_rfc3339(fetched_at) {
Ok(timestamp) => chrono::Utc::now().signed_duration_since(timestamp).num_seconds().max(0) as u64,
Err(_) => 0,
}
}
#[cfg(test)]
mod tests {
use super::*;
use auth_cloudflare::auth::{ACCOUNT_ENV, TOKEN_ENV};
const ALL_VARS: &[&str] = &[
"AUTH_CLOUDFLARE_ACCOUNT_ID",
"AUTH_CLOUDFLARE_API_TOKEN",
"AUTH_CLOUDFLARE_WORKERS_AI_BASE_URL",
"AUTH_CLOUDFLARE_CACHE_DIR",
"AUTH_CLOUDFLARE_CONFIG",
"AUTH_CLOUDFLARE_EXPORT_DIR",
"AUTH_CLOUDFLARE_LIVE_TESTS",
"AUTH_CLOUDFLARE_MAX_COST_USD",
"CLOUDFLARE_ACCOUNT_ID",
"CLOUDFLARE_API_TOKEN",
"HERMES_CUSTOM_API_CLOUDFLARE_COM_API_KEY",
"HERMES_HOME",
"HOME",
];
const ACCOUNT: &str = "0123456789abcdef0123456789abcdef";
const TOKEN: &str = "cfut_test_synthetic_token_0001";
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn with_env<F, R>(vars: &[(&str, Option<&str>)], f: F) -> R
where
F: FnOnce() -> R,
{
let _guard = ENV_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let saved: Vec<(String, Option<String>)> = ALL_VARS
.iter()
.map(|key| ((*key).to_string(), std::env::var(key).ok()))
.collect();
for key in ALL_VARS {
std::env::remove_var(key);
}
for (key, value) in vars {
match value {
Some(value) => std::env::set_var(key, value),
None => std::env::remove_var(key),
}
}
let result = f();
for (key, value) in saved {
match value {
Some(value) => std::env::set_var(&key, value),
None => std::env::remove_var(&key),
}
}
result
}
fn scratch_dir(name: &str) -> PathBuf {
std::env::temp_dir().join(format!("auth-cloudflare-cli-test-{}-{name}", std::process::id()))
}
fn run_json(args: &[&str]) -> (i32, serde_json::Value) {
let owned: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
let mut out: Vec<u8> = Vec::new();
let mut err: Vec<u8> = Vec::new();
let code = run(&owned, &mut out, &mut err);
let value = serde_json::from_slice(&out).unwrap_or_else(|_| serde_json::json!({}));
(code, value)
}
fn run_raw(args: &[&str]) -> (i32, String) {
let owned: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
let mut out: Vec<u8> = Vec::new();
let mut err: Vec<u8> = Vec::new();
let code = run(&owned, &mut out, &mut err);
(code, String::from_utf8_lossy(&out).to_string())
}
fn run_json_with_fetch(args: &[&str], fetch: FetchCatalog) -> (i32, serde_json::Value) {
let owned: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
let mut out: Vec<u8> = Vec::new();
let mut err: Vec<u8> = Vec::new();
let code = run_with_fetch(&owned, &mut out, &mut err, fetch);
let value = serde_json::from_slice(&out).unwrap_or_else(|_| serde_json::json!({}));
(code, value)
}
fn run_raw_with_fetch(args: &[&str], fetch: FetchCatalog) -> (i32, String) {
let owned: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
let mut out: Vec<u8> = Vec::new();
let mut err: Vec<u8> = Vec::new();
let code = run_with_fetch(&owned, &mut out, &mut err, fetch);
(code, String::from_utf8_lossy(&out).to_string())
}
fn fetch_ok(
_account_id: &str,
_token: &SecretString,
_timeout: Duration,
) -> Result<serde_json::Value, CloudflareError> {
Ok(serde_json::json!({
"data": [
{
"id": DEFAULT_MODEL,
"name": "DeepSeek V4 Flash 0731",
"context_length": 1_310_720,
"pricing": { "prompt": "0.00000044", "completion": "0.00000132" },
},
{ "id": "@cf/moonshotai/kimi-k2.7-code", "name": "Kimi K2.7 Code" },
]
}))
}
fn fetch_err(
_account_id: &str,
_token: &SecretString,
_timeout: Duration,
) -> Result<serde_json::Value, CloudflareError> {
Err(CloudflareError::Http("simulated network failure".to_string()))
}
fn seed_cache(fetched_at: &str, model_ids: &[&str]) {
let dir = cache_dir_for_account(&AuthProvider::new(ACCOUNT));
let data: Vec<serde_json::Value> = model_ids
.iter()
.map(|id| serde_json::json!({ "id": id, "name": display_name_from_id(id) }))
.collect();
let payload = serde_json::json!({ "data": data });
let meta = CatalogCacheMeta {
schema_version: CATALOG_SCHEMA_VERSION,
fetched_at: fetched_at.to_string(),
source: "cloudflare-workers-ai".to_string(),
model_count: model_ids.len(),
account_fingerprint: "0123456789abcdef".to_string(),
};
auth_cloudflare::cache::write_catalog_cache(&dir, &meta, &payload).expect("seed cache");
}
fn verification_record(model_id: &str, status: health::VerificationStatus) -> health::ModelVerification {
health::ModelVerification {
model_id: model_id.to_string(),
latest_run_at: Some(chrono::Utc::now()),
expires_at: None,
suite_version: health::CONFORMANCE_SUITE_VERSION.to_string(),
runner_version: "test".to_string(),
status,
agent_eligible: true,
confidence: health::VerificationConfidence::SmokeTested,
total_runs: 3,
successful_runs: 3,
text_completion_success_rate: Some(1.0),
stream_completion_success_rate: Some(1.0),
single_tool_success_rate: Some(1.0),
multi_turn_tool_success_rate: None,
structured_output_success_rate: None,
median_latency_ms: Some(100),
p95_latency_ms: None,
total_failures: 0,
timeout_failures: 0,
transport_failures: 0,
provider_5xx_failures: 0,
malformed_response_failures: 0,
malformed_tool_call_failures: 0,
tool_loop_failures: 0,
last_failure: None,
}
}
fn seed_health_store(records: &[(&str, health::VerificationStatus)]) {
let dir = cache_dir_for_account(&AuthProvider::new(ACCOUNT));
let mut store = verify::HealthStore::new();
for (model_id, status) in records {
store.upsert(verification_record(model_id, *status));
}
verify::save_health_store(&dir, &store).expect("seed health store");
}
#[test]
fn parses_every_command() {
let cases: &[(&[&str], Command)] = &[
(&["version"], Command::Version),
(&["version", "--format", "json"], Command::Version),
(&["--format", "json", "version"], Command::Version),
(&["doctor", "--format=json"], Command::Doctor),
(&["catalog", "get", "--format", "json"], Command::CatalogGet),
(&["catalog", "list"], Command::CatalogList),
(&["catalog", "refresh", "--format", "json"], Command::CatalogRefresh),
(
&["catalog", "export", "yaml"],
Command::CatalogExport { format: ExportFormat::Yaml },
),
(
&["catalog", "export", "markdown"],
Command::CatalogExport { format: ExportFormat::Markdown },
),
(&["catalog", "diff", "--format", "json"], Command::CatalogDiff),
(
&["model", "inspect", "@cf/deepseek-ai/deepseek-v4-flash-0731", "--format", "json"],
Command::ModelInspect { model_id: "@cf/deepseek-ai/deepseek-v4-flash-0731".to_string() },
),
(&["policy", "get", "--format", "json"], Command::PolicyGet),
];
for (args, expected) in cases {
let owned: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
let (command, format) = parse_args(&owned).unwrap_or_else(|e| panic!("{args:?} should parse: {e}"));
assert_eq!(&command, expected, "args {args:?}");
let _ = format;
}
}
#[test]
fn unknown_command_is_a_parse_error() {
let owned = vec!["frobnicate".to_string()];
assert!(parse_args(&owned).is_err(), "unknown command must fail parsing");
let owned = vec!["catalog".to_string(), "fly".to_string()];
assert!(parse_args(&owned).is_err());
let owned = vec!["model".to_string(), "inspect".to_string()];
assert!(parse_args(&owned).is_err(), "inspect without id must fail");
}
#[test]
fn unknown_command_exits_1() {
with_env(&[], || {
assert_eq!(run_raw(&["frobnicate"]).0, EXIT_OPERATIONAL);
});
}
#[test]
fn bad_format_flag_exits_1() {
with_env(&[], || {
assert_eq!(run_raw(&["version", "--format", "xml"]).0, EXIT_OPERATIONAL);
});
}
#[test]
fn bad_export_format_exits_1() {
with_env(&[], || {
assert_eq!(run_raw(&["catalog", "export", "toml"]).0, EXIT_OPERATIONAL);
});
}
#[test]
fn help_exits_0() {
with_env(&[], || {
assert_eq!(run_raw(&["help"]).0, EXIT_OK);
assert_eq!(run_raw(&["--help"]).0, EXIT_OK);
});
}
#[test]
fn version_json_exact_feedback_06_shape() {
with_env(&[], || {
let (code, value) = run_json(&["version", "--format", "json"]);
assert_eq!(code, EXIT_OK);
let object = value.as_object().expect("object");
let keys: std::collections::BTreeSet<&str> = object.keys().map(String::as_str).collect();
let expected: std::collections::BTreeSet<&str> = [
"name",
"package_version",
"protocol_version",
"catalog_schema_versions",
"minimum_hermes_plugin_version",
]
.into_iter()
.collect();
assert_eq!(keys, expected);
assert_eq!(value["name"], "auth-cloudflare");
assert_eq!(value["package_version"], VERSION);
assert_eq!(value["protocol_version"], 1);
assert_eq!(value["catalog_schema_versions"], serde_json::json!([1]));
assert_eq!(value["minimum_hermes_plugin_version"], "0.0.1");
});
}
#[test]
fn version_defaults_to_json() {
with_env(&[], || {
let (code, value) = run_json(&["version"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["name"], "auth-cloudflare");
});
}
#[test]
fn doctor_redacts_token_and_account() {
let home = scratch_dir("doctor-redact");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json(&["doctor", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["status"], "ok");
assert_eq!(value["account_id"]["configured"], true);
assert_eq!(value["account_id"]["redacted"], "012345…cdef");
assert_eq!(value["api_token"]["configured"], true);
assert_eq!(value["api_token"]["value_redacted"], true);
assert_eq!(
value["endpoint"]["base_url"],
"https://api.cloudflare.com/client/v4/accounts/<redacted>/ai/v1"
);
assert_eq!(value["catalog_cache"]["present"], false);
assert_eq!(value["catalog_cache"]["age_seconds"], 0);
let (_, raw) = run_raw(&["doctor", "--format", "json"]);
assert!(!raw.contains(TOKEN), "doctor output must never contain the token: {raw}");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn doctor_missing_creds_exits_2() {
with_env(&[], || {
let (code, value) = run_json(&["doctor", "--format", "json"]);
assert_eq!(code, EXIT_CREDENTIALS);
assert_eq!(value["status"], "error");
assert_eq!(value["account_id"]["configured"], false);
assert_eq!(value["api_token"]["configured"], false);
assert_eq!(value["api_token"]["value_redacted"], true);
assert!(value["endpoint"]["base_url"].is_null(), "base_url omitted when account unset");
});
}
#[test]
fn doctor_token_only_missing_reports_account_configured() {
with_env(&[(ACCOUNT_ENV, Some(ACCOUNT))], || {
let (code, value) = run_json(&["doctor", "--format", "json"]);
assert_eq!(code, EXIT_CREDENTIALS);
assert_eq!(value["account_id"]["configured"], true);
assert_eq!(value["api_token"]["configured"], false);
});
}
#[test]
fn doctor_detects_unsafe_config_exit_7() {
let home = scratch_dir("doctor-unsafe");
let _ = std::fs::remove_dir_all(&home);
std::fs::create_dir_all(&home).expect("create scratch dir");
let config_path = home.join("config.json");
std::fs::write(
&config_path,
format!(r#"{{"account_id":"{ACCOUNT}","api_token":"cfut_leaked_value"}}"#),
)
.expect("write unsafe config");
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("AUTH_CLOUDFLARE_CONFIG", Some(config_path.to_str().unwrap())),
],
|| {
let (code, value) = run_json(&["doctor", "--format", "json"]);
assert_eq!(code, EXIT_UNSAFE_CONFIG);
assert_eq!(value["status"], "error");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn doctor_shows_cache_present_and_age() {
let home = scratch_dir("doctor-cache");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &[DEFAULT_MODEL]);
let (code, value) = run_json(&["doctor", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["catalog_cache"]["present"], true);
assert!(value["catalog_cache"]["age_seconds"].as_u64().unwrap_or(u64::MAX) < 10);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn redact_account_id_patterns() {
assert_eq!(redact_account_id("624acc1234567890abcdef123456789f84"), "624acc…9f84");
assert_eq!(redact_account_id("abcdef"), "ab…ef");
assert_eq!(redact_account_id("ab"), "a…");
assert_eq!(redact_account_id(""), "…");
}
#[test]
fn catalog_get_cache_miss_falls_back_to_bundled_models() {
with_env(&[], || {
let (code, value) = run_json(&["catalog", "get", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["schema_version"], 1);
assert_eq!(value["source"], "fallback");
assert_eq!(value["cache_status"], "none");
assert_eq!(value["default_model"], DEFAULT_MODEL);
let models = value["models"].as_array().expect("models array");
assert!(!models.is_empty());
assert_eq!(value["model_count"].as_u64(), Some(models.len() as u64));
assert_eq!(value["experimental_included"], true);
assert_eq!(value["deprecated_included"], false);
assert_eq!(models[0]["id"], DEFAULT_MODEL);
assert_eq!(models[0]["status"], "recommended");
assert_eq!(models[0]["primary_agent_eligible"], true);
assert!(
models[0]["pricing_per_million"]["input"].is_null(),
"fallback pricing is honestly null"
);
});
}
#[test]
fn catalog_get_fresh_cache_served() {
let home = scratch_dir("get-fresh");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &[DEFAULT_MODEL]);
let (code, value) = run_json(&["catalog", "get", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["source"], "cache");
assert_eq!(value["cache_status"], "fresh");
assert_eq!(value["models"][0]["id"], DEFAULT_MODEL);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn catalog_get_stale_cache_exits_4() {
let home = scratch_dir("get-stale");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache("2016-01-01T00:00:00Z", &[DEFAULT_MODEL]);
let (code, value) = run_json(&["catalog", "get", "--format", "json"]);
assert_eq!(code, EXIT_STALE_CACHE);
assert_eq!(value["source"], "cache");
assert_eq!(value["cache_status"], "stale");
assert_eq!(value["models"][0]["id"], DEFAULT_MODEL, "stale cache is still served");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn catalog_list_is_ordered_ids() {
with_env(&[], || {
let (code, value) = run_json(&["catalog", "list", "--format", "json"]);
assert_eq!(code, EXIT_OK);
let models = value["models"].as_array().expect("models array");
assert!(!models.is_empty());
assert!(models.iter().all(|m| m.is_string()));
assert_eq!(models[0], DEFAULT_MODEL);
assert_eq!(value["model_count"].as_u64(), Some(models.len() as u64));
assert_eq!(value["experimental_included"], true);
assert_eq!(value["deprecated_included"], false);
});
}
#[test]
fn refresh_missing_creds_exits_2() {
with_env(&[], || {
let (code, value) = run_json_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_ok);
assert_eq!(code, EXIT_CREDENTIALS);
assert_eq!(value["status"], "error");
assert_eq!(value["exit_code"], EXIT_CREDENTIALS);
assert!(
value["error"]
.as_str()
.expect("error string")
.contains("AUTH_CLOUDFLARE_ACCOUNT_ID"),
"missing-cred error names the exact env var"
);
});
}
#[test]
fn refresh_live_success_exits_0_and_writes_cache() {
let home = scratch_dir("refresh-live-ok");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_ok);
assert_eq!(code, EXIT_OK);
assert_eq!(value["schema_version"], 1);
assert_eq!(value["source"], "cloudflare-workers-ai");
assert_eq!(value["cache_status"], "fresh");
assert_eq!(value["default_model"], DEFAULT_MODEL);
let models = value["models"].as_array().expect("models array");
assert_eq!(models.len(), 2);
assert_eq!(value["model_count"], 2);
assert_eq!(value["experimental_included"], true);
assert_eq!(value["deprecated_included"], false);
assert_eq!(models[0]["id"], DEFAULT_MODEL);
assert_eq!(models[0]["status"], "recommended");
assert_eq!(models[0]["pricing_per_million"]["input"], 0.44);
assert_eq!(models[0]["capabilities"]["tools"], "confirmed");
let dir = cache_dir_for_account(&AuthProvider::new(ACCOUNT));
let (meta, payload) = read_catalog_cache(&dir).expect("read cache").expect("cache written");
assert_eq!(meta.source, "cloudflare-workers-ai");
assert_eq!(meta.model_count, 2);
assert_eq!(meta.account_fingerprint, AuthProvider::new(ACCOUNT).cache_slug());
assert_eq!(payload["data"][0]["id"], DEFAULT_MODEL);
let (_, raw) = run_raw_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_ok);
assert!(!raw.contains(TOKEN), "refresh output must never contain the token: {raw}");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn refresh_remote_failure_no_cache_exits_3() {
let home = scratch_dir("refresh-fail-nocache");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_err);
assert_eq!(code, EXIT_REMOTE_API);
assert_eq!(value["status"], "error");
assert!(
value["error"]
.as_str()
.expect("error string")
.contains("simulated network failure")
);
assert_eq!(value["exit_code"], EXIT_REMOTE_API);
let (_, raw) = run_raw_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_err);
assert!(!raw.contains(TOKEN), "error output must never contain the token: {raw}");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn refresh_remote_failure_stale_cache_exits_4() {
let home = scratch_dir("refresh-fail-stale");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache("2016-01-01T00:00:00Z", &[DEFAULT_MODEL]);
let (code, value) = run_json_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_err);
assert_eq!(code, EXIT_STALE_CACHE);
assert_eq!(value["source"], "cloudflare-workers-ai");
assert_eq!(value["cache_status"], "stale");
assert_eq!(value["models"][0]["id"], DEFAULT_MODEL, "stale cache is still served");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn refresh_remote_failure_fresh_cache_still_exits_0() {
let home = scratch_dir("refresh-fail-fresh");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &[DEFAULT_MODEL]);
let (code, value) = run_json_with_fetch(&["catalog", "refresh", "--format", "json"], fetch_err);
assert_eq!(code, EXIT_OK, "a still-fresh cache is served, not discarded");
assert_eq!(value["cache_status"], "fresh");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn catalog_get_live_fetch_on_cache_miss() {
let home = scratch_dir("get-live");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json_with_fetch(&["catalog", "get", "--format", "json"], fetch_ok);
assert_eq!(code, EXIT_OK);
assert_eq!(value["source"], "live");
assert_eq!(value["cache_status"], "fresh");
assert_eq!(value["models"][0]["id"], DEFAULT_MODEL);
let dir = cache_dir_for_account(&AuthProvider::new(ACCOUNT));
assert!(
read_catalog_cache(&dir).expect("read cache").is_some(),
"live get seeds the cache"
);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn catalog_get_live_failure_falls_back_to_bundled() {
let home = scratch_dir("get-live-fail");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json_with_fetch(&["catalog", "get", "--format", "json"], fetch_err);
assert_eq!(code, EXIT_OK, "live failure without cache falls back with exit 0");
assert_eq!(value["source"], "fallback");
assert_eq!(value["cache_status"], "none");
assert_eq!(value["models"][0]["id"], DEFAULT_MODEL);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn catalog_list_live_fetch_on_cache_miss() {
let home = scratch_dir("list-live");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json_with_fetch(&["catalog", "list", "--format", "json"], fetch_ok);
assert_eq!(code, EXIT_OK);
assert_eq!(value["source"], "live");
let models = value["models"].as_array().expect("models array");
assert_eq!(models[0], DEFAULT_MODEL);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn diff_without_cache_exits_1() {
with_env(&[], || {
let (code, value) = run_json(&["catalog", "diff", "--format", "json"]);
assert_eq!(code, EXIT_OPERATIONAL);
assert_eq!(value["status"], "error");
});
}
#[test]
fn diff_against_seeded_cache_reports_added_and_common() {
let home = scratch_dir("diff");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &[DEFAULT_MODEL]);
let (code, value) = run_json(&["catalog", "diff", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["common_count"], 1);
assert_eq!(value["added"].as_array().expect("added").len(), FALLBACK_MODELS.len() - 1);
assert!(value["removed"].as_array().expect("removed").is_empty());
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn export_yaml_and_markdown_derive_from_catalog() {
let home = scratch_dir("export");
let _ = std::fs::remove_dir_all(&home);
std::fs::create_dir_all(&home).expect("create scratch dir");
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
(EXPORT_DIR_ENV, Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &[DEFAULT_MODEL]);
let (code, message) = run_raw(&["catalog", "export", "yaml"]);
assert_eq!(code, EXIT_OK, "yaml export: {message}");
let yaml = std::fs::read_to_string(home.join("catalog.generated.yaml")).expect("yaml file");
assert!(yaml.contains("GENERATED FILE"));
assert!(yaml.contains(DEFAULT_MODEL));
let (code, message) = run_raw(&["catalog", "export", "markdown"]);
assert_eq!(code, EXIT_OK, "markdown export: {message}");
let md = std::fs::read_to_string(home.join("catalog.generated.md")).expect("md file");
assert!(md.contains("Cloudflare Workers AI catalog"));
assert!(md.contains("| Model | Status |"));
assert!(md.contains("DeepSeek V4 Flash 0731"));
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn model_inspect_found_offline() {
with_env(&[], || {
let (code, value) = run_json(&["model", "inspect", DEFAULT_MODEL, "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["id"], DEFAULT_MODEL);
assert_eq!(value["source"], "fallback");
assert_eq!(value["status"], "recommended");
});
}
#[test]
fn model_inspect_missing_exits_5() {
with_env(&[], || {
let (code, value) = run_json(&["model", "inspect", "@cf/unknown/not-in-catalog", "--format", "json"]);
assert_eq!(code, EXIT_NO_ELIGIBLE_MODEL);
assert_eq!(value["status"], "error");
});
}
#[test]
fn model_inspect_serves_experimental_from_cache() {
let home = scratch_dir("inspect-cache");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &["@cf/zai-org/glm-5.3-flash"]);
let (code, value) = run_json(&["model", "inspect", "@cf/zai-org/glm-5.3-flash", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["source"], "cache");
assert_eq!(value["status"], "experimental", "policy marks GLM-5.3 Flash experimental");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn model_inspect_degraded_record_surfaces_warning_with_alternative() {
let home = scratch_dir("inspect-degraded");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(&chrono::Utc::now().to_rfc3339(), &["@cf/zai-org/glm-5.3-flash"]);
seed_health_store(&[("@cf/zai-org/glm-5.3-flash", health::VerificationStatus::Degraded)]);
let (code, value) = run_json(&["model", "inspect", "@cf/zai-org/glm-5.3-flash", "--format", "json"]);
assert_eq!(code, EXIT_OK);
let warning = value["warning"].as_str().expect("degraded model surfaces a warning field");
assert!(
warning.contains(DEFAULT_MODEL),
"warning must name the stable alternative: {warning}"
);
assert!(
warning.contains("@cf/zai-org/glm-5.3-flash"),
"warning must name the degraded model: {warning}"
);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn model_inspect_passing_or_absent_record_has_no_warning() {
let home = scratch_dir("inspect-no-warning");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_cache(
&chrono::Utc::now().to_rfc3339(),
&[DEFAULT_MODEL, "@cf/moonshotai/kimi-k2.7-code"],
);
seed_health_store(&[(DEFAULT_MODEL, health::VerificationStatus::Passing)]);
let (code, value) = run_json(&["model", "inspect", DEFAULT_MODEL, "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert!(
value.get("warning").is_none(),
"passing record must not surface a warning: {value}"
);
let (code, value) =
run_json(&["model", "inspect", "@cf/moonshotai/kimi-k2.7-code", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert!(
value.get("warning").is_none(),
"absent record must not surface a warning: {value}"
);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn parses_model_verify_and_health_commands() {
let cases: &[(&[&str], Command)] = &[
(
&["model", "verify", DEFAULT_MODEL],
Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::Smoke,
recommended: false,
},
),
(
&["model", "verify", DEFAULT_MODEL, "--suite", "smoke"],
Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::Smoke,
recommended: false,
},
),
(
&["model", "verify", DEFAULT_MODEL, "--suite=smoke", "--format", "json"],
Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::Smoke,
recommended: false,
},
),
(
&["model", "verify", DEFAULT_MODEL, "--suite", "tool-loop"],
Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::ToolLoop,
recommended: false,
},
),
(
&["model", "verify", "--recommended", "--suite", "tool-loop"],
Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::ToolLoop,
recommended: true,
},
),
(
&["model", "verify", "--recommended"],
Command::ModelVerify {
model_id: DEFAULT_MODEL.to_string(),
suite: SuiteKind::Smoke,
recommended: true,
},
),
(&["model", "health", "--format", "json"], Command::ModelHealth),
];
for (args, expected) in cases {
let owned: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
let (command, format) = parse_args(&owned).unwrap_or_else(|e| panic!("{args:?} should parse: {e}"));
assert_eq!(&command, expected, "args {args:?}");
let _ = format;
}
}
#[test]
fn unknown_suite_and_misplaced_suite_are_parse_errors() {
let owned = vec![
"model".to_string(),
"verify".to_string(),
DEFAULT_MODEL.to_string(),
"--suite".to_string(),
"bogus".to_string(),
];
assert!(parse_args(&owned).is_err(), "unknown suite must fail parsing");
let owned = vec!["version".to_string(), "--suite".to_string(), "smoke".to_string()];
assert!(parse_args(&owned).is_err(), "--suite outside model verify must fail parsing");
let owned = vec!["model".to_string(), "verify".to_string(), "--suite".to_string()];
assert!(parse_args(&owned).is_err(), "--suite without a value must fail parsing");
let owned = vec!["version".to_string(), "--recommended".to_string()];
assert!(
parse_args(&owned).is_err(),
"--recommended outside model verify must fail parsing"
);
let owned = vec!["model".to_string(), "verify".to_string()];
assert!(
parse_args(&owned).is_err(),
"model verify without id or --recommended must fail parsing"
);
}
#[test]
fn model_verify_gate_closed_exits_1_without_network() {
with_env(&[], || {
let (code, value) = run_json(&["model", "verify", DEFAULT_MODEL, "--format", "json"]);
assert_eq!(code, EXIT_OPERATIONAL);
assert_eq!(value["status"], "error");
assert_eq!(value["gate"], "closed");
assert_eq!(
value["error"],
"live tests disabled (set AUTH_CLOUDFLARE_LIVE_TESTS=1 to allow paid inference)"
);
assert_eq!(value["exit_code"], EXIT_OPERATIONAL);
assert_eq!(value["model_id"], DEFAULT_MODEL);
assert_eq!(value["suite"], "smoke");
});
}
#[test]
fn model_verify_gate_precedes_credentials() {
let home = scratch_dir("verify-gate-first");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json(&["model", "verify", DEFAULT_MODEL, "--format", "json"]);
assert_eq!(code, EXIT_OPERATIONAL);
assert_eq!(value["gate"], "closed");
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn model_health_missing_creds_exits_2() {
with_env(&[], || {
let (code, value) = run_json(&["model", "health", "--format", "json"]);
assert_eq!(code, EXIT_CREDENTIALS);
assert_eq!(value["status"], "error");
assert_eq!(value["exit_code"], EXIT_CREDENTIALS);
});
}
#[test]
fn model_health_no_store_returns_empty_records() {
let home = scratch_dir("health-empty");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
let (code, value) = run_json(&["model", "health", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["status"], "ok");
assert_eq!(value["records"], serde_json::json!({}));
assert!(value["updated_at"].is_string());
assert_eq!(value["exit_code"], EXIT_OK);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn policy_get_matches_core_policy() {
with_env(&[], || {
let (code, value) = run_json(&["policy", "get", "--format", "json"]);
assert_eq!(code, EXIT_OK);
assert_eq!(value["version"], "1");
let models = value["models"].as_array().expect("models array");
assert_eq!(models[0]["model_id"], DEFAULT_MODEL);
assert_eq!(models[0]["status"], "recommended");
assert_eq!(models[0]["default"], true);
let guard = models
.iter()
.find(|m| m["model_id"] == "@cf/meta/llama-guard-3-8b")
.expect("guard entry");
assert_eq!(guard["status"], "hidden");
assert_eq!(guard["primary_agent_eligible"], false);
});
}
#[test]
fn policy_get_warnings_array_populated_for_degraded() {
let home = scratch_dir("policy-warnings");
let _ = std::fs::remove_dir_all(&home);
with_env(
&[
(ACCOUNT_ENV, Some(ACCOUNT)),
(TOKEN_ENV, Some(TOKEN)),
("HERMES_HOME", Some(home.to_str().unwrap())),
],
|| {
seed_health_store(&[
("@cf/zai-org/glm-5.3-flash", health::VerificationStatus::Failing),
(DEFAULT_MODEL, health::VerificationStatus::Passing),
]);
let (code, value) = run_json(&["policy", "get", "--format", "json"]);
assert_eq!(code, EXIT_OK);
let warnings = value["warnings"].as_array().expect("warnings array present for degraded model");
assert_eq!(warnings.len(), 1, "only the Failing model is warned: {value}");
assert_eq!(warnings[0]["model_id"], "@cf/zai-org/glm-5.3-flash");
let message = warnings[0]["message"].as_str().expect("message string");
assert!(
message.contains(DEFAULT_MODEL),
"message must name the stable alternative: {message}"
);
},
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn display_name_from_id_heuristics() {
assert_eq!(
display_name_from_id("@cf/deepseek-ai/deepseek-v4-flash-0731"),
"DeepSeek V4 Flash 0731"
);
assert_eq!(display_name_from_id("@cf/zai-org/glm-5.3-flash"), "GLM 5.3 Flash");
assert_eq!(display_name_from_id("plain-id"), "Plain Id");
}
#[test]
fn unsafe_config_detection_ignores_missing_or_safe_files() {
let home = scratch_dir("unsafe-detect");
let _ = std::fs::remove_dir_all(&home);
std::fs::create_dir_all(&home).expect("create scratch dir");
assert!(!detect_unsafe_config(&home.join("missing.json")), "missing file is safe");
let safe = home.join("safe.json");
std::fs::write(&safe, r#"{"account_id":"0123456789abcdef0123456789abcdef"}"#).expect("write safe config");
assert!(!detect_unsafe_config(&safe), "no api_token value is safe");
let env_named = home.join("env-named.json");
std::fs::write(&env_named, r#"{"api_token_env":"MY_CF_TOKEN_VAR"}"#).expect("write env-named config");
assert!(!detect_unsafe_config(&env_named), "env-var NAME is allowed (feedback 02)");
let leaked = home.join("leaked.json");
std::fs::write(&leaked, r#"{"api_token":"cfut_leaked"}"#).expect("write leaked config");
assert!(detect_unsafe_config(&leaked), "a literal token value is unsafe");
let _ = std::fs::remove_dir_all(&home);
}
}