1use std::collections::{BTreeMap, BTreeSet};
9
10use serde::Serialize;
11use serde_json::{json, Value};
12
13use crate::llm;
14use crate::llm_config::{
15 self, AliasDef, AliasToolCallingDef, ModelAvailability, ModelDef, ModelPricing, ProviderDef,
16};
17
18pub const PROVIDER_CATALOG_SCHEMA_VERSION: u32 = 2;
19pub const PROVIDER_CATALOG_SCHEMA_ID: &str =
20 "https://harnlang.com/schemas/provider-catalog.v2.json";
21pub const PROVIDER_CATALOG_GENERATOR: &str = "harn providers export";
22
23#[derive(Debug, Clone, Serialize)]
24pub struct ProviderCatalogArtifact {
25 pub schema_version: u32,
26 pub schema: String,
27 pub generated_by: String,
28 pub providers: Vec<CatalogProvider>,
29 pub models: Vec<CatalogModel>,
30 pub aliases: Vec<CatalogAlias>,
31 pub variants: Vec<CatalogVariant>,
32 pub qc_defaults: BTreeMap<String, String>,
33}
34
35#[derive(Debug, Clone, Serialize)]
36pub struct CatalogProvider {
37 pub id: String,
38 pub display_name: String,
39 #[serde(skip_serializing_if = "Option::is_none")]
40 pub icon: Option<String>,
41 pub classification: ProviderClassification,
42 pub endpoint: ProviderEndpoint,
43 pub auth: ProviderAuth,
44 pub protocols: Vec<String>,
45 pub features: Vec<String>,
46 pub caveats: Vec<String>,
47 #[serde(skip_serializing_if = "Option::is_none")]
48 pub rpm: Option<u32>,
49 #[serde(skip_serializing_if = "Option::is_none")]
50 pub latency_p50_ms: Option<u64>,
51}
52
53#[derive(Debug, Clone, Serialize)]
54#[serde(rename_all = "snake_case")]
55pub enum ProviderClassification {
56 Hosted,
57 Local,
58}
59
60#[derive(Debug, Clone, Serialize)]
61pub struct ProviderEndpoint {
62 pub base_url: String,
63 #[serde(skip_serializing_if = "Option::is_none")]
64 pub base_url_env: Option<String>,
65 pub chat_endpoint: String,
66 #[serde(skip_serializing_if = "Option::is_none")]
67 pub completion_endpoint: Option<String>,
68}
69
70#[derive(Debug, Clone, Serialize)]
71pub struct ProviderAuth {
72 pub style: String,
73 #[serde(skip_serializing_if = "Option::is_none")]
74 pub header: Option<String>,
75 pub env: Vec<String>,
76 pub required: bool,
77}
78
79#[derive(Debug, Clone, Serialize)]
80pub struct CatalogAlias {
81 pub name: String,
82 pub model_id: String,
83 pub provider: String,
84 #[serde(skip_serializing_if = "Option::is_none")]
85 pub tool_format: Option<String>,
86 #[serde(skip_serializing_if = "Option::is_none")]
87 pub tool_calling: Option<AliasToolCallingDef>,
88}
89
90#[derive(Debug, Clone, Serialize)]
91pub struct CatalogModel {
92 pub id: String,
93 pub name: String,
94 pub provider: String,
95 pub aliases: Vec<String>,
96 pub context_window: u64,
97 #[serde(skip_serializing_if = "Option::is_none")]
98 pub runtime_context_window: Option<u64>,
99 #[serde(skip_serializing_if = "Option::is_none")]
100 pub stream_timeout: Option<f64>,
101 pub modalities: ModelModalities,
102 pub tool_support: ModelToolSupport,
103 pub structured_output: String,
104 pub format_preferences: ModelFormatPreferences,
105 pub reasoning: ModelReasoning,
106 pub prompt_cache: bool,
107 #[serde(skip_serializing_if = "Option::is_none")]
108 pub pricing: Option<ModelPricing>,
109 pub deprecation: ModelDeprecation,
110 pub availability: ModelAvailabilityStatus,
111 pub quality_tags: Vec<String>,
112 pub capability_tags: Vec<String>,
113}
114
115#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
116#[serde(rename_all = "snake_case")]
117pub enum ModelAvailabilityStatus {
118 Serverless,
119 Dedicated,
120 Unknown,
121}
122
123impl From<ModelAvailability> for ModelAvailabilityStatus {
124 fn from(value: ModelAvailability) -> Self {
125 match value {
126 ModelAvailability::Serverless => Self::Serverless,
127 ModelAvailability::Dedicated => Self::Dedicated,
128 ModelAvailability::Unknown => Self::Unknown,
129 }
130 }
131}
132
133#[derive(Debug, Clone, Serialize)]
134pub struct ModelModalities {
135 pub input: Vec<String>,
136 pub output: Vec<String>,
137}
138
139#[derive(Debug, Clone, Serialize)]
140pub struct ModelToolSupport {
141 pub native: bool,
142 pub text: bool,
143 #[serde(skip_serializing_if = "Option::is_none")]
144 pub preferred_format: Option<String>,
145 #[serde(skip_serializing_if = "Option::is_none")]
146 pub parity: Option<String>,
147 #[serde(skip_serializing_if = "Option::is_none")]
148 pub parity_notes: Option<String>,
149 #[serde(skip_serializing_if = "Option::is_none")]
150 pub empirical_parity: Option<ModelToolEmpiricalParity>,
151 pub tool_search: Vec<String>,
152 #[serde(skip_serializing_if = "Option::is_none")]
153 pub max_tools: Option<u32>,
154}
155
156#[derive(Debug, Clone, Serialize)]
157pub struct ModelToolEmpiricalParity {
158 pub verdict: String,
159 pub preferred_format: String,
160 pub confidence: String,
161 pub sample_size: u32,
162 pub last_evaluated: String,
163 pub native_pass_rate: f64,
164 pub text_pass_rate: f64,
165 pub verifier_divergence_rate: f64,
166}
167
168#[derive(Debug, Clone, Serialize)]
169pub struct ModelFormatPreferences {
170 pub prefers_xml_scaffolding: bool,
171 pub prefers_markdown_scaffolding: bool,
172 pub structured_output_mode: String,
173 pub supports_assistant_prefill: bool,
174 pub prefers_role_developer: bool,
175 pub prefers_xml_tools: bool,
176 pub thinking_block_style: String,
177}
178
179#[derive(Debug, Clone, Serialize)]
180pub struct ModelReasoning {
181 pub modes: Vec<String>,
182 pub effort_supported: bool,
183 pub none_supported: bool,
184 pub interleaved_supported: bool,
185 pub preserve_thinking: bool,
186}
187
188#[derive(Debug, Clone, Serialize)]
189pub struct ModelDeprecation {
190 pub status: DeprecationStatus,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub note: Option<String>,
193}
194
195#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
196#[serde(rename_all = "snake_case")]
197pub enum DeprecationStatus {
198 Active,
199 Deprecated,
200}
201
202#[derive(Debug, Clone, Serialize)]
203pub struct CatalogVariant {
204 pub id: String,
205 pub label: String,
206 pub description: String,
207 pub model_id: String,
208 pub provider: String,
209 pub source: String,
210}
211
212#[derive(Debug, Clone, Default, PartialEq, Eq)]
213pub struct ProviderCatalogValidation {
214 pub errors: Vec<String>,
215 pub warnings: Vec<String>,
216}
217
218impl ProviderCatalogValidation {
219 pub fn is_ok(&self) -> bool {
220 self.errors.is_empty()
221 }
222}
223
224pub fn artifact() -> ProviderCatalogArtifact {
225 let alias_entries = llm_config::alias_entries();
226 let aliases_by_model = aliases_by_model(&alias_entries);
227 let providers = llm_config::provider_names()
228 .into_iter()
229 .filter_map(|id| {
230 llm_config::provider_config(&id).map(|provider| catalog_provider(id, provider))
231 })
232 .collect();
233 let models = llm_config::model_catalog_entries()
234 .into_iter()
235 .map(|(id, model)| catalog_model(id, model, &aliases_by_model))
236 .collect::<Vec<_>>();
237 let aliases = alias_entries
238 .iter()
239 .map(|(name, alias)| catalog_alias(name, alias))
240 .collect::<Vec<_>>();
241 let variants = catalog_variants(&models, &aliases);
242
243 ProviderCatalogArtifact {
244 schema_version: PROVIDER_CATALOG_SCHEMA_VERSION,
245 schema: PROVIDER_CATALOG_SCHEMA_ID.to_string(),
246 generated_by: PROVIDER_CATALOG_GENERATOR.to_string(),
247 providers,
248 models,
249 aliases,
250 variants,
251 qc_defaults: llm_config::qc_defaults(),
252 }
253}
254
255pub fn artifact_json() -> Result<String, serde_json::Error> {
256 serde_json::to_string_pretty(&artifact()).map(|mut text| {
257 text.push('\n');
258 text
259 })
260}
261
262pub fn schema_json() -> Result<String, serde_json::Error> {
263 serde_json::to_string_pretty(&schema_value()).map(|mut text| {
264 text.push('\n');
265 text
266 })
267}
268
269pub fn typescript_binding() -> Result<String, serde_json::Error> {
270 let json = artifact_json()?;
271 Ok(format!(
272 "{}{}{}{}{}",
273 generated_header("//", "typescript"),
274 TYPESCRIPT_TYPES,
275 "\nexport const harnProviderCatalog: HarnProviderCatalog = ",
276 json.trim_end(),
277 ";\n",
278 ) + TYPESCRIPT_COMPAT_EXPORTS)
279}
280
281pub fn swift_binding() -> Result<String, serde_json::Error> {
282 let json = artifact_json()?;
283 Ok(format!(
284 "{}{}\npublic let harnProviderCatalogJSON = #\"\"\"\n{}\"\"\"#\n",
285 generated_header("//", "swift"),
286 SWIFT_TYPES,
287 json
288 ))
289}
290
291pub fn validate_artifact(artifact: &ProviderCatalogArtifact) -> ProviderCatalogValidation {
292 let mut result = ProviderCatalogValidation::default();
293 if artifact.schema_version != PROVIDER_CATALOG_SCHEMA_VERSION {
294 result.errors.push(format!(
295 "schema_version must be {}, got {}",
296 PROVIDER_CATALOG_SCHEMA_VERSION, artifact.schema_version
297 ));
298 }
299 if artifact.providers.is_empty() {
300 result.errors.push("catalog has no providers".to_string());
301 }
302 if artifact.models.is_empty() {
303 result.errors.push("catalog has no models".to_string());
304 }
305
306 let provider_ids: BTreeSet<_> = artifact.providers.iter().map(|p| p.id.as_str()).collect();
307 for provider in &artifact.providers {
308 if provider.id.trim().is_empty() {
309 result
310 .errors
311 .push("provider id cannot be empty".to_string());
312 }
313 if provider.display_name.trim().is_empty() {
314 result.errors.push(format!(
315 "provider {} display_name cannot be empty",
316 provider.id
317 ));
318 }
319 if provider.endpoint.chat_endpoint.trim().is_empty() {
320 result.errors.push(format!(
321 "provider {} chat_endpoint cannot be empty",
322 provider.id
323 ));
324 }
325 if provider.auth.required
326 && provider.auth.env.is_empty()
327 && provider.auth.style != "aws_sigv4"
328 {
329 result.errors.push(format!(
330 "provider {} requires auth but declares no auth env keys",
331 provider.id
332 ));
333 }
334 }
335
336 let mut alias_names = BTreeSet::new();
337 for alias in &artifact.aliases {
338 if alias.name.trim().is_empty() {
339 result.errors.push("alias name cannot be empty".to_string());
340 }
341 if !alias_names.insert(alias.name.as_str()) {
342 result
343 .errors
344 .push(format!("duplicate alias name {}", alias.name));
345 }
346 if !provider_ids.contains(alias.provider.as_str()) {
347 result.errors.push(format!(
348 "alias {} references unknown provider {}",
349 alias.name, alias.provider
350 ));
351 }
352 }
353
354 let mut model_ids = BTreeSet::new();
355 let mut model_pairs = BTreeSet::new();
356 for model in &artifact.models {
357 if !model_ids.insert(model.id.as_str()) {
358 result
359 .errors
360 .push(format!("duplicate model id {}", model.id));
361 }
362 model_pairs.insert((model.provider.as_str(), model.id.as_str()));
363 if model.name.trim().is_empty() {
364 result
365 .errors
366 .push(format!("model {} name cannot be empty", model.id));
367 }
368 if !provider_ids.contains(model.provider.as_str()) {
369 result.errors.push(format!(
370 "model {} references unknown provider {}",
371 model.id, model.provider
372 ));
373 }
374 if model.context_window == 0 {
375 result.errors.push(format!(
376 "model {} context_window must be positive",
377 model.id
378 ));
379 }
380 if let Some(pricing) = &model.pricing {
381 validate_pricing(model, pricing, &mut result);
382 }
383 if model.deprecation.status == DeprecationStatus::Deprecated
384 && model
385 .deprecation
386 .note
387 .as_deref()
388 .unwrap_or("")
389 .trim()
390 .is_empty()
391 {
392 result.errors.push(format!(
393 "deprecated model {} must include deprecation.note",
394 model.id
395 ));
396 }
397 }
398
399 let dedicated_pairs: BTreeSet<(&str, &str)> = artifact
400 .models
401 .iter()
402 .filter(|model| model.availability == ModelAvailabilityStatus::Dedicated)
403 .map(|model| (model.provider.as_str(), model.id.as_str()))
404 .collect();
405 for alias in &artifact.aliases {
406 if !model_pairs.contains(&(alias.provider.as_str(), alias.model_id.as_str())) {
407 result.errors.push(format!(
408 "alias {} targets {}/{} without a catalog row",
409 alias.name, alias.provider, alias.model_id
410 ));
411 }
412 if is_tier_alias(&alias.name)
413 && dedicated_pairs.contains(&(alias.provider.as_str(), alias.model_id.as_str()))
414 {
415 result.warnings.push(format!(
416 "tier alias {} targets dedicated-only model {}/{}; serverless callers will fail until the dedicated endpoint is provisioned",
417 alias.name, alias.provider, alias.model_id
418 ));
419 }
420 }
421
422 for variant in &artifact.variants {
423 if variant.id.trim().is_empty() {
424 result.errors.push("variant id cannot be empty".to_string());
425 }
426 if !provider_ids.contains(variant.provider.as_str()) {
427 result.errors.push(format!(
428 "variant {} references unknown provider {}",
429 variant.id, variant.provider
430 ));
431 }
432 if !model_pairs.contains(&(variant.provider.as_str(), variant.model_id.as_str())) {
433 result.errors.push(format!(
434 "variant {} targets {}/{} without a catalog row",
435 variant.id, variant.provider, variant.model_id
436 ));
437 }
438 }
439
440 result
441}
442
443pub fn validate_current() -> ProviderCatalogValidation {
444 validate_artifact(&artifact())
445}
446
447pub fn schema_value() -> Value {
448 json!({
449 "$schema": "https://json-schema.org/draft/2020-12/schema",
450 "$id": PROVIDER_CATALOG_SCHEMA_ID,
451 "title": "Harn provider catalog",
452 "type": "object",
453 "required": ["schema_version", "schema", "generated_by", "providers", "models", "aliases", "variants", "qc_defaults"],
454 "properties": {
455 "schema_version": {"const": PROVIDER_CATALOG_SCHEMA_VERSION},
456 "schema": {"const": PROVIDER_CATALOG_SCHEMA_ID},
457 "generated_by": {"type": "string"},
458 "providers": {"type": "array", "items": {"$ref": "#/$defs/provider"}},
459 "models": {"type": "array", "items": {"$ref": "#/$defs/model"}},
460 "aliases": {"type": "array", "items": {"$ref": "#/$defs/alias"}},
461 "variants": {"type": "array", "items": {"$ref": "#/$defs/variant"}},
462 "qc_defaults": {"type": "object", "additionalProperties": {"type": "string"}}
463 },
464 "additionalProperties": false,
465 "$defs": {
466 "provider": {
467 "type": "object",
468 "required": ["id", "display_name", "classification", "endpoint", "auth", "protocols", "features", "caveats"],
469 "properties": {
470 "id": {"type": "string", "minLength": 1},
471 "display_name": {"type": "string", "minLength": 1},
472 "icon": {"type": "string"},
473 "classification": {"enum": ["hosted", "local"]},
474 "endpoint": {"$ref": "#/$defs/endpoint"},
475 "auth": {"$ref": "#/$defs/auth"},
476 "protocols": {"type": "array", "items": {"type": "string"}},
477 "features": {"type": "array", "items": {"type": "string"}},
478 "caveats": {"type": "array", "items": {"type": "string"}},
479 "rpm": {"type": "integer", "minimum": 1},
480 "latency_p50_ms": {"type": "integer", "minimum": 0}
481 },
482 "additionalProperties": false
483 },
484 "endpoint": {
485 "type": "object",
486 "required": ["base_url", "chat_endpoint"],
487 "properties": {
488 "base_url": {"type": "string"},
489 "base_url_env": {"type": "string"},
490 "chat_endpoint": {"type": "string", "minLength": 1},
491 "completion_endpoint": {"type": "string"}
492 },
493 "additionalProperties": false
494 },
495 "auth": {
496 "type": "object",
497 "required": ["style", "env", "required"],
498 "properties": {
499 "style": {"type": "string"},
500 "header": {"type": "string"},
501 "env": {"type": "array", "items": {"type": "string"}},
502 "required": {"type": "boolean"}
503 },
504 "additionalProperties": false
505 },
506 "alias": {
507 "type": "object",
508 "required": ["name", "model_id", "provider"],
509 "properties": {
510 "name": {"type": "string", "minLength": 1},
511 "model_id": {"type": "string", "minLength": 1},
512 "provider": {"type": "string", "minLength": 1},
513 "tool_format": {"type": "string"},
514 "tool_calling": {
515 "type": "object",
516 "properties": {
517 "native": {"type": "string"},
518 "text": {"type": "string"},
519 "streaming_native": {"type": "string"},
520 "fallback_mode": {"type": "string"},
521 "failure_reason": {"type": "string"},
522 "last_probe_at": {"type": "string"}
523 },
524 "additionalProperties": false
525 }
526 },
527 "additionalProperties": false
528 },
529 "model": {
530 "type": "object",
531 "required": [
532 "id",
533 "name",
534 "provider",
535 "aliases",
536 "context_window",
537 "modalities",
538 "tool_support",
539 "structured_output",
540 "format_preferences",
541 "reasoning",
542 "prompt_cache",
543 "deprecation",
544 "availability",
545 "quality_tags",
546 "capability_tags"
547 ],
548 "properties": {
549 "id": {"type": "string", "minLength": 1},
550 "name": {"type": "string", "minLength": 1},
551 "provider": {"type": "string", "minLength": 1},
552 "aliases": {"type": "array", "items": {"type": "string"}},
553 "context_window": {"type": "integer", "minimum": 1},
554 "runtime_context_window": {"type": "integer", "minimum": 1},
555 "stream_timeout": {"type": "number", "exclusiveMinimum": 0},
556 "modalities": {"$ref": "#/$defs/modalities"},
557 "tool_support": {"$ref": "#/$defs/tool_support"},
558 "structured_output": {"type": "string"},
559 "format_preferences": {"$ref": "#/$defs/format_preferences"},
560 "reasoning": {"$ref": "#/$defs/reasoning"},
561 "prompt_cache": {"type": "boolean"},
562 "pricing": {"$ref": "#/$defs/pricing"},
563 "deprecation": {"$ref": "#/$defs/deprecation"},
564 "availability": {"enum": ["serverless", "dedicated", "unknown"]},
565 "quality_tags": {"type": "array", "items": {"type": "string"}},
566 "capability_tags": {"type": "array", "items": {"type": "string"}}
567 },
568 "additionalProperties": false
569 },
570 "modalities": {
571 "type": "object",
572 "required": ["input", "output"],
573 "properties": {
574 "input": {"type": "array", "items": {"type": "string"}, "minItems": 1},
575 "output": {"type": "array", "items": {"type": "string"}, "minItems": 1}
576 },
577 "additionalProperties": false
578 },
579 "tool_support": {
580 "type": "object",
581 "required": ["native", "text", "tool_search"],
582 "properties": {
583 "native": {"type": "boolean"},
584 "text": {"type": "boolean"},
585 "preferred_format": {"type": "string"},
586 "parity": {"type": "string"},
587 "parity_notes": {"type": "string"},
588 "empirical_parity": {"$ref": "#/$defs/tool_empirical_parity"},
589 "tool_search": {"type": "array", "items": {"type": "string"}},
590 "max_tools": {"type": "integer", "minimum": 1}
591 },
592 "additionalProperties": false
593 },
594 "tool_empirical_parity": {
595 "type": "object",
596 "required": [
597 "verdict",
598 "preferred_format",
599 "confidence",
600 "sample_size",
601 "last_evaluated",
602 "native_pass_rate",
603 "text_pass_rate",
604 "verifier_divergence_rate"
605 ],
606 "properties": {
607 "verdict": {"type": "string"},
608 "preferred_format": {"type": "string"},
609 "confidence": {"type": "string"},
610 "sample_size": {"type": "integer", "minimum": 1},
611 "last_evaluated": {"type": "string", "minLength": 1},
612 "native_pass_rate": {"type": "number", "minimum": 0, "maximum": 1},
613 "text_pass_rate": {"type": "number", "minimum": 0, "maximum": 1},
614 "verifier_divergence_rate": {"type": "number", "minimum": 0, "maximum": 1}
615 },
616 "additionalProperties": false
617 },
618 "format_preferences": {
619 "type": "object",
620 "required": [
621 "prefers_xml_scaffolding",
622 "prefers_markdown_scaffolding",
623 "structured_output_mode",
624 "supports_assistant_prefill",
625 "prefers_role_developer",
626 "prefers_xml_tools",
627 "thinking_block_style"
628 ],
629 "properties": {
630 "prefers_xml_scaffolding": {"type": "boolean"},
631 "prefers_markdown_scaffolding": {"type": "boolean"},
632 "structured_output_mode": {"enum": ["native_json", "delimited", "xml_tagged", "none"]},
633 "supports_assistant_prefill": {"type": "boolean"},
634 "prefers_role_developer": {"type": "boolean"},
635 "prefers_xml_tools": {"type": "boolean"},
636 "thinking_block_style": {"enum": ["none", "thinking_blocks", "reasoning_summary", "inline"]}
637 },
638 "additionalProperties": false
639 },
640 "reasoning": {
641 "type": "object",
642 "required": ["modes", "effort_supported", "none_supported", "interleaved_supported", "preserve_thinking"],
643 "properties": {
644 "modes": {"type": "array", "items": {"type": "string"}},
645 "effort_supported": {"type": "boolean"},
646 "none_supported": {"type": "boolean"},
647 "interleaved_supported": {"type": "boolean"},
648 "preserve_thinking": {"type": "boolean"}
649 },
650 "additionalProperties": false
651 },
652 "pricing": {
653 "type": "object",
654 "required": ["input_per_mtok", "output_per_mtok"],
655 "properties": {
656 "input_per_mtok": {"type": "number", "minimum": 0},
657 "output_per_mtok": {"type": "number", "minimum": 0},
658 "cache_read_per_mtok": {"type": ["number", "null"], "minimum": 0},
659 "cache_write_per_mtok": {"type": ["number", "null"], "minimum": 0}
660 },
661 "additionalProperties": false
662 },
663 "deprecation": {
664 "type": "object",
665 "required": ["status"],
666 "properties": {
667 "status": {"enum": ["active", "deprecated"]},
668 "note": {"type": "string"}
669 },
670 "additionalProperties": false
671 },
672 "variant": {
673 "type": "object",
674 "required": ["id", "label", "description", "model_id", "provider", "source"],
675 "properties": {
676 "id": {"type": "string", "minLength": 1},
677 "label": {"type": "string", "minLength": 1},
678 "description": {"type": "string"},
679 "model_id": {"type": "string", "minLength": 1},
680 "provider": {"type": "string", "minLength": 1},
681 "source": {"type": "string", "minLength": 1}
682 },
683 "additionalProperties": false
684 }
685 }
686 })
687}
688
689fn catalog_provider(id: String, provider: ProviderDef) -> CatalogProvider {
690 CatalogProvider {
691 display_name: provider
692 .display_name
693 .clone()
694 .unwrap_or_else(|| title_case(&id)),
695 icon: provider.icon.clone(),
696 classification: provider_classification(&provider),
697 endpoint: ProviderEndpoint {
698 base_url: provider.base_url.clone(),
699 base_url_env: provider.base_url_env.clone(),
700 chat_endpoint: provider.chat_endpoint.clone(),
701 completion_endpoint: provider.completion_endpoint.clone(),
702 },
703 auth: ProviderAuth {
704 style: provider.auth_style.clone(),
705 header: provider.auth_header.clone(),
706 env: llm_config::auth_env_names(&provider.auth_env),
707 required: provider.auth_style != "none",
708 },
709 protocols: provider_protocols(&id, &provider),
710 features: provider.features.clone(),
711 caveats: provider_caveats(&id, &provider),
712 rpm: provider.rpm,
713 latency_p50_ms: provider.latency_p50_ms,
714 id,
715 }
716}
717
718fn catalog_alias(name: &str, alias: &AliasDef) -> CatalogAlias {
719 CatalogAlias {
720 name: name.to_string(),
721 model_id: alias.id.clone(),
722 provider: alias.provider.clone(),
723 tool_format: alias.tool_format.clone(),
724 tool_calling: llm_config::alias_tool_calling_entry(name),
725 }
726}
727
728fn catalog_model(
729 id: String,
730 model: ModelDef,
731 aliases_by_model: &BTreeMap<(String, String), Vec<String>>,
732) -> CatalogModel {
733 let caps = llm::capabilities::lookup(&model.provider, &id);
734 let structured_output = caps
735 .structured_output
736 .clone()
737 .or_else(|| caps.json_schema.clone())
738 .unwrap_or_else(|| "none".to_string());
739 let aliases = aliases_by_model
740 .get(&(model.provider.clone(), id.clone()))
741 .cloned()
742 .unwrap_or_default();
743 let quality_tags = model_quality_tags(&model, &aliases);
744 CatalogModel {
745 aliases,
746 modalities: modalities_from_caps(&caps),
747 tool_support: ModelToolSupport {
748 native: caps.native_tools,
749 text: caps.text_tool_wire_format_supported,
750 preferred_format: caps.preferred_tool_format.clone(),
751 parity: caps.tool_mode_parity.clone(),
752 parity_notes: caps.tool_mode_parity_notes.clone(),
753 empirical_parity: None,
754 tool_search: caps.tool_search.clone(),
755 max_tools: caps.max_tools,
756 },
757 structured_output,
758 format_preferences: ModelFormatPreferences {
759 prefers_xml_scaffolding: caps.prefers_xml_scaffolding,
760 prefers_markdown_scaffolding: caps.prefers_markdown_scaffolding,
761 structured_output_mode: caps.structured_output_mode.clone(),
762 supports_assistant_prefill: caps.supports_assistant_prefill,
763 prefers_role_developer: caps.prefers_role_developer,
764 prefers_xml_tools: caps.prefers_xml_tools,
765 thinking_block_style: caps.thinking_block_style.clone(),
766 },
767 reasoning: ModelReasoning {
768 modes: caps.thinking_modes.clone(),
769 effort_supported: caps.reasoning_effort_supported,
770 none_supported: caps.reasoning_none_supported,
771 interleaved_supported: caps.interleaved_thinking_supported,
772 preserve_thinking: caps.preserve_thinking,
773 },
774 prompt_cache: caps.prompt_caching,
775 pricing: model.pricing.clone(),
776 deprecation: ModelDeprecation {
777 status: if model.deprecated {
778 DeprecationStatus::Deprecated
779 } else {
780 DeprecationStatus::Active
781 },
782 note: model.deprecation_note.clone(),
783 },
784 availability: ModelAvailabilityStatus::from(model.availability),
785 quality_tags,
786 capability_tags: model.capabilities.clone(),
787 id,
788 name: model.name,
789 provider: model.provider,
790 context_window: model.context_window,
791 runtime_context_window: model.runtime_context_window,
792 stream_timeout: model.stream_timeout,
793 }
794}
795
796fn model_quality_tags(model: &ModelDef, aliases: &[String]) -> Vec<String> {
797 let mut tags: BTreeSet<String> = model.quality_tags.iter().cloned().collect();
798 for alias in aliases {
799 match alias.as_str() {
800 "frontier" | "tier/frontier" => {
801 tags.insert("frontier".to_string());
802 }
803 "mid" | "tier/mid" => {
804 tags.insert("balanced".to_string());
805 }
806 "small" | "tier/small" => {
807 tags.insert("small".to_string());
808 }
809 _ => {}
810 }
811 }
812 if is_local_provider(&model.provider) {
813 tags.insert("local".to_string());
814 }
815 tags.into_iter().collect()
816}
817
818fn aliases_by_model(aliases: &[(String, AliasDef)]) -> BTreeMap<(String, String), Vec<String>> {
819 let mut by_model: BTreeMap<(String, String), Vec<String>> = BTreeMap::new();
820 for (name, alias) in aliases {
821 by_model
822 .entry((alias.provider.clone(), alias.id.clone()))
823 .or_default()
824 .push(name.clone());
825 }
826 for names in by_model.values_mut() {
827 names.sort();
828 }
829 by_model
830}
831
832fn modalities_from_caps(caps: &llm::capabilities::Capabilities) -> ModelModalities {
833 let mut input = vec!["text".to_string()];
834 if caps.vision || caps.vision_supported {
835 input.push("image".to_string());
836 }
837 if caps.audio {
838 input.push("audio".to_string());
839 }
840 if caps.pdf {
841 input.push("pdf".to_string());
842 }
843 ModelModalities {
844 input,
845 output: vec!["text".to_string()],
846 }
847}
848
849fn catalog_variants(models: &[CatalogModel], aliases: &[CatalogAlias]) -> Vec<CatalogVariant> {
850 let mut variants = Vec::new();
851 for (id, label, description, alias_name) in [
852 (
853 "fast",
854 "Fast",
855 "Lowest-latency general coding-agent route.",
856 "small",
857 ),
858 (
859 "balanced",
860 "Balanced",
861 "Default cost/quality tradeoff for routine coding-agent work.",
862 "mid",
863 ),
864 (
865 "high-reasoning",
866 "High reasoning",
867 "Frontier route for hard planning, repair, and review tasks.",
868 "frontier",
869 ),
870 ] {
871 if let Some(alias) = aliases.iter().find(|alias| alias.name == alias_name) {
872 variants.push(CatalogVariant {
873 id: id.to_string(),
874 label: label.to_string(),
875 description: description.to_string(),
876 model_id: alias.model_id.clone(),
877 provider: alias.provider.clone(),
878 source: format!("alias:{alias_name}"),
879 });
880 }
881 }
882 push_variant_from_model(
883 &mut variants,
884 "local",
885 "Local",
886 "Best local/offline model route in the checked-in catalog.",
887 models
888 .iter()
889 .filter(|model| is_local_provider(&model.provider))
890 .max_by_key(|model| model.context_window),
891 );
892 push_variant_from_model(
893 &mut variants,
894 "cheap",
895 "Cheap",
896 "Lowest known hosted input+output token price.",
897 models
898 .iter()
899 .filter(|model| !is_local_provider(&model.provider))
900 .min_by(|left, right| {
901 pricing_total(left)
902 .partial_cmp(&pricing_total(right))
903 .unwrap_or(std::cmp::Ordering::Equal)
904 }),
905 );
906 push_variant_from_model(
907 &mut variants,
908 "vision-capable",
909 "Vision capable",
910 "A model route that accepts image input.",
911 models
912 .iter()
913 .filter(|model| model.modalities.input.iter().any(|mode| mode == "image"))
914 .max_by_key(|model| model.context_window),
915 );
916 push_variant_from_model(
917 &mut variants,
918 "long-context",
919 "Long context",
920 "Largest context-window route in the checked-in catalog.",
921 models.iter().max_by_key(|model| model.context_window),
922 );
923 variants
924}
925
926fn push_variant_from_model(
927 variants: &mut Vec<CatalogVariant>,
928 id: &str,
929 label: &str,
930 description: &str,
931 model: Option<&CatalogModel>,
932) {
933 if let Some(model) = model {
934 variants.push(CatalogVariant {
935 id: id.to_string(),
936 label: label.to_string(),
937 description: description.to_string(),
938 model_id: model.id.clone(),
939 provider: model.provider.clone(),
940 source: "catalog".to_string(),
941 });
942 }
943}
944
945fn pricing_total(model: &CatalogModel) -> f64 {
946 model
947 .pricing
948 .as_ref()
949 .map(|pricing| pricing.input_per_mtok + pricing.output_per_mtok)
950 .unwrap_or(f64::MAX)
951}
952
953fn validate_pricing(
954 model: &CatalogModel,
955 pricing: &ModelPricing,
956 result: &mut ProviderCatalogValidation,
957) {
958 for (field, value) in [
959 ("input_per_mtok", Some(pricing.input_per_mtok)),
960 ("output_per_mtok", Some(pricing.output_per_mtok)),
961 ("cache_read_per_mtok", pricing.cache_read_per_mtok),
962 ("cache_write_per_mtok", pricing.cache_write_per_mtok),
963 ] {
964 if value.is_some_and(|value| value < 0.0) {
965 result.errors.push(format!(
966 "model {} pricing.{} must be non-negative",
967 model.id, field
968 ));
969 }
970 }
971}
972
973fn provider_classification(provider: &ProviderDef) -> ProviderClassification {
974 if provider.auth_style == "none"
975 || provider.base_url.contains("localhost")
976 || provider.base_url.contains("127.0.0.1")
977 {
978 ProviderClassification::Local
979 } else {
980 ProviderClassification::Hosted
981 }
982}
983
984fn provider_protocols(id: &str, provider: &ProviderDef) -> Vec<String> {
985 match id {
986 "anthropic" => vec!["anthropic_messages".to_string()],
987 "gemini" => vec!["gemini_generate_content".to_string()],
988 "vertex" => vec!["vertex_generate_content".to_string()],
989 "bedrock" => vec!["bedrock_converse".to_string()],
990 "azure_openai" => vec!["azure_openai_chat_completions".to_string()],
991 "ollama" if provider.chat_endpoint.starts_with("/api/") => {
992 vec!["ollama_native".to_string()]
993 }
994 _ => vec!["openai_chat_completions".to_string()],
995 }
996}
997
998fn provider_caveats(id: &str, provider: &ProviderDef) -> Vec<String> {
999 let mut caveats = Vec::new();
1000 if provider.auth_style == "aws_sigv4" {
1001 caveats.push("Credentials are resolved through the AWS SDK chain.".to_string());
1002 }
1003 if id == "azure_openai" {
1004 caveats.push("The Harn model field names the Azure deployment.".to_string());
1005 }
1006 if id == "ollama" && provider.chat_endpoint == "/api/chat" {
1007 caveats.push(
1008 "Native Ollama chat returns NDJSON and can apply model-family parsers.".to_string(),
1009 );
1010 }
1011 caveats
1012}
1013
1014fn is_local_provider(provider: &str) -> bool {
1015 matches!(
1016 provider,
1017 "ollama" | "local" | "llamacpp" | "mlx" | "vllm" | "tgi"
1018 )
1019}
1020
1021fn is_tier_alias(name: &str) -> bool {
1022 matches!(
1023 name,
1024 "frontier"
1025 | "mid"
1026 | "small"
1027 | "tier/frontier"
1028 | "tier/mid"
1029 | "tier/small"
1030 | "sonnet"
1031 | "opus"
1032 | "haiku"
1033 )
1034}
1035
1036fn title_case(id: &str) -> String {
1037 id.split('_')
1038 .map(|part| {
1039 let mut chars = part.chars();
1040 match chars.next() {
1041 Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
1042 None => String::new(),
1043 }
1044 })
1045 .collect::<Vec<_>>()
1046 .join(" ")
1047}
1048
1049fn generated_header(comment: &str, language: &str) -> String {
1050 format!(
1051 "{comment} GENERATED by `{}` - do not edit by hand.\n{comment} Source: Harn runtime provider catalog schema v{}.\n{comment} Language: {language}.\n\n",
1052 PROVIDER_CATALOG_GENERATOR, PROVIDER_CATALOG_SCHEMA_VERSION
1053 )
1054}
1055
1056const TYPESCRIPT_TYPES: &str = r#"export interface HarnProviderCatalog {
1057 schema_version: 2
1058 schema: string
1059 generated_by: string
1060 providers: HarnCatalogProvider[]
1061 models: HarnCatalogModel[]
1062 aliases: HarnCatalogAlias[]
1063 variants: HarnCatalogVariant[]
1064 qc_defaults: Record<string, string>
1065}
1066
1067export interface HarnCatalogProvider {
1068 id: string
1069 display_name: string
1070 icon?: string
1071 classification: "hosted" | "local"
1072 endpoint: HarnProviderEndpoint
1073 auth: HarnProviderAuth
1074 protocols: string[]
1075 features: string[]
1076 caveats: string[]
1077 rpm?: number
1078 latency_p50_ms?: number
1079}
1080
1081export interface HarnProviderEndpoint {
1082 base_url: string
1083 base_url_env?: string
1084 chat_endpoint: string
1085 completion_endpoint?: string
1086}
1087
1088export interface HarnProviderAuth {
1089 style: string
1090 header?: string
1091 env: string[]
1092 required: boolean
1093}
1094
1095export interface HarnCatalogAlias {
1096 name: string
1097 model_id: string
1098 provider: string
1099 tool_format?: string
1100 tool_calling?: HarnAliasToolCalling
1101}
1102
1103export interface HarnAliasToolCalling {
1104 native?: string
1105 text?: string
1106 streaming_native?: string
1107 fallback_mode?: string
1108 failure_reason?: string
1109 last_probe_at?: string
1110}
1111
1112export interface HarnCatalogModel {
1113 id: string
1114 name: string
1115 provider: string
1116 aliases: string[]
1117 context_window: number
1118 runtime_context_window?: number
1119 stream_timeout?: number
1120 modalities: { input: string[]; output: string[] }
1121 tool_support: {
1122 native: boolean
1123 text: boolean
1124 preferred_format?: string
1125 parity?: string
1126 parity_notes?: string
1127 empirical_parity?: HarnToolEmpiricalParity
1128 tool_search: string[]
1129 max_tools?: number
1130 }
1131 structured_output: string
1132 format_preferences: {
1133 prefers_xml_scaffolding: boolean
1134 prefers_markdown_scaffolding: boolean
1135 structured_output_mode: "native_json" | "delimited" | "xml_tagged" | "none"
1136 supports_assistant_prefill: boolean
1137 prefers_role_developer: boolean
1138 prefers_xml_tools: boolean
1139 thinking_block_style: "none" | "thinking_blocks" | "reasoning_summary" | "inline"
1140 }
1141 reasoning: {
1142 modes: string[]
1143 effort_supported: boolean
1144 none_supported: boolean
1145 interleaved_supported: boolean
1146 preserve_thinking: boolean
1147 }
1148 prompt_cache: boolean
1149 pricing?: HarnModelPricing
1150 deprecation: { status: "active" | "deprecated"; note?: string }
1151 availability: "serverless" | "dedicated" | "unknown"
1152 quality_tags: string[]
1153 capability_tags: string[]
1154}
1155
1156export interface HarnToolEmpiricalParity {
1157 verdict: string
1158 preferred_format: string
1159 confidence: string
1160 sample_size: number
1161 last_evaluated: string
1162 native_pass_rate: number
1163 text_pass_rate: number
1164 verifier_divergence_rate: number
1165}
1166
1167export interface HarnModelPricing {
1168 input_per_mtok: number
1169 output_per_mtok: number
1170 cache_read_per_mtok?: number | null
1171 cache_write_per_mtok?: number | null
1172}
1173
1174export interface HarnCatalogVariant {
1175 id: string
1176 label: string
1177 description: string
1178 model_id: string
1179 provider: string
1180 source: string
1181}
1182
1183export interface CatalogEntry {
1184 id: string
1185 name: string
1186 provider: string
1187 contextWindow: number
1188 runtimeContextWindow?: number
1189 capabilities: string[]
1190 pricing?: {
1191 inputPerMTok: number
1192 outputPerMTok: number
1193 cacheReadPerMTok?: number | null
1194 cacheWritePerMTok?: number | null
1195 }
1196 streamTimeout?: number
1197}
1198
1199export interface CatalogAlias {
1200 alias: string
1201 id: string
1202 provider: string
1203 toolFormat?: string
1204 toolCalling?: HarnAliasToolCalling
1205}
1206
1207"#;
1208
1209const TYPESCRIPT_COMPAT_EXPORTS: &str = r#"
1210export const MODEL_CATALOG: readonly CatalogEntry[] = harnProviderCatalog.models.map((model) => ({
1211 id: model.id,
1212 name: model.name,
1213 provider: model.provider,
1214 contextWindow: model.context_window,
1215 runtimeContextWindow: model.runtime_context_window,
1216 capabilities: model.capability_tags,
1217 pricing: model.pricing
1218 ? {
1219 inputPerMTok: model.pricing.input_per_mtok,
1220 outputPerMTok: model.pricing.output_per_mtok,
1221 cacheReadPerMTok: model.pricing.cache_read_per_mtok,
1222 cacheWritePerMTok: model.pricing.cache_write_per_mtok,
1223 }
1224 : undefined,
1225 streamTimeout: model.stream_timeout,
1226}))
1227
1228export const ALIASES: readonly CatalogAlias[] = harnProviderCatalog.aliases.map((alias) => ({
1229 alias: alias.name,
1230 id: alias.model_id,
1231 provider: alias.provider,
1232 toolFormat: alias.tool_format,
1233 toolCalling: alias.tool_calling,
1234}))
1235
1236export const QC_DEFAULTS: Readonly<Record<string, string>> = harnProviderCatalog.qc_defaults
1237
1238export function pricingFor(modelId: string): CatalogEntry["pricing"] | undefined {
1239 return entryFor(modelId)?.pricing
1240}
1241
1242export function entryFor(modelId: string): CatalogEntry | undefined {
1243 return MODEL_CATALOG.find((entry) => entry.id === modelId)
1244}
1245
1246export function aliasesByProvider(provider: string): readonly CatalogAlias[] {
1247 return ALIASES.filter((alias) => alias.provider === provider)
1248}
1249
1250export function qcDefaultModel(provider: string): string | undefined {
1251 return QC_DEFAULTS[provider]
1252}
1253"#;
1254
1255const SWIFT_TYPES: &str = r#"public struct HarnProviderCatalog: Codable, Sendable, Equatable {
1256 public let schemaVersion: Int
1257 public let schema: String
1258 public let generatedBy: String
1259 public let providers: [HarnCatalogProvider]
1260 public let models: [HarnCatalogModel]
1261 public let aliases: [HarnCatalogAlias]
1262 public let variants: [HarnCatalogVariant]
1263 public let qcDefaults: [String: String]
1264
1265 enum CodingKeys: String, CodingKey {
1266 case schemaVersion = "schema_version"
1267 case schema
1268 case generatedBy = "generated_by"
1269 case providers
1270 case models
1271 case aliases
1272 case variants
1273 case qcDefaults = "qc_defaults"
1274 }
1275}
1276
1277public struct HarnCatalogProvider: Codable, Sendable, Equatable {
1278 public let id: String
1279 public let displayName: String
1280 public let icon: String?
1281 public let classification: String
1282 public let endpoint: HarnProviderEndpoint
1283 public let auth: HarnProviderAuth
1284 public let protocols: [String]
1285 public let features: [String]
1286 public let caveats: [String]
1287 public let rpm: Int?
1288 public let latencyP50Ms: Int?
1289
1290 enum CodingKeys: String, CodingKey {
1291 case id
1292 case displayName = "display_name"
1293 case icon
1294 case classification
1295 case endpoint
1296 case auth
1297 case protocols
1298 case features
1299 case caveats
1300 case rpm
1301 case latencyP50Ms = "latency_p50_ms"
1302 }
1303}
1304
1305public struct HarnProviderEndpoint: Codable, Sendable, Equatable {
1306 public let baseURL: String
1307 public let baseURLEnv: String?
1308 public let chatEndpoint: String
1309 public let completionEndpoint: String?
1310
1311 enum CodingKeys: String, CodingKey {
1312 case baseURL = "base_url"
1313 case baseURLEnv = "base_url_env"
1314 case chatEndpoint = "chat_endpoint"
1315 case completionEndpoint = "completion_endpoint"
1316 }
1317}
1318
1319public struct HarnProviderAuth: Codable, Sendable, Equatable {
1320 public let style: String
1321 public let header: String?
1322 public let env: [String]
1323 public let required: Bool
1324}
1325
1326public struct HarnCatalogAlias: Codable, Sendable, Equatable {
1327 public let name: String
1328 public let modelID: String
1329 public let provider: String
1330 public let toolFormat: String?
1331 public let toolCalling: HarnAliasToolCalling?
1332
1333 enum CodingKeys: String, CodingKey {
1334 case name
1335 case modelID = "model_id"
1336 case provider
1337 case toolFormat = "tool_format"
1338 case toolCalling = "tool_calling"
1339 }
1340}
1341
1342public struct HarnAliasToolCalling: Codable, Sendable, Equatable {
1343 public let native: String?
1344 public let text: String?
1345 public let streamingNative: String?
1346 public let fallbackMode: String?
1347 public let failureReason: String?
1348 public let lastProbeAt: String?
1349
1350 enum CodingKeys: String, CodingKey {
1351 case native
1352 case text
1353 case streamingNative = "streaming_native"
1354 case fallbackMode = "fallback_mode"
1355 case failureReason = "failure_reason"
1356 case lastProbeAt = "last_probe_at"
1357 }
1358}
1359
1360public struct HarnCatalogModel: Codable, Sendable, Equatable {
1361 public let id: String
1362 public let name: String
1363 public let provider: String
1364 public let aliases: [String]
1365 public let contextWindow: Int
1366 public let runtimeContextWindow: Int?
1367 public let streamTimeout: Double?
1368 public let modalities: HarnModelModalities
1369 public let toolSupport: HarnModelToolSupport
1370 public let structuredOutput: String
1371 public let formatPreferences: HarnModelFormatPreferences
1372 public let reasoning: HarnModelReasoning
1373 public let promptCache: Bool
1374 public let pricing: HarnModelPricing?
1375 public let deprecation: HarnModelDeprecation
1376 public let availability: String
1377 public let qualityTags: [String]
1378 public let capabilityTags: [String]
1379
1380 enum CodingKeys: String, CodingKey {
1381 case id
1382 case name
1383 case provider
1384 case aliases
1385 case contextWindow = "context_window"
1386 case runtimeContextWindow = "runtime_context_window"
1387 case streamTimeout = "stream_timeout"
1388 case modalities
1389 case toolSupport = "tool_support"
1390 case structuredOutput = "structured_output"
1391 case formatPreferences = "format_preferences"
1392 case reasoning
1393 case promptCache = "prompt_cache"
1394 case pricing
1395 case deprecation
1396 case availability
1397 case qualityTags = "quality_tags"
1398 case capabilityTags = "capability_tags"
1399 }
1400}
1401
1402public struct HarnModelModalities: Codable, Sendable, Equatable {
1403 public let input: [String]
1404 public let output: [String]
1405}
1406
1407public struct HarnModelToolSupport: Codable, Sendable, Equatable {
1408 public let native: Bool
1409 public let text: Bool
1410 public let preferredFormat: String?
1411 public let parity: String?
1412 public let parityNotes: String?
1413 public let empiricalParity: HarnToolEmpiricalParity?
1414 public let toolSearch: [String]
1415 public let maxTools: Int?
1416
1417 enum CodingKeys: String, CodingKey {
1418 case native
1419 case text
1420 case preferredFormat = "preferred_format"
1421 case parity
1422 case parityNotes = "parity_notes"
1423 case empiricalParity = "empirical_parity"
1424 case toolSearch = "tool_search"
1425 case maxTools = "max_tools"
1426 }
1427}
1428
1429public struct HarnToolEmpiricalParity: Codable, Sendable, Equatable {
1430 public let verdict: String
1431 public let preferredFormat: String
1432 public let confidence: String
1433 public let sampleSize: Int
1434 public let lastEvaluated: String
1435 public let nativePassRate: Double
1436 public let textPassRate: Double
1437 public let verifierDivergenceRate: Double
1438
1439 enum CodingKeys: String, CodingKey {
1440 case verdict
1441 case preferredFormat = "preferred_format"
1442 case confidence
1443 case sampleSize = "sample_size"
1444 case lastEvaluated = "last_evaluated"
1445 case nativePassRate = "native_pass_rate"
1446 case textPassRate = "text_pass_rate"
1447 case verifierDivergenceRate = "verifier_divergence_rate"
1448 }
1449}
1450
1451public struct HarnModelFormatPreferences: Codable, Sendable, Equatable {
1452 public let prefersXMLScaffolding: Bool
1453 public let prefersMarkdownScaffolding: Bool
1454 public let structuredOutputMode: String
1455 public let supportsAssistantPrefill: Bool
1456 public let prefersRoleDeveloper: Bool
1457 public let prefersXMLTools: Bool
1458 public let thinkingBlockStyle: String
1459
1460 enum CodingKeys: String, CodingKey {
1461 case prefersXMLScaffolding = "prefers_xml_scaffolding"
1462 case prefersMarkdownScaffolding = "prefers_markdown_scaffolding"
1463 case structuredOutputMode = "structured_output_mode"
1464 case supportsAssistantPrefill = "supports_assistant_prefill"
1465 case prefersRoleDeveloper = "prefers_role_developer"
1466 case prefersXMLTools = "prefers_xml_tools"
1467 case thinkingBlockStyle = "thinking_block_style"
1468 }
1469}
1470
1471public struct HarnModelReasoning: Codable, Sendable, Equatable {
1472 public let modes: [String]
1473 public let effortSupported: Bool
1474 public let noneSupported: Bool
1475 public let interleavedSupported: Bool
1476 public let preserveThinking: Bool
1477
1478 enum CodingKeys: String, CodingKey {
1479 case modes
1480 case effortSupported = "effort_supported"
1481 case noneSupported = "none_supported"
1482 case interleavedSupported = "interleaved_supported"
1483 case preserveThinking = "preserve_thinking"
1484 }
1485}
1486
1487public struct HarnModelPricing: Codable, Sendable, Equatable {
1488 public let inputPerMTok: Double
1489 public let outputPerMTok: Double
1490 public let cacheReadPerMTok: Double?
1491 public let cacheWritePerMTok: Double?
1492
1493 enum CodingKeys: String, CodingKey {
1494 case inputPerMTok = "input_per_mtok"
1495 case outputPerMTok = "output_per_mtok"
1496 case cacheReadPerMTok = "cache_read_per_mtok"
1497 case cacheWritePerMTok = "cache_write_per_mtok"
1498 }
1499}
1500
1501public struct HarnModelDeprecation: Codable, Sendable, Equatable {
1502 public let status: String
1503 public let note: String?
1504}
1505
1506public struct HarnCatalogVariant: Codable, Sendable, Equatable {
1507 public let id: String
1508 public let label: String
1509 public let description: String
1510 public let modelID: String
1511 public let provider: String
1512 public let source: String
1513
1514 enum CodingKeys: String, CodingKey {
1515 case id
1516 case label
1517 case description
1518 case modelID = "model_id"
1519 case provider
1520 case source
1521 }
1522}
1523"#;
1524
1525#[cfg(test)]
1526mod tests {
1527 use super::*;
1528
1529 struct OverrideGuard;
1530
1531 impl Drop for OverrideGuard {
1532 fn drop(&mut self) {
1533 llm_config::clear_user_overrides();
1534 }
1535 }
1536
1537 fn install_overlay(toml_src: &str) -> OverrideGuard {
1538 let overlay = llm_config::parse_config_toml(toml_src).expect("overlay parses");
1539 llm_config::set_user_overrides(Some(overlay));
1540 OverrideGuard
1541 }
1542
1543 #[test]
1544 fn generated_catalog_validates() {
1545 llm_config::clear_user_overrides();
1546 let report = validate_current();
1547 assert!(
1548 report.errors.is_empty(),
1549 "catalog validation errors: {:?}",
1550 report.errors
1551 );
1552 }
1553
1554 #[test]
1555 fn generated_catalog_derives_quality_tags_from_routes() {
1556 let catalog = artifact();
1557 let frontier = catalog
1558 .models
1559 .iter()
1560 .find(|model| model.aliases.iter().any(|alias| alias == "frontier"))
1561 .expect("frontier alias target is exported");
1562 assert!(frontier.quality_tags.iter().any(|tag| tag == "frontier"));
1563
1564 let local = catalog
1565 .models
1566 .iter()
1567 .find(|model| model.aliases.iter().any(|alias| alias == "local-gemma4"))
1568 .expect("local alias target is exported");
1569 assert!(local.quality_tags.iter().any(|tag| tag == "local"));
1570 }
1571
1572 #[test]
1573 fn validation_rejects_missing_required_metadata() {
1574 let mut catalog = artifact();
1575 catalog.providers[0].display_name.clear();
1576 let report = validate_artifact(&catalog);
1577 assert!(
1578 report
1579 .errors
1580 .iter()
1581 .any(|message| message.contains("display_name cannot be empty")),
1582 "expected provider metadata validation error, got {:?}",
1583 report.errors
1584 );
1585 }
1586
1587 #[test]
1588 fn validation_rejects_duplicate_and_dangling_aliases() {
1589 let mut duplicated = artifact();
1590 duplicated.aliases.push(duplicated.aliases[0].clone());
1591 let duplicate_report = validate_artifact(&duplicated);
1592 assert!(
1593 duplicate_report
1594 .errors
1595 .iter()
1596 .any(|message| message.contains("duplicate alias name")),
1597 "expected duplicate alias validation error, got {:?}",
1598 duplicate_report.errors
1599 );
1600
1601 let mut dangling = artifact();
1602 dangling.aliases[0].model_id = "missing-model".to_string();
1603 let dangling_report = validate_artifact(&dangling);
1604 assert!(
1605 dangling_report
1606 .errors
1607 .iter()
1608 .any(|message| message.contains("without a catalog row")),
1609 "expected dangling alias validation error, got {:?}",
1610 dangling_report.errors
1611 );
1612 }
1613
1614 #[test]
1615 fn overlay_merge_surfaces_private_model() {
1616 let _guard = install_overlay(
1617 r#"
1618[providers.private]
1619display_name = "Private"
1620base_url = "http://127.0.0.1:9000"
1621auth_style = "none"
1622chat_endpoint = "/v1/chat/completions"
1623
1624[aliases]
1625private-fast = { id = "private/fast", provider = "private" }
1626
1627[models."private/fast"]
1628name = "Private Fast"
1629provider = "private"
1630context_window = 8192
1631quality_tags = ["experiment"]
1632"#,
1633 );
1634 let catalog = artifact();
1635 assert!(catalog.providers.iter().any(|p| p.id == "private"));
1636 let model = catalog
1637 .models
1638 .iter()
1639 .find(|model| model.id == "private/fast")
1640 .expect("private model is exported");
1641 assert_eq!(model.aliases, vec!["private-fast"]);
1642 assert_eq!(model.quality_tags, vec!["experiment"]);
1643 }
1644
1645 #[test]
1646 fn cataloged_models_default_to_serverless_availability() {
1647 llm_config::clear_user_overrides();
1648 let catalog = artifact();
1649 let qwen_dedicated = catalog
1650 .models
1651 .iter()
1652 .find(|model| model.id == "Qwen/Qwen3-Coder-Next-FP8")
1653 .expect("Together dedicated route is exported");
1654 assert_eq!(
1655 qwen_dedicated.availability,
1656 ModelAvailabilityStatus::Dedicated
1657 );
1658
1659 let bundled_serverless = catalog
1660 .models
1661 .iter()
1662 .find(|model| model.id == "qwen/qwen3-coder")
1663 .expect("OpenRouter Qwen3 Coder is exported");
1664 assert_eq!(
1665 bundled_serverless.availability,
1666 ModelAvailabilityStatus::Serverless
1667 );
1668 }
1669
1670 #[test]
1671 fn tier_alias_targeting_dedicated_model_emits_warning() {
1672 let _guard = install_overlay(
1673 r#"
1674[providers.together_test]
1675display_name = "Together (test)"
1676base_url = "https://api.together.xyz/v1"
1677auth_style = "bearer"
1678auth_env = "TOGETHER_AI_API_KEY"
1679chat_endpoint = "/chat/completions"
1680
1681[aliases.frontier]
1682id = "Qwen/Test-Dedicated-Only"
1683provider = "together_test"
1684
1685[models."Qwen/Test-Dedicated-Only"]
1686name = "Qwen Dedicated Only"
1687provider = "together_test"
1688context_window = 8192
1689availability = "dedicated"
1690"#,
1691 );
1692 let report = validate_current();
1693 assert!(
1694 report.warnings.iter().any(|message| {
1695 message.contains("tier alias frontier") && message.contains("dedicated-only model")
1696 }),
1697 "expected dedicated-alias warning, got {:?}",
1698 report.warnings
1699 );
1700 }
1701
1702 #[test]
1703 fn overlay_parses_availability_strings() {
1704 let _guard = install_overlay(
1705 r#"
1706[providers.experiment_co]
1707display_name = "Experiment Co"
1708base_url = "https://example.test/v1"
1709auth_style = "bearer"
1710auth_env = "EXPERIMENT_API_KEY"
1711chat_endpoint = "/chat/completions"
1712
1713[models."exp/discovered"]
1714name = "Discovered Route"
1715provider = "experiment_co"
1716context_window = 4096
1717availability = "unknown"
1718"#,
1719 );
1720 let catalog = artifact();
1721 let model = catalog
1722 .models
1723 .iter()
1724 .find(|model| model.id == "exp/discovered")
1725 .expect("overlay model is exported");
1726 assert_eq!(model.availability, ModelAvailabilityStatus::Unknown);
1727 }
1728
1729 #[test]
1730 fn deprecated_models_require_notes() {
1731 let _guard = install_overlay(
1732 r#"
1733[models."old-model"]
1734name = "Old Model"
1735provider = "openai"
1736context_window = 4096
1737deprecated = true
1738"#,
1739 );
1740 let report = validate_current();
1741 assert!(
1742 report
1743 .errors
1744 .iter()
1745 .any(|message| message.contains("deprecated model old-model")),
1746 "expected deprecation validation error, got {:?}",
1747 report.errors
1748 );
1749 }
1750
1751 #[test]
1752 fn generated_schema_accepts_generated_artifact_shape() {
1753 let schema = schema_value();
1754 assert_eq!(schema["$id"], PROVIDER_CATALOG_SCHEMA_ID);
1755 assert_eq!(
1756 schema["$defs"]["tool_support"]["properties"]["empirical_parity"]["$ref"],
1757 "#/$defs/tool_empirical_parity"
1758 );
1759 let artifact_value = serde_json::to_value(artifact()).expect("artifact serializes");
1760 assert_eq!(
1761 artifact_value["schema_version"],
1762 PROVIDER_CATALOG_SCHEMA_VERSION
1763 );
1764 assert!(artifact_value["providers"]
1765 .as_array()
1766 .is_some_and(|v| !v.is_empty()));
1767 assert!(artifact_value["models"]
1768 .as_array()
1769 .is_some_and(|v| !v.is_empty()));
1770 }
1771
1772 #[test]
1773 fn downstream_bindings_include_empirical_tool_parity_shape() {
1774 let typescript = typescript_binding().expect("typescript binding renders");
1775 assert!(typescript.contains("empirical_parity?: HarnToolEmpiricalParity"));
1776 assert!(typescript.contains("export interface HarnToolEmpiricalParity"));
1777
1778 let swift = swift_binding().expect("swift binding renders");
1779 assert!(swift.contains("public let empiricalParity: HarnToolEmpiricalParity?"));
1780 assert!(swift.contains("public struct HarnToolEmpiricalParity"));
1781 }
1782}