1pub mod action_ledger;
26pub mod adaptive_router;
27pub mod aws_sigv4;
28pub mod backend;
29pub mod backend_cache;
30pub mod billing;
31pub mod calibration;
32pub mod catalog;
33pub mod catalog_identity;
34pub mod concierge;
35pub mod concierge_answer;
36pub mod discovery;
37pub mod doctor;
38pub mod download;
39pub mod handle;
40pub mod hardware;
41pub mod hf_cache;
42pub mod hf_schema;
43pub mod http_body;
44pub mod intent;
45pub mod key_pool;
46pub mod lane_defaults;
47pub mod maintenance;
48pub mod managed_venv;
49pub mod media_tokens;
50pub mod model_management;
51pub mod models;
52pub mod nudge;
53pub mod offload;
54pub mod openrouter;
55pub mod outcome;
56pub mod parslee_credential;
57pub mod portfolio;
58pub mod protocol;
59pub mod recommend;
60pub mod registry;
61pub mod remote;
62pub mod resource_policy;
63pub mod retire;
64pub mod router;
65pub mod routing_ext;
66pub mod runner;
67pub mod schema;
68pub mod scoreboard;
69pub mod search;
70pub mod service;
71pub mod stream;
72pub mod tasks;
73pub(crate) mod tls_client;
77pub mod uninstall;
78pub mod update_prefs;
79pub mod upgrade;
80pub mod usage_profile;
81pub mod vllm_mlx;
82pub mod vllm_pool;
83pub mod vllm_runtime;
84
85use std::path::{Path, PathBuf};
86use std::sync::Arc;
87use std::time::Instant;
88use std::time::{SystemTime, UNIX_EPOCH};
89
90use reqwest::multipart::{Form, Part};
91use serde::Serialize;
92use thiserror::Error;
93use tokio::io::AsyncReadExt;
94use tokio::process::Command;
95use tokio::sync::Mutex;
96use tokio::sync::RwLock;
97use tracing::{debug, instrument};
98
99pub use action_ledger::{ConciergeActionEntry, ConciergeActionKind};
101pub use adaptive_router::{
102 AdaptiveRouter, AdaptiveRoutingDecision, RoutingConfig, RoutingStrategy,
103};
104pub use catalog_identity::{CatalogModelRow, CatalogSnapshot};
105pub use concierge::{
106 decide_concierge, evaluate_concierge, ConciergeDecision, ConciergeMode, ConciergeStatus,
107 ConciergeSuggestion, DismissReason, DismissalRecord, ModelHealth,
108 DEFAULT_CONCIERGE_THROTTLE_SECS, DEFAULT_WATCHED_USE_CASES,
109};
110pub use download::{DownloadEvent, DownloadProgress, ProgressSink};
111pub use handle::InferenceHandle;
112pub use intent::{IntentHint, Privacy, QualityTier, TaskHint, TierWeights, UseCase, UseCaseRole};
113pub use key_pool::{KeyPool, KeyStats};
114pub use lane_defaults::{LaneDefault, LaneDefaults};
115pub use nudge::{NudgeDecision, NudgeState, UpgradeNudge};
116pub use outcome::{
117 prune_ledger, read_ledger, CodeOutcome, InferenceOutcome, InferenceTask, InferredOutcome,
118 ModelProfile, OutcomeLedgerEntry, OutcomeTracker,
119};
120pub use recommend::{
121 model_fit, platform_compatible, recommend, recommend_with_policy, FitStatus, ModelFit,
122 ModelFitStatus, Recommendation, RecommendationSet,
123};
124pub use resource_policy::{
125 estimate_model_memory, estimate_model_memory_for_checkpoint,
126 estimate_model_memory_with_measured_weights, evaluate_resources, AcceleratorResourceBudget,
127 CheckpointGeometry, EffectiveResourceBudget, FileResourcePolicyRepository,
128 LocalAdmissionCoordinator, LocalLoadPreflight, LocalLoadVerdict, ModelMemoryEstimate,
129 ModelResourceEvidence, ResourceEvaluation, ResourcePolicy, ResourcePolicyError,
130 ResourcePolicyLoadEvidence, ResourcePolicyLoadSource, ResourcePolicyRepository,
131 ResourceProfile, RECOMMENDATION_CONTEXT_TOKENS,
132};
133pub use update_prefs::{UpdateChannel, UpdatePolicy, UpdatePreferences};
134pub use upgrade::{HuggingFaceProbe, UpgradeFinding, UpgradeSource, UpstreamProbe};
135pub use usage_profile::{use_case_for_task, LaneUsage, UsageProfile};
136
137fn now_unix() -> u64 {
139 std::time::SystemTime::now()
140 .duration_since(std::time::UNIX_EPOCH)
141 .map(|d| d.as_secs())
142 .unwrap_or(0)
143}
144pub use offload::{
145 clear_remote_deadline, current_controlled_termination_token, current_inference_control_id,
146 current_local_offload, current_remote_deadline, ensure_not_controlled_terminated,
147 is_offload_worker, scope_inference_control_id, set_local_offload, set_remote_deadline,
148 ControlledTerminationToken, InferenceTerminationAck, LocalGenerationOffload,
149 LocalOffloadResult, LocalOffloadStream, LocalWorkerAdmission, LocalWorkerResidency,
150 RemoteDeadline,
151};
152pub use registry::{
153 is_builtin_subscription_billed, model_unavailable_reason, ModelFilter, ModelInfo,
154 ModelRuntimeRequirement, ModelUpgrade, UnifiedRegistry,
155};
156pub use remote::RemoteBackend;
157pub use routing_ext::{
158 CircuitBreaker, CircuitBreakerRegistry, CircuitState, ImplicitSignal, ImplicitSignalType,
159 RoutingMode, SpendControl, SpendLimitExceeded, SpendLimits, SpendStatus,
160};
161pub use runner::{
162 current_inference_runner, set_inference_runner, EventEmitter, InferenceRunner, RunnerError,
163 RunnerResult,
164};
165pub use schema::{
166 ApiProtocol, ApproxCost, BenchmarkScore, CostModel, ModelCapability, ModelSchema, ModelSource,
167 PerformanceEnvelope, ProprietaryAuth, ProprietaryProtocol, ProprietaryWire, QuantScheme,
168 Quantization, TrustTier, OPENAI_CODEX_SIGN_IN_HINT,
169};
170
171pub use adaptive_router::{RouterExclusion, RouterExclusionReason, TaskComplexity};
173#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
174pub use backend::CandleBackend;
175#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
176pub use backend::EmbeddingBackend;
177#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
178pub use backend::SwiftLmBackend;
179pub use hardware::HardwareInfo;
180pub use models::{ModelRegistry, ModelRole};
181pub use router::{ModelRouter, RoutingDecision};
182pub use stream::{StreamAccumulator, StreamEvent, StreamSubstitution};
183pub use tasks::{
184 parse_boxes, BoundingBox, ClassifyRequest, ClassifyResult, ContentBlock, EmbedRequest,
185 GenerateImageRequest, GenerateImageResult, GenerateParams, GenerateRequest,
186 GenerateVideoRequest, GenerateVideoResult, GroundRequest, GroundResult, Message,
187 OptionProbabilities, Provenance, RerankRequest, RerankResult, RerankedDocument, ResponseFormat,
188 RoutingWorkload, SynthesizeRequest, SynthesizeResult, ThinkingMode, ToolCall,
189 TranscribeRequest, TranscribeResult, VideoMode,
190};
191#[derive(Debug, Clone, PartialEq, Eq)]
199pub struct InferenceRetryProgress {
200 pub model: String,
202 pub attempt: u32,
204 pub reason: &'static str,
206 pub backoff_ms: u64,
208}
209
210#[derive(Error, Debug)]
211pub enum InferenceError {
212 #[error("model not found: {0}")]
213 ModelNotFound(String),
214
215 #[error("no eligible model remains after required exclusions: {excluded_models}")]
219 NoEligibleModel { excluded_models: String },
220
221 #[error("model download failed: {0}")]
222 DownloadFailed(String),
223
224 #[error("inference failed: {0}")]
225 InferenceFailed(String),
226
227 #[error("classify: the model's reply {reply:?} {reason}")]
231 ClassifyNoAnswer { reply: String, reason: String },
232
233 #[error("classify: invalid labels: {0}")]
237 InvalidClassifyLabels(String),
238
239 #[error("catalog precondition mismatch: {detail}")]
244 CatalogPreconditionMismatch { detail: String },
245
246 #[error("controlled inference termination confirmed")]
251 ControlledTermination,
252
253 #[error(transparent)]
254 ModelManagement(#[from] model_management::ModelManagementError),
255
256 #[error("{recovery}")]
259 LocalResourceBlocked {
260 preflight: resource_policy::LocalLoadPreflight,
261 recovery: String,
262 },
263
264 #[error("transient remote failure after retries (status={status:?}): {message}")]
273 Transient {
274 status: Option<u16>,
275 message: String,
276 },
277
278 #[error("deadline exceeded: the caller's {applied_ms} ms infer deadline elapsed after {elapsed_ms} ms; last attempt: {last_error}")]
286 DeadlineExceeded {
287 applied_ms: u64,
288 elapsed_ms: u64,
289 last_error: String,
290 },
291
292 #[error("mode {mode} not implemented on backend {backend}: {reason}")]
297 UnsupportedMode {
298 mode: &'static str,
299 backend: &'static str,
300 reason: &'static str,
301 },
302
303 #[error("{provider} account rejected the request (HTTP {status}): {message}")]
318 ProviderAccount {
319 provider: String,
320 status: u16,
321 message: String,
322 },
323
324 #[error("inference failed: {message}")]
333 ProviderKeyMissing {
334 provider: String,
335 model: String,
336 env_vars: Vec<String>,
339 message: String,
342 },
343
344 #[error("no credential for proprietary provider '{provider}' (model {model}): {detail}")]
361 CredentialUnavailable {
362 provider: String,
363 model: String,
364 reason: CredentialFailure,
366 detail: String,
368 },
369
370 #[error("{provider} account has no workspace yet: {detail}")]
387 WorkspaceRequired {
388 provider: String,
390 detail: String,
392 },
393
394 #[error("{provider} refused this request on content grounds{}: {message}",
414 content_refusal_detail(.kind.as_deref(), .code.as_deref()))]
415 ContentRefused {
416 provider: String,
417 kind: Option<String>,
419 code: Option<String>,
420 message: String,
421 },
422
423 #[error("{provider} gateway has no upstream configured for '{namespace}' (HTTP {status}): {message}")]
441 GatewayUnconfigured {
442 provider: String,
443 namespace: String,
444 status: u16,
445 message: String,
446 },
447
448 #[error("tokenization error: {0}")]
449 TokenizationError(String),
450
451 #[error("device error: {0}")]
452 DeviceError(String),
453
454 #[error("io error: {0}")]
455 Io(#[from] std::io::Error),
456}
457
458impl From<resource_policy::LocalAdmissionError> for InferenceError {
459 fn from(error: resource_policy::LocalAdmissionError) -> Self {
460 let recovery = match error.preflight.verdict {
461 resource_policy::LocalLoadVerdict::DisabledByPolicy => {
462 "Local model loading is disabled by the 0 GB allocation. Increase Local Models RAM in Settings, or choose a remote model. Downloads remain available.".to_string()
463 }
464 resource_policy::LocalLoadVerdict::ExceedsConfiguredCeiling => format!(
465 "This model needs about {} MB for this request, beyond the configured {} MB local-model allocation. Increase the allocation or choose a smaller model.",
466 error.preflight.estimated_incremental_mb,
467 error.preflight.configured_ceiling_mb
468 ),
469 resource_policy::LocalLoadVerdict::InsufficientLiveMemory => format!(
470 "Not enough memory is free to start this model while preserving CAR's {} MB emergency reserve. Close memory-heavy apps or choose a smaller model.",
471 error.preflight.emergency_reserve_mb
472 ),
473 resource_policy::LocalLoadVerdict::LiveMemoryUnknown => {
474 "CAR could not measure live memory. The static allocation fits, but current safety is unknown.".to_string()
475 }
476 resource_policy::LocalLoadVerdict::ModelMaintenance => {
477 "This local model is being removed or maintained. Wait for that operation to finish, then retry.".to_string()
478 }
479 resource_policy::LocalLoadVerdict::PendingTeardown => {
480 "CAR is still confirming that the previous local model process exited. Wait for teardown to finish, then retry.".to_string()
481 }
482 resource_policy::LocalLoadVerdict::Allowed => error.to_string(),
483 };
484 Self::LocalResourceBlocked {
485 preflight: error.preflight,
486 recovery,
487 }
488 }
489}
490
491fn error_counts_against_circuit_breaker(e: &InferenceError) -> bool {
528 failure_attribution(e).is_none()
529}
530
531fn content_refusal_detail(kind: Option<&str>, code: Option<&str>) -> String {
537 match (kind, code) {
538 (Some(k), Some(c)) => format!(" (type={k}, code={c})"),
539 (Some(k), None) => format!(" (type={k})"),
540 (None, Some(c)) => format!(" (code={c})"),
541 (None, None) => String::new(),
542 }
543}
544
545fn record_dispatch_failure(tracker: &mut OutcomeTracker, trace_id: &str, error: &InferenceError) {
551 match failure_attribution(error) {
552 Some(reason) => tracker.record_not_model_fault(trace_id, &error.to_string(), reason),
553 None => tracker.record_failure(trace_id, &error.to_string()),
554 }
555}
556
557fn failure_attribution(error: &InferenceError) -> Option<outcome::UnattributedReason> {
563 use outcome::UnattributedReason as R;
564 match error {
565 InferenceError::UnsupportedMode { .. }
566 | InferenceError::CatalogPreconditionMismatch { .. }
567 | InferenceError::InvalidClassifyLabels { .. } => Some(R::Capability),
569 InferenceError::ProviderAccount { .. }
570 | InferenceError::ProviderKeyMissing { .. }
571 | InferenceError::CredentialUnavailable { .. }
572 | InferenceError::WorkspaceRequired { .. }
573 | InferenceError::GatewayUnconfigured { .. } => Some(R::Account),
574 InferenceError::ContentRefused { .. } => Some(R::ContentRefused),
575 InferenceError::LocalResourceBlocked { .. } => Some(R::ResourceBlocked),
576 InferenceError::ControlledTermination => Some(R::Cancelled),
577 InferenceError::ModelNotFound { .. }
578 | InferenceError::NoEligibleModel { .. }
579 | InferenceError::DownloadFailed { .. }
580 | InferenceError::InferenceFailed { .. }
581 | InferenceError::ClassifyNoAnswer { .. }
582 | InferenceError::ModelManagement { .. }
583 | InferenceError::Transient { .. }
584 | InferenceError::DeadlineExceeded { .. }
585 | InferenceError::TokenizationError { .. }
586 | InferenceError::DeviceError { .. }
587 | InferenceError::Io { .. } => None,
588 }
589}
590
591#[cfg(any(
597 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
598 all(target_os = "ios", target_arch = "aarch64")
599))]
600fn send_stream_failure_blocking(
601 tx: &tokio::sync::mpsc::Sender<stream::StreamEvent>,
602 error: &InferenceError,
603) {
604 if let Some(reason) = failure_attribution(error) {
605 let _ = tx.blocking_send(stream::StreamEvent::Attribution(reason));
606 }
607 let _ = tx.blocking_send(stream::StreamEvent::Error(error.to_string()));
608}
609
610#[cfg(any(
618 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
619 all(target_os = "ios", target_arch = "aarch64")
620))]
621fn send_foundation_models_usage(
622 tx: &tokio::sync::mpsc::Sender<stream::StreamEvent>,
623 instructions: Option<&str>,
624 prompt: &str,
625 completion: &str,
626) {
627 if let Some((prompt_tokens, completion_tokens)) =
628 crate::backend::foundation_models::count_tokens(instructions, prompt, completion)
629 {
630 let _ = tx.blocking_send(stream::StreamEvent::Usage {
631 input_tokens: prompt_tokens,
632 output_tokens: completion_tokens,
633 cache_read_input_tokens: 0,
635 cache_creation_input_tokens: 0,
636 });
637 }
638}
639
640fn error_ends_fallback_chain(e: &InferenceError) -> bool {
662 if matches!(e, InferenceError::ContentRefused { provider, .. } if provider == "apple") {
669 return false;
670 }
671 matches!(
672 e,
673 InferenceError::ContentRefused { .. }
674 | InferenceError::CatalogPreconditionMismatch { .. }
675 | InferenceError::ControlledTermination
676 | InferenceError::WorkspaceRequired { .. }
682 )
683}
684
685fn apply_exhaustion_recovery_hint(underlying: InferenceError) -> InferenceError {
696 if matches!(
697 underlying,
698 InferenceError::ContentRefused { .. }
699 | InferenceError::CatalogPreconditionMismatch { .. }
700 | InferenceError::ControlledTermination
701 | InferenceError::WorkspaceRequired { .. }
706 ) {
707 return underlying;
708 }
709 let underlying_str = underlying.to_string();
710 match no_backend_recovery_hint(&underlying_str)
711 .or_else(|| auth_expired_recovery_hint(&underlying_str))
712 {
713 Some(msg) => InferenceError::InferenceFailed(msg),
714 None => underlying,
715 }
716}
717
718const AUTH_LOGIN_MARKER: &str = "auth login";
719const AUTH_STORE_UNREADABLE_MARKER: &str = "credential store unreadable";
720const AUTH_ENV_MISSING_MARKER: &str = "credential environment variable missing";
721
722const AUTH_FAILURE_MESSAGE_MARKERS: &[&str] = &[
727 "no credential for proprietary",
728 AUTH_LOGIN_MARKER,
729 "session has expired",
730 "cannot read parslee credentials",
731 AUTH_STORE_UNREADABLE_MARKER,
732 AUTH_ENV_MISSING_MARKER,
733 "credential was rejected",
739 "repair its provider login",
740];
741
742#[derive(Debug, Clone, PartialEq, Eq)]
752struct RouteCredentialFailure {
753 summary: String,
754 source_error: String,
755}
756
757fn parslee_signed_out_route_failure() -> RouteCredentialFailure {
758 let summary = "Parslee login is absent — run `car auth login` before retrying".to_string();
759 RouteCredentialFailure {
760 source_error: summary.clone(),
761 summary,
762 }
763}
764
765fn route_credential_failure(
766 candidate: &str,
767 error: &InferenceError,
768 promote_missing_credential: bool,
769) -> Option<String> {
770 match error {
771 InferenceError::CredentialUnavailable {
772 provider,
773 reason,
774 detail,
775 ..
776 } => {
777 let provider_name = if provider.eq_ignore_ascii_case("parslee") {
778 "Parslee".to_string()
779 } else {
780 provider.clone()
781 };
782 let summary = match reason {
783 CredentialFailure::Expired { .. } => format!(
784 "{provider_name} login expired for `{candidate}` — run `car auth login`"
785 ),
786 CredentialFailure::SignedOut => format!(
787 "{provider_name} login is absent for `{candidate}` — run `car auth login`"
788 ),
789 CredentialFailure::StoreUnreadable => format!(
790 "{provider_name} {AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
791 ),
792 CredentialFailure::EnvVarMissing { .. } if !promote_missing_credential => {
793 return None;
794 }
795 CredentialFailure::EnvVarMissing { env_var } => format!(
796 "{provider_name} {AUTH_ENV_MISSING_MARKER}: `{env_var}` for explicitly requested `{candidate}` — {detail}"
797 ),
798 CredentialFailure::RaceRetryable => return None,
801 };
802 Some(summary)
803 }
804 InferenceError::ProviderAccount {
808 provider, status, ..
809 } if *status == 403 => {
810 if provider.eq_ignore_ascii_case("parslee") {
811 Some(format!(
812 "Parslee denied this account access to `{candidate}` (HTTP 403) — \
813 signing in again does not change access; ask an organization admin"
814 ))
815 } else {
816 Some(format!(
817 "{provider} denied access to `{candidate}` (HTTP 403) — the configured key \
818 works but lacks permission for it"
819 ))
820 }
821 }
822 InferenceError::ProviderAccount {
823 provider, status, ..
824 } if *status == 401 => {
825 if provider.eq_ignore_ascii_case("parslee") {
826 Some(format!(
827 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
828 ))
829 } else {
830 Some(format!(
831 "{provider} credential was rejected for `{candidate}` (HTTP {status})"
832 ))
833 }
834 }
835 _ => {
836 let rendered = error.to_string();
837 let lower = rendered.to_ascii_lowercase();
838 if is_auth_rejection_message(&rendered) {
839 if candidate
840 .split_once('/')
841 .is_some_and(|(provider, _)| provider.eq_ignore_ascii_case("parslee"))
842 {
843 Some(format!(
844 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
845 ))
846 } else {
847 Some(format!(
848 "credential expired or was rejected for `{candidate}` — repair its provider login"
849 ))
850 }
851 } else if lower.contains(AUTH_STORE_UNREADABLE_MARKER)
852 || (promote_missing_credential && lower.contains("keychain lookup failed"))
853 {
854 Some(format!(
855 "{AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
856 ))
857 } else {
858 None
859 }
860 }
861 }
862}
863
864fn record_route_credential_failure(
865 slot: &mut Option<RouteCredentialFailure>,
866 candidate: &str,
867 error: &InferenceError,
868 promote_missing_credential: bool,
869) {
870 if let Some(summary) = route_credential_failure(candidate, error, promote_missing_credential) {
871 *slot = Some(RouteCredentialFailure {
876 summary,
877 source_error: error.to_string(),
878 });
879 }
880}
881
882fn chain_includes_parslee_route<'a>(
887 mut resolve: impl FnMut(&str) -> Option<&'a ModelSchema>,
888 chain: &[String],
889) -> bool {
890 chain.iter().any(|candidate| {
891 resolve(candidate).is_some_and(|schema| schema.provider.eq_ignore_ascii_case("parslee"))
892 })
893}
894
895fn apply_route_failure_context(
915 underlying: InferenceError,
916 credential: Option<&RouteCredentialFailure>,
917) -> InferenceError {
918 if matches!(
919 underlying,
920 InferenceError::ContentRefused { .. }
921 | InferenceError::CatalogPreconditionMismatch { .. }
922 | InferenceError::ControlledTermination
923 | InferenceError::WorkspaceRequired { .. }
931 ) {
932 return underlying;
933 }
934 let Some(credential) = credential else {
935 return apply_exhaustion_recovery_hint(underlying);
936 };
937 let rendered = underlying.to_string();
938 let summary = credential.summary.as_str();
939 let setup_hint = no_backend_recovery_hint(&rendered);
940 let connective = if rendered == credential.source_error {
941 "provider detail"
942 } else {
943 "fallback then failed"
944 };
945 augment_error_message(underlying, |field| match &setup_hint {
946 Some(hint) => format!("{summary}; {hint}"),
948 None => format!("{summary}; {connective}: {field}"),
949 })
950}
951
952fn append_adopt_hint(underlying: InferenceError, adoptable: &[String]) -> InferenceError {
959 if adoptable.is_empty()
960 || matches!(
961 underlying,
962 InferenceError::ContentRefused { .. }
963 | InferenceError::CatalogPreconditionMismatch { .. }
964 | InferenceError::ControlledTermination
965 | InferenceError::WorkspaceRequired { .. }
966 )
967 {
968 return underlying;
969 }
970 let commands = adoptable
971 .iter()
972 .map(|id| format!("`car models adopt {id}`"))
973 .collect::<Vec<_>>()
974 .join(", ");
975 augment_error_message(underlying, |message| {
976 format!(
977 "{message}; no on-device fallback was used because the ready local model(s) \
978 are in the shared Hugging Face cache, not installed through CAR — run {commands} \
979 to let CAR fall back to one"
980 )
981 })
982}
983
984fn augment_error_message(
988 underlying: InferenceError,
989 augment: impl Fn(String) -> String,
990) -> InferenceError {
991 match underlying {
992 InferenceError::InferenceFailed(message) => {
993 InferenceError::InferenceFailed(augment(message))
994 }
995 InferenceError::ProviderKeyMissing {
996 provider,
997 model,
998 env_vars,
999 message,
1000 } => InferenceError::ProviderKeyMissing {
1001 provider,
1002 model,
1003 env_vars,
1004 message: augment(message),
1005 },
1006 InferenceError::Transient { status, message } => InferenceError::Transient {
1007 status,
1008 message: augment(message),
1009 },
1010 InferenceError::ProviderAccount {
1011 provider,
1012 status,
1013 message,
1014 } => InferenceError::ProviderAccount {
1015 provider,
1016 status,
1017 message: augment(message),
1018 },
1019 InferenceError::CredentialUnavailable {
1020 provider,
1021 model,
1022 reason,
1023 detail,
1024 } => InferenceError::CredentialUnavailable {
1025 provider,
1026 model,
1027 reason,
1028 detail: augment(detail),
1029 },
1030 InferenceError::GatewayUnconfigured {
1031 provider,
1032 namespace,
1033 status,
1034 message,
1035 } => InferenceError::GatewayUnconfigured {
1036 provider,
1037 namespace,
1038 status,
1039 message: augment(message),
1040 },
1041 InferenceError::LocalResourceBlocked {
1042 preflight,
1043 recovery,
1044 } => InferenceError::LocalResourceBlocked {
1045 preflight,
1046 recovery: augment(recovery),
1047 },
1048 other => InferenceError::InferenceFailed(augment(other.to_string())),
1051 }
1052}
1053
1054#[derive(Debug, Clone, PartialEq, Eq)]
1063pub enum CredentialFailure {
1064 Expired { expires_at: u64 },
1071 SignedOut,
1074 StoreUnreadable,
1078 EnvVarMissing { env_var: String },
1080 RaceRetryable,
1083}
1084
1085#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1087pub enum Device {
1088 Cpu,
1089 Metal,
1090 Cuda(usize), }
1092
1093impl Device {
1094 pub fn auto() -> Self {
1102 #[cfg(all(target_os = "macos", feature = "metal"))]
1103 {
1104 return Device::Metal;
1105 }
1106 #[cfg(all(
1107 any(target_os = "linux", target_os = "windows"),
1108 target_arch = "x86_64",
1109 not(car_skip_cuda)
1110 ))]
1111 {
1112 return Device::Cuda(0);
1113 }
1114 #[cfg(not(any(
1115 all(target_os = "macos", feature = "metal"),
1116 all(
1117 any(target_os = "linux", target_os = "windows"),
1118 target_arch = "x86_64",
1119 not(car_skip_cuda)
1120 )
1121 )))]
1122 {
1123 Device::Cpu
1124 }
1125 }
1126}
1127
1128#[derive(Debug, Clone)]
1130pub struct InferenceConfig {
1131 pub models_dir: std::path::PathBuf,
1137 pub state_root: std::path::PathBuf,
1146 pub device: Option<Device>,
1148 pub generation_model: String,
1150 pub preferred_generation_model: Option<String>,
1152 pub embedding_model: String,
1154 pub preferred_embedding_model: Option<String>,
1156 pub classification_model: String,
1158 pub preferred_classification_model: Option<String>,
1160}
1161
1162impl Default for InferenceConfig {
1163 fn default() -> Self {
1164 let models_dir = default_models_dir();
1181
1182 let hw = HardwareInfo::detect();
1183
1184 Self {
1185 models_dir,
1186 state_root: car_home::root_or_relative(),
1187 device: None,
1188 generation_model: hw.recommended_model,
1189 preferred_generation_model: None,
1190 embedding_model: "Qwen3-Embedding-0.6B".to_string(),
1191 preferred_embedding_model: None,
1192 classification_model: "Qwen3-0.6B".to_string(),
1193 preferred_classification_model: None,
1194 }
1195 }
1196}
1197
1198impl InferenceConfig {
1199 pub fn state_models_dir(&self) -> std::path::PathBuf {
1208 self.state_root.join("models")
1209 }
1210}
1211
1212pub fn default_models_dir() -> std::path::PathBuf {
1219 dirs_next()
1220 .unwrap_or_else(|| std::path::PathBuf::from("."))
1221 .join(".car")
1222 .join("models")
1223}
1224
1225fn dirs_next() -> Option<std::path::PathBuf> {
1226 std::env::var_os("HOME")
1230 .or_else(|| std::env::var_os("USERPROFILE"))
1231 .map(std::path::PathBuf::from)
1232}
1233
1234fn model_source_identity(schema: &ModelSchema) -> &str {
1235 match &schema.source {
1236 ModelSource::Local { hf_repo, .. }
1237 | ModelSource::Mlx { hf_repo, .. }
1238 | ModelSource::ManagedVllmMlx { hf_repo, .. } => hf_repo,
1239 ModelSource::WhisperCpp { model } | ModelSource::CodexCli { model } => model,
1240 _ => &schema.id,
1241 }
1242}
1243
1244#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1246pub struct TokenUsage {
1247 pub prompt_tokens: u64,
1257 pub completion_tokens: u64,
1259 pub total_tokens: u64,
1261 pub context_window: u64,
1263 #[serde(default)]
1268 pub cache_read_input_tokens: u64,
1269 #[serde(default)]
1274 pub cache_creation_input_tokens: u64,
1275}
1276
1277pub(crate) fn tool_calls_text(calls: &[ToolCall]) -> String {
1286 calls
1287 .iter()
1288 .map(|call| {
1289 let arguments = serde_json::to_string(&call.arguments).unwrap_or_default();
1290 format!("{} {arguments}", call.name)
1291 })
1292 .collect::<Vec<_>>()
1293 .join("\n")
1294}
1295
1296pub(crate) fn estimate_output_tokens(text: &str, tool_calls: &[ToolCall]) -> usize {
1307 (text.len() + tool_calls_text(tool_calls).len()).div_ceil(4)
1308}
1309
1310pub(crate) fn booked_output_tokens(
1319 usage: Option<&TokenUsage>,
1320 text: &str,
1321 tool_calls: &[ToolCall],
1322) -> usize {
1323 match usage {
1324 Some(u) if u.completion_tokens > 0 => u.completion_tokens as usize,
1325 _ => estimate_output_tokens(text, tool_calls),
1326 }
1327}
1328
1329#[derive(
1331 Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq, schemars::JsonSchema,
1332)]
1333pub struct InferenceModelIdentity {
1334 #[serde(default)]
1337 pub requested_model_id: Option<String>,
1338 #[serde(default)]
1352 pub resolved_model_id: String,
1353 #[serde(default)]
1355 pub row_digest: String,
1356 #[serde(default)]
1358 pub catalog_revision: String,
1359}
1360
1361#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1363pub struct InferenceResult {
1364 pub text: String,
1366 pub tool_calls: Vec<crate::tasks::generate::ToolCall>,
1368 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1375 pub bounding_boxes: Vec<crate::tasks::grounding::BoundingBox>,
1376 pub trace_id: String,
1378 pub model_used: String,
1381 #[serde(flatten)]
1384 pub model_identity: InferenceModelIdentity,
1385 pub latency_ms: u64,
1387 #[serde(default)]
1399 pub time_to_first_token_ms: Option<u64>,
1400 pub usage: Option<TokenUsage>,
1419 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1432 pub provider_output_items: Vec<serde_json::Value>,
1433 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1441 pub thinking: Vec<crate::tasks::generate::ThinkingBlock>,
1442 #[serde(default)]
1458 pub stop_reason: Option<String>,
1459 #[serde(default, skip_serializing_if = "Option::is_none")]
1468 pub auth_fallback_from: Option<String>,
1469 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
1476 pub local_last_resort: bool,
1477 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1497 pub fallback_from: Vec<FallbackFrom>,
1498}
1499
1500#[derive(
1502 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1503)]
1504pub struct FallbackFrom {
1505 pub candidate: String,
1507 pub reason: FallbackReason,
1508}
1509
1510#[derive(
1518 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1519)]
1520#[serde(rename_all = "snake_case")]
1521pub enum FallbackReason {
1522 CredentialRejected,
1531 CredentialAbsent,
1534 RateLimited,
1537 QuotaExhausted,
1543 TimedOut,
1551 Failed,
1557}
1558
1559pub fn classify_fallback_reason(error: &InferenceError) -> FallbackReason {
1592 use FallbackReason as R;
1593 match error {
1594 InferenceError::ProviderAccount { status, .. } => match status {
1597 402 => R::QuotaExhausted,
1598 _ => R::CredentialRejected,
1599 },
1600 InferenceError::CredentialUnavailable { reason, .. } => match reason {
1602 CredentialFailure::Expired { .. } => R::CredentialRejected,
1604 CredentialFailure::SignedOut | CredentialFailure::EnvVarMissing { .. } => {
1606 R::CredentialAbsent
1607 }
1608 CredentialFailure::StoreUnreadable | CredentialFailure::RaceRetryable => R::Failed,
1611 },
1612 InferenceError::Transient {
1613 status: Some(429), ..
1614 } => R::RateLimited,
1615 InferenceError::Transient { status: None, .. } => R::Failed,
1622 InferenceError::Transient { .. } => R::Failed,
1623 InferenceError::DeadlineExceeded { .. } => R::TimedOut,
1626 InferenceError::InferenceFailed(msg)
1627 | InferenceError::ProviderKeyMissing { message: msg, .. } => classify_untyped_failure(msg),
1628 _ => R::Failed,
1629 }
1630}
1631
1632fn classify_untyped_failure(msg: &str) -> FallbackReason {
1640 use FallbackReason as R;
1641 if let Some(status) = parse_api_returned_status(msg) {
1644 return match status {
1645 401 | 403 => R::CredentialRejected,
1646 402 => R::QuotaExhausted,
1647 429 => R::RateLimited,
1648 408 | 504 => R::TimedOut,
1649 _ => R::Failed,
1650 };
1651 }
1652 let l = msg.to_ascii_lowercase();
1653 if l.contains("org lookup failed") {
1656 return match parse_org_lookup_status(msg) {
1657 Some(401) => R::CredentialRejected,
1658 Some(403) => R::Failed,
1661 Some(429) => R::RateLimited,
1662 _ => R::Failed,
1663 };
1664 }
1665 if l.contains("authentication required")
1666 || l.contains("invalid_grant")
1667 || l.contains("token expired")
1668 {
1669 return R::CredentialRejected;
1670 }
1671 if l.contains("no credential for proprietary") || l.contains("no api key") {
1672 return R::CredentialAbsent;
1673 }
1674 if l.contains("too many requests") || l.contains("rate limit") {
1675 return R::RateLimited;
1676 }
1677 if l.contains("timed out") || l.contains("deadline exceeded") {
1678 return R::TimedOut;
1679 }
1680 R::Failed
1681}
1682
1683pub fn record_fallback_from(hops: &mut Vec<FallbackFrom>, candidate: &str, error: &InferenceError) {
1689 hops.push(FallbackFrom {
1690 candidate: candidate.to_string(),
1691 reason: classify_fallback_reason(error),
1692 });
1693}
1694
1695pub struct TrackedStream {
1702 pub model_used: String,
1704 pub trace_id: String,
1706 pub events: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
1708}
1709
1710struct AbortOnDropTask<T>(Option<tokio::task::JoinHandle<T>>);
1711
1712impl<T> AbortOnDropTask<T> {
1713 async fn join(mut self) -> Result<T, tokio::task::JoinError> {
1714 self.0.take().expect("owned task handle available").await
1715 }
1716}
1717
1718impl<T> Drop for AbortOnDropTask<T> {
1719 fn drop(&mut self) {
1720 if let Some(task) = self.0.take() {
1721 task.abort();
1722 }
1723 }
1724}
1725
1726tokio::task_local! {
1727 static PINNED_REGISTRY: (usize, Arc<UnifiedRegistry>);
1731}
1732
1733fn bound_model_identity(
1734 snapshot: &CatalogSnapshot,
1735 requested_model_id: Option<&str>,
1736 resolved_model_id: &str,
1737) -> Result<InferenceModelIdentity, InferenceError> {
1738 let row = snapshot.model_by_exact_id(resolved_model_id).ok_or_else(|| {
1739 InferenceError::InferenceFailed(format!(
1740 "resolved model `{resolved_model_id}` was absent from the catalog snapshot bound to this request"
1741 ))
1742 })?;
1743 Ok(InferenceModelIdentity {
1744 requested_model_id: requested_model_id.map(str::to_string),
1745 resolved_model_id: row.model.id.clone(),
1746 row_digest: row.row_digest.clone(),
1747 catalog_revision: snapshot.catalog_revision.clone(),
1748 })
1749}
1750
1751fn validate_expected_catalog_revision(
1752 req: &GenerateRequest,
1753 snapshot: &CatalogSnapshot,
1754) -> Result<(), InferenceError> {
1755 if let Some(expected) = req.expected_catalog_revision.as_deref() {
1756 if expected != snapshot.catalog_revision {
1757 return Err(InferenceError::CatalogPreconditionMismatch {
1758 detail: format!(
1759 "expected catalog revision {expected}, got {}",
1760 snapshot.catalog_revision
1761 ),
1762 });
1763 }
1764 }
1765 Ok(())
1766}
1767
1768fn validate_expected_catalog_row(
1769 req: &GenerateRequest,
1770 snapshot: &CatalogSnapshot,
1771 resolved_model_id: &str,
1772) -> Result<(), InferenceError> {
1773 let Some(expected) = req.expected_row_digest.as_deref() else {
1774 return Ok(());
1775 };
1776 let row = snapshot
1777 .model_by_exact_id(resolved_model_id)
1778 .ok_or_else(|| InferenceError::CatalogPreconditionMismatch {
1779 detail: format!(
1780 "resolved model `{resolved_model_id}` is absent from the bound catalog snapshot"
1781 ),
1782 })?;
1783 if expected != row.row_digest {
1784 return Err(InferenceError::CatalogPreconditionMismatch {
1785 detail: format!(
1786 "expected row digest {expected} for `{resolved_model_id}`, got {}",
1787 row.row_digest
1788 ),
1789 });
1790 }
1791 Ok(())
1792}
1793
1794const EXACT_MODEL_ID_PREFIX: &str = "\0car-exact-model-id:";
1795
1796pub fn pin_exact_model_id(req: &mut GenerateRequest, model_id: String) -> Result<(), String> {
1799 if req.model.is_some() {
1800 return Err("`model` and `model_id` are mutually exclusive".to_string());
1801 }
1802 if model_id.trim().is_empty() {
1803 return Err("`model_id` must be a non-empty immutable id".to_string());
1804 }
1805 req.model = Some(format!("{EXACT_MODEL_ID_PREFIX}{model_id}"));
1806 req.params.strict_model = true;
1807 Ok(())
1808}
1809
1810fn default_system_one_classifier<'a>(
1815 schemas: impl Iterator<Item = &'a ModelSchema>,
1816) -> Option<String> {
1817 schemas
1818 .filter(|schema| {
1819 schema.has_capability(ModelCapability::Classify)
1820 && schema.available_now()
1821 && matches!(
1822 &schema.source,
1823 ModelSource::Proprietary { protocol, .. }
1824 if protocol.wire == ProprietaryWire::SystemOne
1825 )
1826 })
1827 .map(|schema| schema.id.clone())
1828 .min()
1829}
1830
1831pub fn exact_pinned_model_id(req: &GenerateRequest) -> Option<&str> {
1832 req.model
1833 .as_deref()
1834 .and_then(|model| model.strip_prefix(EXACT_MODEL_ID_PREFIX))
1835}
1836
1837fn auto_thinking_budget(
1846 is_code_intent: bool,
1847 is_complex: bool,
1848 supports_thinking: bool,
1849) -> Option<usize> {
1850 if !supports_thinking {
1851 return None;
1852 }
1853 if is_code_intent {
1854 Some(24_000)
1855 } else if is_complex {
1856 Some(8_000)
1857 } else {
1858 None
1859 }
1860}
1861
1862fn is_explicit_code_intent(intent: Option<&intent::IntentHint>) -> bool {
1872 intent.and_then(|h| h.task) == Some(intent::TaskHint::Code)
1873}
1874
1875fn should_append_local_last_resort(chain_has_local: bool, strict_model: bool) -> bool {
1880 !chain_has_local && !strict_model
1881}
1882
1883pub fn reasoning_output_allowance(
1893 schema: Option<&ModelSchema>,
1894 answer_tokens: usize,
1895 budget_tokens: usize,
1896) -> usize {
1897 let charged = schema.is_some_and(|schema| match &schema.source {
1898 ModelSource::RemoteApi { protocol, .. } => {
1899 protocol::reasoning_counts_against_max_tokens(*protocol, &schema.name)
1900 }
1901 _ => false,
1902 });
1903 match schema {
1904 Some(schema) if charged && budget_tokens > 0 => (answer_tokens + budget_tokens)
1905 .min(schema.effective_max_output())
1906 .max(answer_tokens),
1907 _ => answer_tokens,
1908 }
1909}
1910
1911fn resolved_max_tokens(requested: usize, schema: &ModelSchema) -> usize {
1935 if requested != crate::tasks::generate::DEFAULT_MAX_TOKENS
1936 || schema.decodes_in_process()
1937 || schema.is_codex_cli()
1938 {
1939 return requested;
1940 }
1941 schema.effective_max_output()
1942}
1943
1944const DEFAULT_LOCAL_DECODE_TIMEOUT_SECS: u64 = 300;
1957
1958pub const LOCAL_DECODE_TIMEOUT_STOP_REASON: &str = "local_decode_timeout";
1965
1966#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1974const LOCAL_DECODE_HEARTBEAT_SECS: u64 = 10;
1975
1976fn parse_decode_timeout(raw: Option<&str>) -> Option<std::time::Duration> {
1979 let secs = match raw {
1980 Some(v) => v
1981 .trim()
1982 .parse::<u64>()
1983 .unwrap_or(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS),
1984 None => DEFAULT_LOCAL_DECODE_TIMEOUT_SECS,
1985 };
1986 (secs > 0).then(|| std::time::Duration::from_secs(secs))
1987}
1988
1989#[cfg_attr(
2002 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
2003 allow(dead_code)
2004)]
2005fn deadline_exceeded(elapsed: std::time::Duration, timeout: Option<std::time::Duration>) -> bool {
2006 timeout.is_some_and(|limit| elapsed >= limit)
2007}
2008
2009#[cfg_attr(
2015 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
2016 allow(dead_code)
2017)]
2018fn heartbeat_due(
2019 elapsed: std::time::Duration,
2020 last: std::time::Duration,
2021 interval: std::time::Duration,
2022) -> bool {
2023 elapsed.saturating_sub(last) >= interval
2024}
2025
2026#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2030enum EmptyPassAction {
2031 RetryWithoutThinking,
2034 FailDecodeCeiling,
2038 Accept,
2040}
2041
2042fn classify_empty_pass(
2050 recover: bool,
2051 stop_reason: Option<&str>,
2052 text: &str,
2053 tool_calls_empty: bool,
2054) -> EmptyPassAction {
2055 if !text.trim().is_empty() || !tool_calls_empty {
2056 return EmptyPassAction::Accept;
2057 }
2058 if stop_reason == Some(LOCAL_DECODE_TIMEOUT_STOP_REASON) {
2059 return EmptyPassAction::FailDecodeCeiling;
2060 }
2061 if recover {
2062 return EmptyPassAction::RetryWithoutThinking;
2063 }
2064 EmptyPassAction::Accept
2065}
2066
2067fn local_decode_timeout() -> Option<std::time::Duration> {
2068 parse_decode_timeout(
2069 std::env::var("CAR_LOCAL_DECODE_TIMEOUT_SECS")
2070 .ok()
2071 .as_deref(),
2072 )
2073}
2074
2075impl InferenceResult {
2076 pub fn served_model_id(&self) -> &str {
2081 if self.model_identity.resolved_model_id.is_empty() {
2082 &self.model_used
2083 } else {
2084 &self.model_identity.resolved_model_id
2085 }
2086 }
2087
2088 pub fn has_tool_calls(&self) -> bool {
2090 !self.tool_calls.is_empty()
2091 }
2092
2093 pub fn was_truncated(&self) -> bool {
2105 matches!(
2106 self.stop_reason.as_deref(),
2107 Some(
2108 "length"
2109 | "max_tokens"
2110 | "max_output_tokens"
2111 | "MAX_TOKENS"
2112 | crate::LOCAL_DECODE_TIMEOUT_STOP_REASON
2113 )
2114 )
2115 }
2116
2117 pub fn append_assistant_history(
2126 &self,
2127 messages: &mut Vec<crate::tasks::generate::Message>,
2128 tool_calls: Vec<crate::tasks::generate::ToolCall>,
2129 ) {
2130 if !self.provider_output_items.is_empty() {
2131 messages.push(crate::tasks::generate::Message::ProviderOutputItems {
2132 protocol: crate::protocol::OPENAI_RESPONSES_PROTOCOL.to_string(),
2133 items: self.provider_output_items.clone(),
2134 });
2135 }
2136 messages.push(crate::tasks::generate::Message::Assistant {
2137 content: self.text.clone(),
2138 tool_calls,
2139 thinking: self.thinking.clone(),
2140 model_id: Some(self.served_model_id().to_string()),
2141 local_last_resort: self.local_last_resort,
2142 });
2143 }
2144}
2145
2146#[derive(Debug, Clone, Serialize)]
2147pub struct SpeechRuntimeHealth {
2148 pub root: PathBuf,
2149 pub installed: bool,
2150 pub python: PathBuf,
2151 pub stt_command: PathBuf,
2152 pub tts_command: PathBuf,
2153 pub configured_python: Option<String>,
2154 pub detected_python: Option<String>,
2155}
2156
2157#[derive(Debug, Clone, Serialize)]
2158pub struct SpeechModelHealth {
2159 pub id: String,
2160 pub name: String,
2161 pub provider: String,
2162 pub capability: ModelCapability,
2163 pub is_local: bool,
2164 pub available: bool,
2165 pub cached: bool,
2166 pub selected_by_default: bool,
2167 pub source: String,
2168}
2169
2170#[derive(Debug, Clone, Serialize)]
2171pub struct SpeechHealthReport {
2172 pub runtime: SpeechRuntimeHealth,
2173 pub local_models: Vec<SpeechModelHealth>,
2174 pub remote_models: Vec<SpeechModelHealth>,
2175 pub elevenlabs_configured: bool,
2176 pub prefer_local: bool,
2177 pub allow_remote_fallback: bool,
2178 pub preferred_local_stt: Option<String>,
2179 pub preferred_local_tts: Option<String>,
2180 pub preferred_remote_stt: Option<String>,
2181 pub preferred_remote_tts: Option<String>,
2182 pub local_stt_default: Option<String>,
2183 pub local_tts_default: Option<String>,
2184 pub remote_stt_default: Option<String>,
2185 pub remote_tts_default: Option<String>,
2186}
2187
2188#[derive(Debug, Clone, Serialize)]
2189pub struct ModelDefaultHealth {
2190 pub capability: ModelCapability,
2191 pub configured_model: String,
2192 pub available: bool,
2193 pub is_local: bool,
2194 pub provider: Option<String>,
2195}
2196
2197#[derive(Debug, Clone, Serialize)]
2198pub struct ModelProviderHealth {
2199 pub provider: String,
2200 pub configured: bool,
2201 pub local_models: usize,
2202 pub remote_models: usize,
2203 pub available_models: usize,
2204 pub capabilities: Vec<ModelCapability>,
2205}
2206
2207#[derive(Debug, Clone, Serialize)]
2208pub struct ModelCapabilityHealth {
2209 pub capability: ModelCapability,
2210 pub total_models: usize,
2211 pub available_models: usize,
2212 pub local_available_models: usize,
2213 pub remote_available_models: usize,
2214}
2215
2216#[derive(Debug, Clone, Serialize)]
2217pub struct RoutingScenarioHealth {
2218 pub name: String,
2219 pub workload: RoutingWorkload,
2220 pub task_family: String,
2221 pub has_tools: bool,
2222 pub has_vision: bool,
2223 pub prefer_local: bool,
2224 pub quality_first_cold_start: bool,
2225 pub bootstrap_min_task_observations: u64,
2226 pub bootstrap_quality_floor: f64,
2227 pub model_id: String,
2228 pub model_name: String,
2229 pub reason: String,
2230 pub strategy: RoutingStrategy,
2231}
2232
2233#[derive(Debug, Clone, Serialize)]
2234pub struct ModelBenchmarkPriorHealth {
2235 pub model_id: String,
2236 pub model_name: Option<String>,
2237 pub overall_score: f64,
2238 pub overall_latency_ms: Option<f64>,
2239 pub task_scores: std::collections::HashMap<String, f64>,
2240 pub task_latency_ms: std::collections::HashMap<String, f64>,
2241 pub source_path: PathBuf,
2242}
2243
2244#[derive(Debug, Clone, Serialize)]
2245pub struct ModelHealthReport {
2246 pub total_models: usize,
2247 pub available_models: usize,
2248 pub local_models: usize,
2249 pub remote_models: usize,
2250 pub defaults: Vec<ModelDefaultHealth>,
2251 pub providers: Vec<ModelProviderHealth>,
2252 pub capabilities: Vec<ModelCapabilityHealth>,
2253 pub routing_prefer_local: bool,
2254 pub routing_quality_first_cold_start: bool,
2255 pub routing_min_observations: u64,
2256 pub routing_bootstrap_min_task_observations: u64,
2257 pub routing_bootstrap_quality_floor: f64,
2258 pub routing_quality_weight: f64,
2259 pub routing_latency_weight: f64,
2260 pub routing_cost_weight: f64,
2261 pub routing_scenarios: Vec<RoutingScenarioHealth>,
2262 pub benchmark_priors: Vec<ModelBenchmarkPriorHealth>,
2263 pub speech: SpeechHealthReport,
2264}
2265
2266#[derive(Debug, Clone, Serialize)]
2267pub struct SpeechInstallReport {
2268 pub name: String,
2269 pub hf_repo: String,
2270 pub snapshot_path: PathBuf,
2271 pub files_downloaded: usize,
2272}
2273
2274#[derive(Debug, Clone, Serialize)]
2275pub struct SpeechSmokePathReport {
2276 pub path: String,
2277 pub tts_model: String,
2278 pub stt_model: String,
2279 pub audio_path: PathBuf,
2280 pub transcript: String,
2281}
2282
2283#[derive(Debug, Clone, Serialize, Default)]
2284pub struct SpeechSmokeReport {
2285 pub local: Option<SpeechSmokePathReport>,
2286 pub remote: Option<SpeechSmokePathReport>,
2287 pub skipped: Vec<String>,
2288}
2289
2290#[derive(Debug, Clone, Serialize, Default)]
2291pub struct SpeechPolicy {
2292 pub prefer_local: bool,
2293 pub allow_remote_fallback: bool,
2294 pub preferred_local_stt: Option<String>,
2295 pub preferred_local_tts: Option<String>,
2296 pub preferred_remote_stt: Option<String>,
2297 pub preferred_remote_tts: Option<String>,
2298}
2299
2300fn render_for_local_backend(mut req: GenerateRequest) -> GenerateRequest {
2309 let has_msgs = req.messages.as_ref().is_some_and(|m| !m.is_empty());
2310 let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
2311 if has_msgs || has_tools {
2312 req.prompt = tasks::generate::render_chat_prompt(&req);
2313 req.messages = None;
2314 req.tools = None;
2315 }
2316 req
2317}
2318
2319#[cfg(test)]
2320static UNRETIRED_RELEASE_WARNING_COUNT: std::sync::atomic::AtomicUsize =
2321 std::sync::atomic::AtomicUsize::new(0);
2322
2323#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2340fn foundation_models_usage(
2341 instructions: Option<&str>,
2342 prompt: &str,
2343 completion: &str,
2344 context_window: u64,
2345) -> Option<crate::TokenUsage> {
2346 let (prompt_tokens, completion_tokens) =
2347 crate::backend::foundation_models::count_tokens(instructions, prompt, completion)?;
2348 Some(crate::TokenUsage {
2349 prompt_tokens,
2350 completion_tokens,
2351 total_tokens: prompt_tokens + completion_tokens,
2352 context_window,
2353 ..Default::default()
2355 })
2356}
2357
2358#[cfg(any(
2364 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
2365 all(target_os = "ios", target_arch = "aarch64")
2366))]
2367fn foundation_models_instructions(
2368 context: Option<&str>,
2369 response_format: Option<&crate::tasks::generate::ResponseFormat>,
2370) -> Option<String> {
2371 if matches!(
2372 response_format,
2373 Some(crate::tasks::generate::ResponseFormat::JsonObject)
2374 ) {
2375 tracing::warn!(
2376 "FoundationModels: JsonObject response_format has no native \
2377 constrained mode — enforcing via instruction injection"
2378 );
2379 let base = context.unwrap_or_default();
2380 Some(format!(
2381 "{base}\n\nRespond with a single valid JSON object and nothing else."
2382 ))
2383 } else {
2384 context.map(str::to_string)
2385 }
2386}
2387
2388#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2394fn foundation_models_inline_images(blocks: &[ContentBlock]) -> Result<Vec<String>, InferenceError> {
2395 let unsupported = |reason: &'static str| InferenceError::UnsupportedMode {
2396 mode: "multimodal-content",
2397 backend: "foundation-models",
2398 reason,
2399 };
2400 blocks
2401 .iter()
2402 .map(|block| match block {
2403 ContentBlock::ImageBase64 { data, .. } => Ok(data.clone()),
2404 ContentBlock::ImageUrl { .. } => Err(unsupported(
2407 "FoundationModels takes inline image bytes; \
2408 image URLs must be fetched by the caller first",
2409 )),
2410 ContentBlock::Text { .. } => Err(unsupported(
2411 "FoundationModels takes only image blocks in `images` — \
2412 send text in the prompt",
2413 )),
2414 ContentBlock::VideoPath { .. }
2415 | ContentBlock::VideoUrl { .. }
2416 | ContentBlock::VideoBase64 { .. }
2417 | ContentBlock::AudioPath { .. }
2418 | ContentBlock::AudioUrl { .. }
2419 | ContentBlock::AudioBase64 { .. } => Err(unsupported(
2420 "the FoundationModels bridge exposes text and image \
2421 input — route audio/video to a remote VL model",
2422 )),
2423 })
2424 .collect()
2425}
2426
2427pub struct InferenceEngine {
2428 pub config: InferenceConfig,
2429 unified: std::sync::RwLock<Arc<UnifiedRegistry>>,
2435 registry_write: std::sync::Mutex<()>,
2438 pub adaptive_router: AdaptiveRouter,
2440 pub outcome_tracker: Arc<RwLock<OutcomeTracker>>,
2442 last_outcome_flush: Arc<std::sync::Mutex<Option<Instant>>>,
2448 outcome_profiles_loaded: bool,
2452 ledger_io_lock: Arc<tokio::sync::Mutex<()>>,
2457 spend_limits: Arc<std::sync::RwLock<Option<SpendLimits>>>,
2463 lane_defaults_cache: Arc<std::sync::RwLock<crate::lane_defaults::LaneDefaults>>,
2468 external_pins: Arc<std::sync::RwLock<std::collections::BTreeMap<String, Vec<String>>>>,
2472 disk_pressured: Arc<std::sync::atomic::AtomicBool>,
2475 failed_upgrades: Arc<std::sync::Mutex<std::collections::HashMap<String, u64>>>,
2479 concierge_action_lock: Arc<tokio::sync::Mutex<()>>,
2485 concierge_action_seq: Arc<std::sync::atomic::AtomicU64>,
2489 remote_backend: RemoteBackend,
2491 model_management: model_management::ModelManagementStore,
2494 local_admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2496 _runtime_scope: Arc<ScopedInferenceRuntime>,
2501 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2502 resource_policy_evidence: Arc<std::sync::RwLock<resource_policy::ResourcePolicyLoadEvidence>>,
2506 model_budget: Arc<backend_cache::SharedModelBudget>,
2509 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2515 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2516 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2523 local_backends:
2524 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2525 pub registry: models::ModelRegistry,
2527 pub router: ModelRouter,
2528 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2529 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2530 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2531 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2532 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2533 speech_policy: SpeechPolicy,
2534 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2538}
2539
2540enum SpeechCandidateAdmission {
2541 Proceed(Option<resource_policy::LocalLoadReservation>),
2542 SkipBlocked(InferenceError),
2543 FailBlocked(InferenceError),
2544}
2545
2546struct ScopedInferenceRuntime {
2547 model_budget: Arc<backend_cache::SharedModelBudget>,
2548 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2549 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2550 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2551 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2552 local_backends:
2553 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2554 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2555 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2556 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2557 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2558 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2559 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2560 #[cfg(test)]
2561 load_probe: Arc<backend_cache::BackendCache<()>>,
2562}
2563
2564fn scoped_inference_runtime_registry() -> &'static std::sync::Mutex<
2565 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2566> {
2567 static REGISTRY: std::sync::OnceLock<
2568 std::sync::Mutex<
2569 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2570 >,
2571 > = std::sync::OnceLock::new();
2572 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
2573}
2574
2575fn configured_cache_budget_mb(default_mb: u64) -> u64 {
2576 std::env::var("CAR_INFERENCE_MODEL_CACHE_MB")
2577 .ok()
2578 .and_then(|value| value.parse::<u64>().ok())
2579 .unwrap_or(default_mb)
2580}
2581
2582fn scoped_inference_runtime(
2583 state_root: &Path,
2584 configured_ceiling_mb: u64,
2585 admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2586) -> Arc<ScopedInferenceRuntime> {
2587 let state_root = resource_policy::normalized_state_root_key(state_root);
2588 let mut registry = scoped_inference_runtime_registry()
2589 .lock()
2590 .unwrap_or_else(std::sync::PoisonError::into_inner);
2591 registry.retain(|_, runtime| runtime.strong_count() > 0);
2592 if let Some(runtime) = registry.get(&state_root).and_then(std::sync::Weak::upgrade) {
2593 runtime.model_budget.set_budget_bytes(
2594 configured_cache_budget_mb(configured_ceiling_mb).saturating_mul(1024 * 1024),
2595 );
2596 return runtime;
2597 }
2598
2599 let model_budget = backend_cache::SharedModelBudget::from_env_or(configured_ceiling_mb);
2600 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2601 let cache_idle = backend_cache::idle_ttl_from_env();
2602 let runtime = Arc::new(ScopedInferenceRuntime {
2603 model_budget: model_budget.clone(),
2604 resource_policy_generation: Arc::new(std::sync::atomic::AtomicU64::new(1)),
2605 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2606 text_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2607 model_budget.clone(),
2608 cache_idle,
2609 Some(admission.clone()),
2610 )),
2611 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2612 local_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2613 model_budget.clone(),
2614 cache_idle,
2615 Some(admission.clone()),
2616 )),
2617 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2618 backend: Arc::new(RwLock::new(std::collections::HashMap::new())),
2619 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2620 embedding_backend: Arc::new(RwLock::new(None)),
2621 speech_runtime: Arc::new(Mutex::new(None)),
2622 vllm_pool: Arc::new(
2626 vllm_pool::VllmServerPool::with_admission(
2627 std::time::Duration::from_secs(
2628 std::env::var("CAR_VLLM_IDLE_SECS")
2629 .ok()
2630 .and_then(|value| value.parse().ok())
2631 .unwrap_or(300),
2632 ),
2633 admission,
2634 )
2635 .with_process_records(state_root.join("run").join("vllm-mlx")),
2636 ),
2637 #[cfg(test)]
2638 load_probe: Arc::new(backend_cache::BackendCache::new(1024)),
2639 });
2640 registry.insert(state_root, Arc::downgrade(&runtime));
2641 runtime
2642}
2643
2644impl InferenceEngine {
2645 fn requires_local_admission(schema: &ModelSchema) -> bool {
2646 schema.is_car_managed_vllm_mlx() || Self::supports_worker_offload(schema)
2647 }
2648
2649 fn reserve_in_outer_dispatch(schema: &ModelSchema) -> bool {
2655 Self::requires_local_admission(schema) && !schema.is_car_managed_vllm_mlx()
2656 }
2657
2658 fn supports_worker_offload(schema: &ModelSchema) -> bool {
2659 matches!(
2660 schema.source,
2661 ModelSource::Local { .. } | ModelSource::Mlx { .. }
2662 )
2663 }
2664
2665 pub fn local_model_retention(&self, model_id: &str) -> backend_cache::BackendRetention {
2668 if self.local_admission.is_resident(model_id) {
2669 backend_cache::BackendRetention::Resident
2670 } else {
2671 backend_cache::BackendRetention::Transient
2672 }
2673 }
2674 pub fn local_model_preflight(
2676 &self,
2677 model_id: &str,
2678 context_tokens: usize,
2679 ) -> Result<resource_policy::LocalLoadPreflight, InferenceError> {
2680 let schema = self
2681 .resolve_schema(model_id)
2682 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
2683 self.ensure_model_enabled(&schema.id)?;
2684 Ok(self.local_admission.preflight(&schema, context_tokens))
2685 }
2686
2687 pub fn active_local_resource_policy(&self) -> resource_policy::ResourcePolicyLoadEvidence {
2694 let mut evidence = self
2695 .resource_policy_evidence
2696 .read()
2697 .unwrap_or_else(|poisoned| poisoned.into_inner())
2698 .clone();
2699 evidence.policy = self.local_admission.policy();
2700 evidence
2701 }
2702
2703 pub fn apply_local_resource_policy(&self, policy: resource_policy::ResourcePolicy) {
2707 let ceiling_mb = policy
2708 .effective_budget(HardwareInfo::detect().total_ram_mb)
2709 .configured_model_ceiling_mb;
2710 self.local_admission.set_policy(policy.clone());
2711 *self
2712 .resource_policy_evidence
2713 .write()
2714 .unwrap_or_else(|poisoned| poisoned.into_inner()) =
2715 resource_policy::ResourcePolicyLoadEvidence {
2716 policy,
2717 source: resource_policy::ResourcePolicyLoadSource::Loaded,
2718 warning: None,
2719 };
2720 let cache_ceiling_mb = configured_cache_budget_mb(ceiling_mb);
2721 self.model_budget
2722 .set_budget_bytes(cache_ceiling_mb.saturating_mul(1024 * 1024));
2723 let generation = self
2724 .resource_policy_generation
2725 .fetch_add(1, std::sync::atomic::Ordering::AcqRel)
2726 .saturating_add(1);
2727 if let Some(offload) = crate::offload::current_local_offload() {
2728 offload.refresh_resource_policy(generation);
2729 }
2730 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2731 {
2732 self.text_backends.enforce_budget();
2733 self.local_backends.enforce_budget();
2734 }
2735 }
2736
2737 pub fn begin_local_model_maintenance(
2738 &self,
2739 model_id: &str,
2740 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2741 {
2742 self.local_admission.begin_model_maintenance(model_id)
2743 }
2744
2745 pub async fn prepare_local_model_removal(
2746 &self,
2747 model_id: &str,
2748 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2749 {
2750 let maintenance = self.local_admission.begin_model_maintenance(model_id)?;
2751 if let Some(offload) = crate::offload::current_local_offload() {
2752 if offload
2753 .resident_models()
2754 .await
2755 .iter()
2756 .any(|resident| resident == model_id)
2757 {
2758 let acknowledged = offload.release_model(model_id).await.map_err(|error| {
2759 resource_policy::ModelMaintenanceError::ReleaseFailed(error.to_string())
2760 })?;
2761 if !acknowledged
2762 || offload
2763 .resident_models()
2764 .await
2765 .iter()
2766 .any(|resident| resident == model_id)
2767 {
2768 return Err(
2769 resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
2770 model_id.to_string(),
2771 ),
2772 );
2773 }
2774 }
2775 }
2776 if self
2777 .vllm_pool
2778 .release_model_if_present(model_id)
2779 .await
2780 .is_err()
2781 {
2782 return Err(
2783 resource_policy::ModelMaintenanceError::ProcessReleaseUnacknowledged(
2784 model_id.to_string(),
2785 ),
2786 );
2787 }
2788 if !self.evict_local_model_if_idle(model_id) {
2789 return Err(resource_policy::ModelMaintenanceError::CacheReleaseBlocked(
2790 model_id.to_string(),
2791 ));
2792 }
2793 let allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2794 if !allocation_ids.is_empty() {
2795 return Err(resource_policy::ModelMaintenanceError::ResidualResidency {
2796 model_id: model_id.to_string(),
2797 allocation_ids,
2798 });
2799 }
2800 Ok(maintenance)
2801 }
2802
2803 pub fn evict_local_model_if_idle(&self, model_id: &str) -> bool {
2807 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2808 {
2809 let mut allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2810 if !allocation_ids.iter().any(|id| id == model_id) {
2811 allocation_ids.push(model_id.to_string());
2812 }
2813 allocation_ids.into_iter().all(|allocation_id| {
2814 [
2815 self.text_backends.evict_if_idle(&allocation_id),
2816 self.local_backends.evict_if_idle(&allocation_id),
2817 ]
2818 .into_iter()
2819 .all(|evicted| evicted)
2820 })
2821 }
2822 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2823 {
2824 let _ = model_id;
2825 let backend_released = self.backend.try_write().is_ok_and(|mut backends| {
2826 if backends.remove(model_id).is_some() {
2827 self.local_admission.mark_evicted(model_id);
2828 }
2829 true
2830 });
2831 backend_released
2835 && self
2836 .embedding_backend
2837 .try_write()
2838 .is_ok_and(|slot| slot.is_none())
2839 }
2840 }
2841
2842 fn reserve_local_request(
2843 &self,
2844 schema: &ModelSchema,
2845 context_tokens: usize,
2846 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2847 self.ensure_model_enabled(&schema.id)?;
2848 let activity_lease = self.model_management.acquire_lease(&schema.id)?;
2849 let mut reservation = self
2850 .local_admission
2851 .reserve(schema, context_tokens)
2852 .map_err(InferenceError::from)?;
2853 reservation.attach_activity_lease(activity_lease);
2854 Ok(reservation)
2855 }
2856
2857 async fn reserve_local_request_with_worker_retry(
2862 &self,
2863 schema: &ModelSchema,
2864 context_tokens: usize,
2865 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2866 let original_error = match self.reserve_local_request(schema, context_tokens) {
2867 Ok(reservation) => return Ok(reservation),
2868 Err(error) => error,
2869 };
2870 let first_verdict = match &original_error {
2871 InferenceError::LocalResourceBlocked { preflight, .. } => preflight.verdict.clone(),
2872 _ => return Err(original_error),
2873 };
2874 if first_verdict != resource_policy::LocalLoadVerdict::InsufficientLiveMemory {
2875 return Err(original_error);
2876 }
2877
2878 let Some(offload) = crate::offload::current_local_offload() else {
2879 tracing::info!(
2880 model = %schema.id,
2881 first_verdict = ?first_verdict,
2882 release_happened = false,
2883 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2884 "local admission worker release-and-retry decision"
2885 );
2886 return Err(original_error);
2887 };
2888 if !offload
2889 .resident_models()
2890 .await
2891 .iter()
2892 .any(|model_id| model_id == &schema.id)
2893 {
2894 tracing::info!(
2895 model = %schema.id,
2896 first_verdict = ?first_verdict,
2897 release_happened = false,
2898 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2899 "local admission worker release-and-retry decision"
2900 );
2901 return Err(original_error);
2902 }
2903 let Some(scoped_allocation_id) = offload.resident_allocation_id(&schema.id) else {
2904 tracing::debug!(
2905 model = %schema.id,
2906 "release-and-retry skipped because the offload exposes no allocation id"
2907 );
2908 return Err(original_error);
2909 };
2910 match offload.release_model(&schema.id).await {
2911 Ok(true) => {
2912 let offload_still_resident = offload
2916 .resident_models()
2917 .await
2918 .iter()
2919 .any(|model_id| model_id == &schema.id);
2920 let allocation_still_charged = self
2923 .local_admission
2924 .resident_allocation_ids(&schema.id)
2925 .iter()
2926 .any(|allocation_id| allocation_id == &scoped_allocation_id);
2927 if offload_still_resident || allocation_still_charged {
2928 tracing::warn!(
2932 model = %schema.id,
2933 allocation = %scoped_allocation_id,
2934 offload_still_resident,
2935 allocation_still_charged,
2936 "release acknowledged but allocation {} still charged/pending",
2937 scoped_allocation_id
2938 );
2939 #[cfg(test)]
2940 UNRETIRED_RELEASE_WARNING_COUNT
2941 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2942 return Err(original_error);
2943 }
2944 let second = self.reserve_local_request(schema, context_tokens);
2945 let second_verdict = match &second {
2946 Ok(_) => Some("admitted".to_string()),
2947 Err(InferenceError::LocalResourceBlocked { preflight, .. }) => {
2948 Some(format!("{:?}", preflight.verdict))
2949 }
2950 Err(_) => None,
2951 };
2952 tracing::info!(
2953 model = %schema.id,
2954 first_verdict = ?first_verdict,
2955 release_happened = true,
2956 second_verdict = ?second_verdict,
2957 "local admission worker release-and-retry decision"
2958 );
2959 second
2960 }
2961 Ok(false) | Err(_) => {
2962 tracing::info!(
2963 model = %schema.id,
2964 first_verdict = ?first_verdict,
2965 release_happened = false,
2966 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2967 "local admission worker release-and-retry decision"
2968 );
2969 Err(original_error)
2970 }
2971 }
2972 }
2973
2974 #[cfg(any(
2979 test,
2980 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))
2981 ))]
2982 fn reconcile_transient_local_allocation(
2983 reservation: &mut resource_policy::LocalLoadReservation,
2984 measured_weights_bytes: u64,
2985 ) -> Result<(), InferenceError> {
2986 reservation
2987 .reconcile_measured_weights(measured_weights_bytes)
2988 .map(|_| ())
2989 .map_err(InferenceError::from)
2990 }
2991
2992 fn prepare_worker_admission(
2993 &self,
2994 schema: &ModelSchema,
2995 reservation: &mut resource_policy::LocalLoadReservation,
2996 ) -> Result<crate::offload::LocalWorkerAdmission, InferenceError> {
2997 let installed = self.config.models_dir.join(&schema.name);
2998 let installed_weights_bytes = backend_cache::estimate_model_size(&installed);
2999 let measured_weights_bytes =
3006 installed_weights_bytes.max(reservation.reconciled_weights_bytes());
3007 reservation
3008 .reconcile_measured_weights(measured_weights_bytes)
3009 .map_err(InferenceError::from)?;
3010 Ok(crate::offload::LocalWorkerAdmission {
3011 policy: self.local_admission.policy(),
3012 policy_generation: self
3013 .resource_policy_generation
3014 .load(std::sync::atomic::Ordering::Acquire),
3015 state_root: resource_policy::normalized_state_root_key(&self.config.state_root),
3016 measured_weights_bytes,
3017 })
3018 }
3019
3020 async fn reconcile_worker_residency(
3021 offload: &dyn crate::offload::LocalGenerationOffload,
3022 expected_model_id: &str,
3023 residency: &crate::offload::LocalWorkerResidency,
3024 retention: backend_cache::BackendRetention,
3025 reservation: &mut resource_policy::LocalLoadReservation,
3026 ) -> Result<(), InferenceError> {
3027 if residency.model_id != expected_model_id {
3028 if retention == backend_cache::BackendRetention::Resident {
3029 let released = offload
3030 .release_model(&residency.model_id)
3031 .await
3032 .unwrap_or(false);
3033 if !released {
3034 let allocation_id = offload
3035 .resident_allocation_id(&residency.model_id)
3036 .unwrap_or_else(|| {
3037 resource_policy::worker_process_allocation_id(&residency.model_id)
3038 });
3039 reservation.publish_resident_weights_as(
3040 &allocation_id,
3041 residency.measured_weights_bytes,
3042 );
3043 }
3044 }
3045 return Err(InferenceError::InferenceFailed(format!(
3046 "local worker acknowledged model '{}' for requested '{}'",
3047 residency.model_id, expected_model_id
3048 )));
3049 }
3050 if retention == backend_cache::BackendRetention::Resident {
3051 let allocation_id = offload
3052 .resident_allocation_id(expected_model_id)
3053 .unwrap_or_else(|| {
3054 resource_policy::worker_process_allocation_id(expected_model_id)
3055 });
3056 reservation
3057 .publish_resident_weights_as(&allocation_id, residency.measured_weights_bytes);
3058 }
3059 Ok(())
3060 }
3061
3062 fn admit_speech_candidate(
3063 &self,
3064 schema: &ModelSchema,
3065 explicit: bool,
3066 ) -> SpeechCandidateAdmission {
3067 if !schema.is_local()
3068 || matches!(
3069 schema.source,
3070 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. }
3071 )
3072 {
3073 return SpeechCandidateAdmission::Proceed(None);
3074 }
3075 match self.reserve_local_request(schema, 0) {
3076 Ok(reservation) => SpeechCandidateAdmission::Proceed(Some(reservation)),
3077 Err(error) if explicit => SpeechCandidateAdmission::FailBlocked(error),
3078 Err(error) => SpeechCandidateAdmission::SkipBlocked(error),
3079 }
3080 }
3081
3082 fn hold_local_reservation_for_stream(
3083 mut source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
3084 reservation: resource_policy::LocalLoadReservation,
3085 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
3086 let (tx, rx) = tokio::sync::mpsc::channel(64);
3087 tokio::spawn(async move {
3088 let _reservation = reservation;
3089 while let Some(event) = source.recv().await {
3090 if tx.send(event).await.is_err() {
3091 break;
3092 }
3093 }
3094 });
3095 rx
3096 }
3097
3098 fn hold_optional_reservation_for_stream(
3099 source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
3100 reservation: Option<resource_policy::LocalLoadReservation>,
3101 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
3102 match reservation {
3103 Some(reservation) => Self::hold_local_reservation_for_stream(source, reservation),
3104 None => source,
3105 }
3106 }
3107
3108 pub fn set_spend_limits(&self, limits: Option<SpendLimits>) {
3114 *self.spend_limits.write().unwrap() = limits;
3115 }
3116
3117 fn preferred_model_for_capability(&self, capability: ModelCapability) -> Option<&str> {
3118 match capability {
3119 ModelCapability::Generate => self.config.preferred_generation_model.as_deref(),
3120 ModelCapability::Embed => self.config.preferred_embedding_model.as_deref(),
3121 ModelCapability::Classify => self.config.preferred_classification_model.as_deref(),
3122 _ => None,
3123 }
3124 }
3125
3126 fn request_has_tools(req: &GenerateRequest) -> bool {
3132 req.tools.as_ref().is_some_and(|t| !t.is_empty())
3133 }
3134
3135 fn request_needs_vision(req: &GenerateRequest) -> bool {
3136 req.images.as_ref().is_some_and(|images| !images.is_empty())
3137 || req.messages.as_ref().is_some_and(|messages| {
3138 messages
3139 .iter()
3140 .any(|msg| matches!(msg, Message::UserMultimodal { .. }))
3141 })
3142 }
3143
3144 fn request_has_media_codex_cli_leaves_out(req: &GenerateRequest) -> bool {
3150 let sent_by_codex_cli = |block: &ContentBlock| {
3151 matches!(
3152 block,
3153 ContentBlock::Text { .. } | ContentBlock::ImageBase64 { .. }
3154 )
3155 };
3156 req.images
3157 .iter()
3158 .flatten()
3159 .any(|block| !sent_by_codex_cli(block))
3160 || req.messages.iter().flatten().any(|message| match message {
3161 Message::UserMultimodal { content } => {
3162 content.iter().any(|block| !sent_by_codex_cli(block))
3163 }
3164 _ => false,
3165 })
3166 }
3167
3168 fn withhold_codex_cli_rows(
3178 req: &GenerateRequest,
3179 streaming: bool,
3180 registry: &mut UnifiedRegistry,
3181 ) -> bool {
3182 let unservable = if streaming {
3183 Self::request_needs_vision(req)
3184 } else {
3185 Self::request_has_media_codex_cli_leaves_out(req)
3186 };
3187 if !unservable {
3188 return false;
3189 }
3190 let rows: Vec<String> = registry
3191 .all()
3192 .filter(|schema| matches!(schema.source, ModelSource::CodexCli { .. }))
3193 .map(|schema| schema.id.clone())
3194 .collect();
3195 for id in &rows {
3196 registry.unregister(id);
3197 }
3198 !rows.is_empty()
3199 }
3200
3201 fn override_still_listed(withheld: bool, registry: &UnifiedRegistry, model: &str) -> bool {
3205 !withheld
3206 || registry
3207 .get(model)
3208 .or_else(|| registry.find_by_name(model))
3209 .is_some()
3210 }
3211
3212 #[allow(dead_code)] fn request_has_video(req: &GenerateRequest) -> bool {
3218 let images_have_video = req
3219 .images
3220 .as_ref()
3221 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_video));
3222 let messages_have_video = req.messages.as_ref().is_some_and(|messages| {
3223 messages.iter().any(|msg| match msg {
3224 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_video),
3225 _ => false,
3226 })
3227 });
3228 images_have_video || messages_have_video
3229 }
3230
3231 fn vision_stream_intent(
3238 intent: Option<&crate::intent::IntentHint>,
3239 has_vision: bool,
3240 registry: &UnifiedRegistry,
3241 ) -> Option<crate::intent::IntentHint> {
3242 has_vision.then(|| {
3243 let mut hint = intent.cloned().unwrap_or_default();
3244 hint.exclude_models.extend(
3245 registry
3246 .all()
3247 .filter(|schema| schema.is_foundation_models())
3248 .map(|schema| schema.id.clone()),
3249 );
3250 hint
3251 })
3252 }
3253
3254 #[allow(dead_code)] fn request_has_message_images(req: &GenerateRequest) -> bool {
3260 req.messages.as_ref().is_some_and(|messages| {
3261 messages.iter().any(|msg| match msg {
3262 Message::UserMultimodal { content } => content.iter().any(|block| {
3263 matches!(
3264 block,
3265 ContentBlock::ImageBase64 { .. } | ContentBlock::ImageUrl { .. }
3266 )
3267 }),
3268 Message::System { .. }
3269 | Message::User { .. }
3270 | Message::Assistant { .. }
3271 | Message::ToolResult { .. }
3272 | Message::ProviderOutputItems { .. } => false,
3273 })
3274 })
3275 }
3276
3277 #[allow(dead_code)] fn request_has_audio(req: &GenerateRequest) -> bool {
3282 let images_have_audio = req
3283 .images
3284 .as_ref()
3285 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_audio));
3286 let messages_have_audio = req.messages.as_ref().is_some_and(|messages| {
3287 messages.iter().any(|msg| match msg {
3288 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_audio),
3289 _ => false,
3290 })
3291 });
3292 images_have_audio || messages_have_audio
3293 }
3294
3295 pub fn new(config: InferenceConfig) -> Self {
3296 let registry = models::ModelRegistry::new(config.models_dir.clone());
3297 let hw = HardwareInfo::detect();
3298 let policy_evidence =
3299 resource_policy::FileResourcePolicyRepository::new(config.state_root.clone())
3300 .load_with_evidence()
3301 .unwrap_or_else(|error| {
3302 tracing::warn!(%error, "failed to read local model resource policy; using Everyday");
3303 resource_policy::ResourcePolicyLoadEvidence {
3304 policy: resource_policy::ResourcePolicy::everyday(),
3305 source: resource_policy::ResourcePolicyLoadSource::CorruptDefault,
3306 warning: Some(format!(
3307 "The local-model resource policy could not be read ({error}); CAR used Everyday."
3308 )),
3309 }
3310 });
3311 let policy = policy_evidence.policy.clone();
3312 let effective_budget = policy.effective_budget(hw.total_ram_mb);
3313 let local_admission = resource_policy::scoped_local_admission_with_models_dir(
3314 &config.state_root,
3315 policy,
3316 hw.clone(),
3317 Some(config.models_dir.clone()),
3318 );
3319 let runtime_scope = scoped_inference_runtime(
3320 &config.state_root,
3321 effective_budget.configured_model_ceiling_mb,
3322 local_admission.clone(),
3323 );
3324 let router = ModelRouter::new(hw.clone());
3325 let unified_registry = UnifiedRegistry::new_with_state_root(
3326 config.state_root.clone(),
3327 config.models_dir.clone(),
3328 );
3329 let adaptive_router = AdaptiveRouter::with_default_config(hw);
3330 let mut tracker = OutcomeTracker::new();
3331 let profiles_path = config.state_models_dir().join("outcome_profiles.json");
3333 let outcome_profiles_loaded = match tracker.load_from_file(&profiles_path) {
3336 Ok(n) => {
3337 if n > 0 {
3338 tracing::info!(loaded = n, "loaded persisted model profiles");
3339 }
3340 true
3341 }
3342 Err(error) => error.kind() == std::io::ErrorKind::NotFound,
3343 };
3344 let mut benchmark_models_loaded = 0usize;
3345 for path in benchmark_priors_paths(&config.state_models_dir()) {
3346 match routing_ext::load_benchmark_priors(&path) {
3347 Ok(priors) if !priors.is_empty() => {
3348 benchmark_models_loaded += priors.len();
3349 routing_ext::apply_benchmark_priors(&mut tracker, &priors);
3350 tracing::info!(
3351 path = %path.display(),
3352 loaded = priors.len(),
3353 "loaded benchmark quality priors"
3354 );
3355 }
3356 Ok(_) => {}
3357 Err(error) => {
3358 tracing::warn!(path = %path.display(), %error, "failed to load benchmark priors");
3359 }
3360 }
3361 }
3362 if benchmark_models_loaded > 0 {
3363 tracing::info!(
3364 loaded = benchmark_models_loaded,
3365 "applied benchmark priors to cold-start routing"
3366 );
3367 }
3368 let outcome_tracker = Arc::new(RwLock::new(tracker));
3369
3370 let remote_backend = RemoteBackend::new();
3371 let model_management = model_management::ModelManagementStore::new(
3372 config.state_root.clone(),
3373 config.models_dir.clone(),
3374 );
3375
3376 let lane_defaults_path = config
3377 .state_root
3378 .join(crate::lane_defaults::LANE_DEFAULTS_FILE);
3379 let next_action_seq = crate::action_ledger::read_actions(
3380 &config
3381 .state_root
3382 .join(crate::action_ledger::ACTION_LEDGER_FILE),
3383 0,
3384 )
3385 .iter()
3386 .map(|a| a.seq)
3387 .max()
3388 .map(|m| m + 1)
3389 .unwrap_or(1);
3390 Self {
3391 config,
3392 unified: std::sync::RwLock::new(Arc::new(unified_registry)),
3393 registry_write: std::sync::Mutex::new(()),
3394 adaptive_router,
3395 outcome_tracker,
3396 last_outcome_flush: Arc::new(std::sync::Mutex::new(None)),
3397 outcome_profiles_loaded,
3398 ledger_io_lock: Arc::new(tokio::sync::Mutex::new(())),
3399 spend_limits: Arc::new(std::sync::RwLock::new(None)),
3400 lane_defaults_cache: Arc::new(std::sync::RwLock::new(crate::lane_defaults::load_from(
3401 &lane_defaults_path,
3402 ))),
3403 concierge_action_lock: Arc::new(tokio::sync::Mutex::new(())),
3404 external_pins: Default::default(),
3405 disk_pressured: Default::default(),
3406 failed_upgrades: Default::default(),
3407 concierge_action_seq: Arc::new(std::sync::atomic::AtomicU64::new(next_action_seq)),
3408 remote_backend,
3409 model_management,
3410 local_admission: local_admission.clone(),
3411 _runtime_scope: runtime_scope.clone(),
3412 resource_policy_generation: runtime_scope.resource_policy_generation.clone(),
3413 resource_policy_evidence: Arc::new(std::sync::RwLock::new(policy_evidence)),
3414 model_budget: runtime_scope.model_budget.clone(),
3415 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3416 text_backends: runtime_scope.text_backends.clone(),
3417 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3418 local_backends: runtime_scope.local_backends.clone(),
3419 registry,
3420 router,
3421 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3422 backend: runtime_scope.backend.clone(),
3423 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3424 embedding_backend: runtime_scope.embedding_backend.clone(),
3425 speech_runtime: runtime_scope.speech_runtime.clone(),
3426 speech_policy: SpeechPolicy {
3427 prefer_local: cfg!(all(
3428 target_os = "macos",
3429 target_arch = "aarch64",
3430 not(car_skip_mlx)
3431 )),
3432 allow_remote_fallback: true,
3433 preferred_local_stt: None,
3434 preferred_local_tts: None,
3435 preferred_remote_stt: None,
3436 preferred_remote_tts: None,
3437 },
3438 vllm_pool: runtime_scope.vllm_pool.clone(),
3439 }
3440 }
3441
3442 pub async fn init_key_pool(&self) {
3445 for schema in self.unified_registry().list() {
3447 if schema.is_remote() {
3448 self.remote_backend.register_model_keys(schema).await;
3449 }
3450 }
3451
3452 let stats_path = self.config.state_models_dir().join("key_pool_stats.json");
3454 if let Ok(n) = self.remote_backend.key_pool.load_stats(&stats_path).await {
3455 if n > 0 {
3456 tracing::info!(loaded = n, "loaded persisted key pool stats");
3457 }
3458 }
3459
3460 let total = self.remote_backend.key_pool.total_keys().await;
3461 if total > 0 {
3462 tracing::info!(keys = total, "key pool initialized");
3463 }
3464 }
3465
3466 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3469 async fn ensure_backend(
3470 &self,
3471 schema: &ModelSchema,
3472 reservation: &mut resource_policy::LocalLoadReservation,
3473 ) -> Result<(), InferenceError> {
3474 let read = self.backend.read().await;
3475 if read.contains_key(&schema.id) {
3476 return Ok(());
3477 }
3478 drop(read);
3479
3480 let mut write = self.backend.write().await;
3481 if write.contains_key(&schema.id) {
3482 return Ok(());
3483 }
3484
3485 let model_path = self.registry.ensure_model(&schema.name).await?;
3486 let mut measured = backend_cache::estimate_model_size(&model_path);
3487 reservation
3488 .reconcile_measured_weights(measured)
3489 .map_err(InferenceError::from)?;
3490 let device = self.config.device.unwrap_or_else(Device::auto);
3491 let backend = match CandleBackend::load(&model_path, device) {
3493 Ok(b) => b,
3494 Err(load_err) => {
3495 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3499 return Err(load_err);
3500 }
3501 tracing::warn!(
3502 model = %schema.id,
3503 error = %load_err,
3504 "candle backend load failed; purged corrupt cache files and re-pulling once"
3505 );
3506 let model_path = self.registry.ensure_model(&schema.name).await?;
3507 measured = backend_cache::estimate_model_size(&model_path);
3508 reservation
3509 .reconcile_measured_weights(measured)
3510 .map_err(InferenceError::from)?;
3511 CandleBackend::load(&model_path, device)?
3512 }
3513 };
3514 write.insert(schema.id.clone(), backend);
3515 reservation.publish_resident_weights(measured);
3516 Ok(())
3517 }
3518
3519 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3522 async fn ensure_embedding_backend(
3523 &self,
3524 reservation: &mut resource_policy::LocalLoadReservation,
3525 ) -> Result<(), InferenceError> {
3526 let read = self.embedding_backend.read().await;
3527 if read.is_some() {
3528 return Ok(());
3529 }
3530 drop(read);
3531
3532 let mut write = self.embedding_backend.write().await;
3533 if write.is_some() {
3534 return Ok(());
3535 }
3536
3537 let embedding_model = self
3538 .preferred_model_for_capability(ModelCapability::Embed)
3539 .unwrap_or(&self.config.embedding_model);
3540 let model_path = self.registry.ensure_model(embedding_model).await?;
3541 let mut measured = backend_cache::estimate_model_size(&model_path);
3542 reservation
3543 .reconcile_measured_weights(measured)
3544 .map_err(InferenceError::from)?;
3545 let device = self.config.device.unwrap_or_else(Device::auto);
3546 let backend = match EmbeddingBackend::load(&model_path, device) {
3548 Ok(b) => b,
3549 Err(load_err) => {
3550 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3551 return Err(load_err);
3552 }
3553 tracing::warn!(
3554 model = embedding_model,
3555 error = %load_err,
3556 "embedding backend load failed; purged corrupt cache files and re-pulling once"
3557 );
3558 let model_path = self.registry.ensure_model(embedding_model).await?;
3559 measured = backend_cache::estimate_model_size(&model_path);
3560 reservation
3561 .reconcile_measured_weights(measured)
3562 .map_err(InferenceError::from)?;
3563 EmbeddingBackend::load(&model_path, device)?
3564 }
3565 };
3566 *write = Some(backend);
3567 reservation.publish_resident_weights(measured);
3568 Ok(())
3569 }
3570
3571 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3574 async fn ensure_mlx_embedding_backend(&self) -> Result<String, InferenceError> {
3575 let embedding_model_name = self
3576 .preferred_model_for_capability(ModelCapability::Embed)
3577 .unwrap_or(&self.config.embedding_model)
3578 .to_string();
3579 let schema = self
3580 .resolve_schema(&embedding_model_name)
3581 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model_name.clone()))?
3582 .clone();
3583 Ok(schema.id)
3584 }
3585
3586 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3602 async fn load_backend_healing<T, F, RF, R>(
3603 schema_id: &str,
3604 model_dir: std::path::PathBuf,
3605 cache: &backend_cache::BackendCache<T>,
3606 size: u64,
3607 reservation: &mut resource_policy::LocalLoadReservation,
3608 loader: F,
3609 repull: RF,
3610 ) -> Result<
3611 (
3612 backend_cache::CachedBackend<T>,
3613 backend_cache::BackendRetention,
3614 ),
3615 InferenceError,
3616 >
3617 where
3618 T: Send + 'static,
3619 F: Fn(&Path) -> Result<T, InferenceError>,
3620 RF: FnOnce() -> R,
3621 R: std::future::Future<Output = Result<std::path::PathBuf, InferenceError>>,
3622 {
3623 match cache.get_or_load_admitted(schema_id, size, reservation, || loader(&model_dir)) {
3624 Ok(admitted) => Ok(admitted),
3625 Err(load_err) => {
3626 let purged = crate::download::purge_corrupt_cache_files(&model_dir);
3634 if purged == 0 {
3635 return Err(load_err);
3637 }
3638 tracing::warn!(
3639 model = schema_id,
3640 purged,
3641 error = %load_err,
3642 "backend load failed; purged corrupt cache files and re-pulling once"
3643 );
3644 let fresh_dir = repull().await?;
3645 let fresh_size = backend_cache::estimate_model_size(&fresh_dir);
3646 cache
3647 .get_or_load_admitted(schema_id, fresh_size, reservation, || loader(&fresh_dir))
3648 }
3649 }
3650 }
3651
3652 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3656 async fn ensure_text_backend(
3657 &self,
3658 schema: &ModelSchema,
3659 reservation: &mut resource_policy::LocalLoadReservation,
3660 ) -> Result<
3661 (
3662 backend_cache::CachedBackend<backend::SwiftLmBackend>,
3663 backend_cache::BackendRetention,
3664 ),
3665 InferenceError,
3666 > {
3667 if !Self::supports_native_mlx(schema) {
3668 return Err(InferenceError::InferenceFailed(format!(
3669 "native MLX backend does not support {} ({}) yet; use vLLM-MLX or add a family-specific MLX backend",
3670 schema.name, schema.family
3671 )));
3672 }
3673
3674 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3675 let size = backend_cache::estimate_model_size(&model_dir);
3676 if !reservation.authorizes_model(&schema.id) {
3677 return Err(InferenceError::InferenceFailed(format!(
3678 "local admission reservation does not authorize {}",
3679 schema.id
3680 )));
3681 }
3682 let role = if schema.has_capability(ModelCapability::Embed)
3692 && !schema.has_capability(ModelCapability::Generate)
3693 {
3694 backend::swift_lm::BackendRole::Embedding
3695 } else {
3696 backend::swift_lm::BackendRole::Text
3697 };
3698 let loader = move |dir: &Path| {
3699 let dir = dir.to_path_buf();
3700 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3701 backend::SwiftLmBackend::load_with_role(&dir, role)
3703 }))
3704 .map_err(|e| {
3705 InferenceError::InferenceFailed(format!(
3706 "MLX backend loading panicked (possible Metal/accelerate exception): {:?}",
3707 e
3708 ))
3709 })?
3710 };
3711 let registry = self.unified_registry();
3712 Self::load_backend_healing(
3713 &schema.id,
3714 model_dir,
3715 &self.text_backends,
3716 size,
3717 reservation,
3718 loader,
3719 || registry.redownload_local(&schema.id),
3720 )
3721 .await
3722 }
3723
3724 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3729 async fn ensure_local_backend(
3730 &self,
3731 schema: &ModelSchema,
3732 reservation: &mut resource_policy::LocalLoadReservation,
3733 ) -> Result<
3734 (
3735 backend_cache::CachedBackend<Box<dyn backend::local::LocalInferenceBackend>>,
3736 backend_cache::BackendRetention,
3737 ),
3738 InferenceError,
3739 > {
3740 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3741 let size = backend_cache::estimate_model_size(&model_dir);
3742 if !reservation.authorizes_model(&schema.id) {
3743 return Err(InferenceError::InferenceFailed(format!(
3744 "local admission reservation does not authorize {}",
3745 schema.id
3746 )));
3747 }
3748 let loader = |dir: &Path| {
3749 let dir = dir.to_path_buf();
3750 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3751 backend::SwiftLmBackend::load(&dir)
3761 .map(|b| Box::new(b) as Box<dyn backend::local::LocalInferenceBackend>)
3762 .map_err(|e| {
3763 InferenceError::InferenceFailed(format!(
3764 "mlx-swift-lm could not load model_type '{}' ({}): {e}",
3765 backend::local::describe_model_type(&dir),
3766 dir.display()
3767 ))
3768 })
3769 }))
3770 .map_err(|e| {
3771 InferenceError::InferenceFailed(format!(
3772 "local backend loading panicked (possible Metal/accelerate exception): {:?}",
3773 e
3774 ))
3775 })?
3776 };
3777 let registry = self.unified_registry();
3778 Self::load_backend_healing(
3779 &schema.id,
3780 model_dir,
3781 &self.local_backends,
3782 size,
3783 reservation,
3784 loader,
3785 || registry.redownload_local(&schema.id),
3786 )
3787 .await
3788 }
3789
3790 pub async fn reset_local_kv_cache(&self, model_id: &str) {
3801 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3805 {
3806 let Some(schema) = self.unified_registry().get(model_id).cloned() else {
3807 return;
3808 };
3809 if !matches!(
3813 schema.source,
3814 ModelSource::Mlx { .. } | ModelSource::Local { .. }
3815 ) {
3816 return;
3817 }
3818 if !self.local_backends.contains(&schema.id) {
3821 return;
3822 }
3823 let Ok(mut reservation) = self.reserve_local_request(&schema, 0) else {
3824 return;
3825 };
3826 if let Ok((handle, _retention)) =
3827 self.ensure_local_backend(&schema, &mut reservation).await
3828 {
3829 if let Ok(mut guard) = handle.lock() {
3830 guard.clear_kv_cache();
3831 }
3832 }
3833 }
3834 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3835 let _ = model_id;
3836 }
3837
3838 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3845 pub async fn warm_up<S: AsRef<str>>(
3846 &self,
3847 schema_ids: &[S],
3848 ) -> Vec<Result<(), InferenceError>> {
3849 let mut results = Vec::with_capacity(schema_ids.len());
3850 for id in schema_ids {
3851 let id = id.as_ref();
3852 let outcome: Result<(), InferenceError> = async {
3853 let schema = self.unified_registry().get(id).cloned().ok_or_else(|| {
3854 InferenceError::InferenceFailed(format!("warm_up: unknown schema id {id}"))
3855 })?;
3856 let mut reservation = self.reserve_local_request(&schema, 0)?;
3858 match schema.capabilities.first().copied() {
3859 Some(ModelCapability::ImageGeneration) => {
3860 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3865 }
3866 Some(ModelCapability::VideoGeneration) => {
3867 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3872 }
3873 Some(ModelCapability::TextToSpeech) => {
3874 }
3878 _ => {
3879 let _ = self.ensure_text_backend(&schema, &mut reservation).await?;
3880 }
3881 }
3882 Ok(())
3883 }
3884 .await;
3885 results.push(outcome);
3886 }
3887 results
3888 }
3889
3890 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3892 pub async fn warm_up<S: AsRef<str>>(
3893 &self,
3894 _schema_ids: &[S],
3895 ) -> Vec<Result<(), InferenceError>> {
3896 Vec::new()
3897 }
3898
3899 async fn vllm_live_schema(
3905 &self,
3906 schema: ModelSchema,
3907 reservation: Option<resource_policy::LocalLoadReservation>,
3908 context_tokens: usize,
3909 ) -> Result<(ModelSchema, Option<resource_policy::LocalLoadReservation>), InferenceError> {
3910 match &schema.source {
3911 ModelSource::ManagedVllmMlx { .. } => {}
3912 ModelSource::VllmMlx { .. } => return Ok((schema, None)),
3913 _ => return Ok((schema, None)),
3914 }
3915 let _dispatch = self.vllm_pool.acquire_dispatch(&schema.id).await;
3920 self.vllm_pool
3921 .wait_for_teardown(&schema.id, std::time::Duration::from_secs(5))
3922 .await
3923 .map_err(InferenceError::InferenceFailed)?;
3924 self.vllm_pool
3929 .reap_dead(&schema.id)
3930 .await
3931 .map_err(InferenceError::InferenceFailed)?;
3932 let mut reservation = match reservation {
3933 Some(reservation) if reservation.model_id() == schema.id => reservation,
3934 _ => self.reserve_local_request(&schema, context_tokens)?,
3935 };
3936 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3937 let model_name = model_dir.display().to_string();
3938 let measured = backend_cache::estimate_model_size(&model_dir)
3939 .max(reservation.reconciled_weights_bytes());
3940 reservation
3941 .reconcile_measured_weights(measured)
3942 .map_err(InferenceError::from)?;
3943 let endpoint = self
3945 .vllm_pool
3946 .ensure(&schema.id, &model_name, &reservation, &schema.family)
3949 .await
3950 .map_err(InferenceError::InferenceFailed)?;
3951 let allocation_id = resource_policy::vllm_process_allocation_id(&schema.id);
3952 reservation.publish_resident_weights_as(&allocation_id, measured);
3953 let mut schema = schema;
3954 schema.source = ModelSource::VllmMlx {
3955 endpoint,
3956 model_name,
3957 };
3958 Ok((schema, Some(reservation)))
3959 }
3960
3961 pub async fn evict_idle_vllm_servers(&self) -> usize {
3964 self.vllm_pool.evict_idle().await
3965 }
3966
3967 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3978 pub fn evict_idle_backends(&self) -> (usize, u64) {
3979 let mut entries = 0usize;
3980 let mut bytes = 0u64;
3981 for (n, b) in [
3982 self.text_backends.evict_idle(),
3983 self.local_backends.evict_idle(),
3984 ] {
3985 entries += n;
3986 bytes = bytes.saturating_add(b);
3987 }
3988 (entries, bytes)
3989 }
3990
3991 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3993 pub fn evict_idle_backends(&self) -> (usize, u64) {
3994 (0, 0)
3995 }
3996
3997 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3998 fn supports_native_mlx(schema: &ModelSchema) -> bool {
3999 matches!(schema.family.as_str(), "qwen3" | "qwen2.5-vl" | "qwen2-vl")
4000 }
4001
4002 fn resolve_schema(&self, name: &str) -> Option<ModelSchema> {
4006 let registry = self.unified_registry();
4007 registry
4008 .get(name)
4009 .or_else(|| registry.find_by_name(name))
4010 .cloned()
4011 }
4012
4013 fn resolve_schema_name_first(&self, name: &str) -> Option<ModelSchema> {
4016 let registry = self.unified_registry();
4017 registry
4018 .find_by_name(name)
4019 .or_else(|| registry.get(name))
4020 .cloned()
4021 }
4022
4023 pub fn unified_registry(&self) -> Arc<UnifiedRegistry> {
4028 let me = self as *const Self as usize;
4029 PINNED_REGISTRY
4030 .try_with(|(engine, pinned)| (*engine == me).then(|| pinned.clone()))
4031 .ok()
4032 .flatten()
4033 .unwrap_or_else(|| self.current_registry())
4034 }
4035
4036 fn current_registry(&self) -> Arc<UnifiedRegistry> {
4038 self.unified
4039 .read()
4040 .unwrap_or_else(std::sync::PoisonError::into_inner)
4041 .clone()
4042 }
4043
4044 async fn with_pinned_registry<F: std::future::Future>(&self, operation: F) -> F::Output {
4053 let me = self as *const Self as usize;
4054 let pinned_here = PINNED_REGISTRY
4055 .try_with(|(engine, _)| *engine == me)
4056 .unwrap_or(false);
4057 if pinned_here {
4058 operation.await
4059 } else {
4060 PINNED_REGISTRY
4061 .scope((me, self.current_registry()), operation)
4062 .await
4063 }
4064 }
4065
4066 pub fn unified_registry_mut(&mut self) -> &mut UnifiedRegistry {
4070 Arc::make_mut(
4071 self.unified
4072 .get_mut()
4073 .unwrap_or_else(std::sync::PoisonError::into_inner),
4074 )
4075 }
4076
4077 fn update_registry<R>(&self, edit: impl FnOnce(&mut UnifiedRegistry) -> R) -> R {
4083 let _writer = self
4084 .registry_write
4085 .lock()
4086 .unwrap_or_else(std::sync::PoisonError::into_inner);
4087 let mut next = (*self.current_registry()).clone();
4088 let out = edit(&mut next);
4089 *self
4090 .unified
4091 .write()
4092 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(next);
4093 out
4094 }
4095
4096 pub fn swap_signed_catalog(&self, catalog: catalog::VerifiedCatalog) -> usize {
4105 let origin = Arc::new(catalog.envelope());
4106 let revoked: std::collections::HashSet<String> =
4107 catalog.revoked().iter().cloned().collect();
4108 self.update_registry(|registry| {
4109 registry.refresh_availability();
4112 let mut keep: std::collections::HashSet<String> =
4114 self.protected_models(registry, true).into_keys().collect();
4115 keep.extend(
4116 registry
4117 .list()
4118 .into_iter()
4119 .filter(|m| {
4120 (m.is_local() && m.downloads_weights() && m.weights_ready)
4123 || matches!(self.model_management.load_receipt(&m.id), Ok(Some(_)))
4124 })
4125 .map(|m| m.id.clone()),
4126 );
4127 let loaded = registry.replace_signed_catalog(
4128 catalog.into_models(),
4129 Some(origin),
4130 &keep,
4131 &revoked,
4132 );
4133 registry.refresh_availability();
4134 self.persist_retained(registry);
4135 loaded
4136 })
4137 }
4138
4139 fn persist_retained(&self, registry: &UnifiedRegistry) {
4142 let path = catalog::retained_path(&self.config.state_root);
4143 if let Err(error) = catalog::save_retained(&path, ®istry.retained_rows()) {
4144 tracing::warn!(path = %path.display(), %error, "could not record retained catalog rows");
4145 }
4146 }
4147
4148 fn keep_pulled_row_if_withdrawn(&self, name: &str) {
4152 let Some(schema) = self.resolve_schema_name_first(name) else {
4153 return;
4154 };
4155 if self.current_registry().get(&schema.id).is_some() {
4156 return;
4157 }
4158 let Some(origin) = self.unified_registry().signed_origin_of(&schema.id) else {
4159 return;
4160 };
4161 self.update_registry(|registry| {
4162 registry.restore_retained(vec![(schema, origin)]);
4163 registry.refresh_availability();
4164 self.persist_retained(registry);
4165 });
4166 }
4167
4168 fn catalog_registry_snapshot(&self) -> UnifiedRegistry {
4169 self.catalog_registry_snapshot_with(false)
4170 }
4171
4172 fn catalog_registry_snapshot_with(&self, authoritative_credentials: bool) -> UnifiedRegistry {
4181 let mut registry = (*self.unified_registry()).clone();
4182 registry.prune_missing_on_disk_models();
4183 if authoritative_credentials {
4188 registry.refresh_availability_for_explicit_status();
4189 } else {
4190 registry.refresh_availability();
4191 }
4192 self.filter_disabled_local_models(&mut registry);
4193 registry
4194 }
4195
4196 fn filter_disabled_local_models(&self, registry: &mut UnifiedRegistry) {
4197 let disabled = registry
4198 .all()
4199 .filter(|schema| {
4200 schema.downloads_weights()
4201 && !self
4202 .model_management
4203 .car_enabled(&schema.id)
4204 .unwrap_or(false)
4205 })
4206 .map(|schema| schema.id.clone())
4207 .collect::<Vec<_>>();
4208 for model_id in disabled {
4209 registry.unregister(&model_id);
4210 }
4211 }
4212
4213 async fn routing_registry_snapshot_with_credential_failure(
4214 &self,
4215 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
4216 let parslee = car_auth::resolve_credential(car_auth::CredentialReadMode::Use).await;
4222 let (parslee_api_base, parslee_signed_out, credential_failure) = match &parslee {
4223 Ok(Some(credential)) => (Some(credential.api_base.as_str()), false, None),
4224 Ok(None) => (None, true, Some(parslee_signed_out_route_failure())),
4225 Err(error) => {
4226 let source_error = error.to_string();
4227 (
4228 None,
4229 false,
4230 Some(RouteCredentialFailure {
4231 summary: format!(
4232 "Parslee {AUTH_STORE_UNREADABLE_MARKER} — unlock or grant access to the credential store, then retry"
4233 ),
4234 source_error,
4235 }),
4236 )
4237 }
4238 };
4239 let mut registry = (*self.unified_registry()).clone();
4240 registry.prune_missing_on_disk_models();
4241 registry.refresh_routing_availability(parslee_api_base, parslee_signed_out);
4242 self.filter_disabled_local_models(&mut registry);
4243 (registry, credential_failure)
4244 }
4245
4246 async fn work_context_routing_snapshot(
4247 &self,
4248 context: &car_auth::context::CredentialContext,
4249 requested_model: Option<&str>,
4250 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
4251 let credential = if requested_model.is_some() {
4254 None
4255 } else {
4256 Some(car_auth::context::resolve_context_credential(context).await)
4257 };
4258 let available = context.organization_id.is_some()
4259 && credential.as_ref().is_none_or(|result| result.is_ok());
4260 let failure =
4261 credential
4262 .and_then(|result| result.err())
4263 .map(|error| RouteCredentialFailure {
4264 summary: format!("Work-context credentials unavailable: {error}"),
4265 source_error: error.to_string(),
4266 });
4267 let mut registry = (*self.unified_registry()).clone();
4268 registry.prune_missing_on_disk_models();
4269 registry.refresh_work_context_availability(&context.api_base, available);
4270 self.filter_disabled_local_models(&mut registry);
4271 (registry, failure)
4272 }
4273
4274 async fn routing_registry_snapshot(&self) -> UnifiedRegistry {
4275 self.routing_registry_snapshot_with_credential_failure()
4276 .await
4277 .0
4278 }
4279
4280 async fn request_routing_registry_snapshot(
4285 &self,
4286 requested_model: Option<&str>,
4287 ) -> UnifiedRegistry {
4288 if requested_model.is_some() {
4289 self.catalog_registry_snapshot()
4290 } else {
4291 self.routing_registry_snapshot().await
4292 }
4293 }
4294
4295 pub async fn route_adaptive(&self, prompt: &str) -> AdaptiveRoutingDecision {
4297 self.route_adaptive_with_intent(prompt, None).await
4298 }
4299
4300 pub async fn route_adaptive_with_intent(
4310 &self,
4311 prompt: &str,
4312 intent: Option<crate::intent::IntentHint>,
4313 ) -> AdaptiveRoutingDecision {
4314 if let Some(model) = self.preferred_model_for_capability(ModelCapability::Generate) {
4315 let exclude_set = self
4316 .adaptive_router
4317 .build_exclude_set(intent.as_ref(), &self.unified_registry());
4318 let overridden_is_excluded =
4319 Self::model_is_excluded(&exclude_set, &self.unified_registry(), model);
4320 if !overridden_is_excluded {
4321 let ctx_len = self
4327 .resolve_schema(model)
4328 .map(|s| s.context_length)
4329 .unwrap_or(0);
4330 return AdaptiveRoutingDecision {
4331 model_id: model.to_string(),
4332 model_name: model.to_string(),
4333 task: InferenceTask::Generate,
4334 complexity: TaskComplexity::assess(prompt),
4335 reason: "preferred generation model override".into(),
4336 strategy: RoutingStrategy::Explicit,
4337 predicted_quality: 0.5,
4338 fallbacks: vec![],
4339 context_length: ctx_len,
4340 needs_compaction: false,
4341 candidates: vec![],
4342 };
4343 }
4344 }
4345 let routing_registry = self.routing_registry_snapshot().await;
4346 let tracker = self.outcome_tracker.read().await;
4347 match intent {
4348 Some(hint) => self
4349 .adaptive_router
4350 .route_with(crate::adaptive_router::RouteRequest {
4351 intent: Some(&hint),
4352 ..crate::adaptive_router::RouteRequest::new(prompt, &routing_registry, &tracker)
4353 }),
4354 None => self
4355 .adaptive_router
4356 .route(prompt, &routing_registry, &tracker),
4357 }
4358 }
4359
4360 pub async fn route_adaptive_with_work_context(
4363 &self,
4364 prompt: &str,
4365 intent: Option<crate::intent::IntentHint>,
4366 context: &car_auth::context::CredentialContext,
4367 ) -> Result<AdaptiveRoutingDecision, String> {
4368 context.validate()?;
4369 let (registry, failure) = self.work_context_routing_snapshot(context, None).await;
4370 if let Some(failure) = failure {
4371 return Err(failure.summary);
4372 }
4373 let tracker = self.outcome_tracker.read().await;
4374 Ok(self
4375 .adaptive_router
4376 .route_with(crate::adaptive_router::RouteRequest {
4377 intent: intent.as_ref(),
4378 ..crate::adaptive_router::RouteRequest::new(prompt, ®istry, &tracker)
4379 }))
4380 }
4381
4382 pub fn route(&self, prompt: &str) -> RoutingDecision {
4384 self.router.route_generate(prompt, &self.registry)
4385 }
4386
4387 pub fn estimated_tokens(
4398 &self,
4399 req: &GenerateRequest,
4400 model_id: Option<&str>,
4401 ) -> (usize, usize, bool) {
4402 let prompt_tokens = remote::estimate_tokens(&req.prompt);
4403 let context_tokens = req
4404 .context
4405 .as_ref()
4406 .map(|c| remote::estimate_tokens(c))
4407 .unwrap_or(0);
4408 let tools_tokens = req
4409 .tools
4410 .as_ref()
4411 .map(|t| remote::estimate_tokens(&serde_json::to_string(t).unwrap_or_default()))
4412 .unwrap_or(0);
4413 let media_tokens = media_tokens::request_media_and_history_tokens(
4414 req.images.as_deref(),
4415 req.messages.as_deref(),
4416 );
4417 let total_input = prompt_tokens + context_tokens + tools_tokens + media_tokens;
4418
4419 let context_window = match model_id {
4428 Some(id) => {
4429 let routing_registry = self.catalog_registry_snapshot();
4430 routing_registry
4431 .get(id)
4432 .or_else(|| routing_registry.find_by_name(id))
4433 .map(|s| s.context_length)
4434 .unwrap_or(0)
4435 }
4436 None => 0,
4437 };
4438
4439 let fits = context_window == 0 || (total_input + req.params.max_tokens) <= context_window;
4440 (total_input, context_window, fits)
4441 }
4442
4443 fn routing_cache_estimates(req: &GenerateRequest, estimated_input: usize) -> (usize, usize) {
4449 let read = req
4450 .params
4451 .estimated_cache_read_input_tokens
4452 .min(estimated_input);
4453 let write = req
4454 .params
4455 .estimated_cache_write_input_tokens
4456 .min(estimated_input.saturating_sub(read));
4457 (read, write)
4458 }
4459
4460 pub fn model_context_window(&self, model_id: &str) -> usize {
4466 let routing_registry = self.catalog_registry_snapshot();
4467 routing_registry
4468 .get(model_id)
4469 .or_else(|| routing_registry.find_by_name(model_id))
4470 .map(|s| s.context_length)
4471 .unwrap_or(0)
4472 }
4473
4474 pub fn reasoning_output_allowance(
4478 &self,
4479 model_id: &str,
4480 answer_tokens: usize,
4481 budget_tokens: usize,
4482 ) -> usize {
4483 let routing_registry = self.catalog_registry_snapshot();
4484 let schema = routing_registry
4485 .get(model_id)
4486 .or_else(|| routing_registry.find_by_name(model_id));
4487 reasoning_output_allowance(schema, answer_tokens, budget_tokens)
4488 }
4489
4490 pub async fn generate_tracked(
4505 &self,
4506 req: GenerateRequest,
4507 ) -> Result<InferenceResult, InferenceError> {
4508 let mut ignore_retry = |_: InferenceRetryProgress| {};
4509 self.generate_tracked_with_retry_observer(req, &mut ignore_retry)
4510 .await
4511 }
4512
4513 pub async fn generate_tracked_with_retry_observer(
4517 &self,
4518 req: GenerateRequest,
4519 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4520 ) -> Result<InferenceResult, InferenceError> {
4521 let mut ignore_exclusion = |_: RouterExclusion| {};
4522 self.generate_tracked_with_observers(req, retry_observer, &mut ignore_exclusion)
4523 .await
4524 }
4525
4526 pub async fn generate_tracked_with_observers(
4530 &self,
4531 req: GenerateRequest,
4532 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4533 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4534 ) -> Result<InferenceResult, InferenceError> {
4535 self.with_pinned_registry(self.generate_tracked_with_observers_unpinned(
4537 req,
4538 retry_observer,
4539 exclusion_observer,
4540 ))
4541 .await
4542 }
4543
4544 async fn generate_tracked_with_observers_unpinned(
4545 &self,
4546 req: GenerateRequest,
4547 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4548 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4549 ) -> Result<InferenceResult, InferenceError> {
4550 crate::offload::ensure_not_controlled_terminated()?;
4551 let catalog_snapshot = self
4552 .catalog_snapshot()
4553 .map_err(InferenceError::InferenceFailed)?;
4554 let recover = matches!(req.params.thinking, ThinkingMode::Auto);
4555 let mut result = self
4556 .generate_tracked_inner(
4557 req.clone(),
4558 &catalog_snapshot,
4559 retry_observer,
4560 exclusion_observer,
4561 )
4562 .await?;
4563
4564 let action = classify_empty_pass(
4565 recover,
4566 result.stop_reason.as_deref(),
4567 &result.text,
4568 result.tool_calls.is_empty(),
4569 );
4570 let hit_decode_ceiling = action == EmptyPassAction::FailDecodeCeiling;
4571
4572 if action == EmptyPassAction::RetryWithoutThinking {
4573 result.stop_reason = Some("thinking_truncated".to_string());
4574 let mut retry = req;
4575 retry.params.thinking = ThinkingMode::Off;
4576 crate::offload::ensure_not_controlled_terminated()?;
4577 match self
4578 .generate_tracked_inner(
4579 retry,
4580 &catalog_snapshot,
4581 retry_observer,
4582 exclusion_observer,
4583 )
4584 .await
4585 {
4586 Ok(mut recovered) => {
4587 if !recovered.text.trim().is_empty() || !recovered.tool_calls.is_empty() {
4588 recovered.stop_reason = Some("thinking_recovered".to_string());
4589 return Ok(recovered);
4590 }
4591 }
4592 Err(error @ InferenceError::ControlledTermination) => return Err(error),
4593 Err(_) => {}
4594 }
4595 }
4596
4597 if hit_decode_ceiling && result.text.trim().is_empty() && result.tool_calls.is_empty() {
4603 return Err(InferenceError::InferenceFailed(format!(
4604 "local generation hit its {}s wall-clock ceiling before producing any output. \
4605 The `local prefill starting` / `local decode in progress` log lines show where \
4606 the time went — a large prompt can spend most of it on prefill. Try a smaller \
4607 model, shorten the prompt, check for another process contending for the GPU, \
4608 or raise the ceiling with CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables it).",
4609 local_decode_timeout().map_or(0, |t| t.as_secs())
4610 )));
4611 }
4612
4613 Ok(result)
4614 }
4615
4616 #[instrument(
4631 name = "inference.generate",
4632 skip_all,
4633 fields(
4634 model = tracing::field::Empty,
4635 max_tokens = req.params.max_tokens,
4636 prompt_tokens = tracing::field::Empty,
4637 completion_tokens = tracing::field::Empty,
4638 latency_ms = tracing::field::Empty,
4639 ttft_ms = tracing::field::Empty,
4643 gen_ai.operation.name = "chat",
4644 gen_ai.request.model = tracing::field::Empty,
4645 gen_ai.response.model = tracing::field::Empty,
4646 gen_ai.request.max_tokens = req.params.max_tokens,
4647 gen_ai.request.temperature = req.params.temperature,
4648 gen_ai.usage.input_tokens = tracing::field::Empty,
4649 gen_ai.usage.output_tokens = tracing::field::Empty,
4650 )
4651 )]
4652 async fn generate_tracked_inner(
4653 &self,
4654 mut req: GenerateRequest,
4655 catalog_snapshot: &CatalogSnapshot,
4656 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4657 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4658 ) -> Result<InferenceResult, InferenceError> {
4659 self.refuse_system_one_generate(&req)?;
4660 crate::offload::ensure_not_controlled_terminated()?;
4661 validate_expected_catalog_revision(&req, catalog_snapshot)?;
4662 if let Some(requested) = req.model.as_deref() {
4666 tracing::Span::current().record("gen_ai.request.model", requested);
4667 }
4668 let requested_model_id = exact_pinned_model_id(&req).map(str::to_string);
4669 if let Some(model_id) = requested_model_id.as_ref() {
4670 req.model = Some(model_id.clone());
4671 req.params.strict_model = true;
4672 }
4673 let start = Instant::now();
4674 let has_requested_route = req.model.is_some();
4675 let (mut routing_registry, initial_route_credential_failure) =
4676 if let Some(context) = &req.work_context {
4677 self.work_context_routing_snapshot(context, req.model.as_deref())
4678 .await
4679 } else if has_requested_route {
4680 (self.catalog_registry_snapshot(), None)
4681 } else {
4682 self.routing_registry_snapshot_with_credential_failure()
4683 .await
4684 };
4685 if let Some(requested) = requested_model_id.as_deref() {
4686 if routing_registry.get(requested).is_none() {
4687 return Err(InferenceError::ModelNotFound(requested.to_string()));
4688 }
4689 } else if let Some(requested) = req.model.as_deref() {
4690 if routing_registry
4691 .get(requested)
4692 .or_else(|| routing_registry.find_by_name(requested))
4693 .is_none()
4694 {
4695 return Err(InferenceError::ModelNotFound(requested.to_string()));
4696 }
4697 }
4698
4699 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
4701 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
4709 let (estimated_cache_read, estimated_cache_write) =
4710 Self::routing_cache_estimates(&req, estimated_input);
4711 let codex_cli_withheld = req.model.is_none()
4714 && Self::withhold_codex_cli_rows(&req, false, &mut routing_registry);
4715 let tracker_read = self.outcome_tracker.read().await;
4716 let has_tools = Self::request_has_tools(&req);
4717 let has_vision = Self::request_needs_vision(&req);
4718 let preferred_model = self
4719 .preferred_model_for_capability(ModelCapability::Generate)
4720 .map(str::to_string);
4721 let exclude_set = self
4722 .adaptive_router
4723 .build_exclude_set(req.intent.as_ref(), &routing_registry);
4724 let unpinned_override = self
4725 .lane_pin_for(&req, &routing_registry)
4726 .or(preferred_model)
4727 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model))
4728 .filter(|model| {
4729 Self::override_still_listed(codex_cli_withheld, &routing_registry, model)
4730 });
4731 let decision = match req.model.clone().or(unpinned_override) {
4732 Some(m) => {
4733 let ctx_len = routing_registry
4734 .get(&m)
4735 .or_else(|| routing_registry.find_by_name(&m))
4736 .map(|s| s.context_length)
4737 .unwrap_or(0);
4738 AdaptiveRoutingDecision {
4739 model_id: m.clone(),
4740 model_name: m.clone(),
4741 task: InferenceTask::Generate,
4742 complexity: TaskComplexity::assess(&req.prompt),
4743 reason: "explicit model".into(),
4744 strategy: RoutingStrategy::Explicit,
4745 predicted_quality: 0.5,
4746 fallbacks: vec![],
4747 context_length: ctx_len,
4748 needs_compaction: ctx_len > 0 && estimated_footprint > ctx_len,
4749 candidates: vec![],
4750 }
4751 }
4752 None => self.adaptive_router.route_with_exclusion_observer(
4753 crate::adaptive_router::RouteRequest {
4754 estimated_total_tokens: estimated_footprint,
4755 estimated_input_tokens: estimated_input,
4756 estimated_output_tokens: req.params.max_tokens,
4757 estimated_cache_read_tokens: estimated_cache_read,
4758 estimated_cache_write_tokens: estimated_cache_write,
4759 has_tools,
4760 has_vision,
4761 workload: req.params.workload,
4762 intent: req.intent.as_ref(),
4763 ..crate::adaptive_router::RouteRequest::new(
4764 &req.prompt,
4765 &routing_registry,
4766 &tracker_read,
4767 )
4768 },
4769 exclusion_observer,
4770 ),
4771 };
4772 drop(tracker_read);
4773
4774 if decision.model_id.is_empty() {
4775 let excluded_models = req
4776 .intent
4777 .as_ref()
4778 .map(|hint| hint.exclude_models.join(", "))
4779 .unwrap_or_default();
4780 return Err(InferenceError::NoEligibleModel { excluded_models });
4781 }
4782
4783 if decision.needs_compaction {
4784 tracing::info!(
4785 model = %decision.model_name,
4786 prompt_tokens = estimated_input,
4787 context_window = decision.context_length,
4788 "prompt exceeds model context window — compaction or truncation needed"
4789 );
4790 }
4791
4792 debug!(
4797 model = %decision.model_name,
4798 strategy = ?decision.strategy,
4799 reason = %decision.reason,
4800 "adaptive-routed generate request"
4801 );
4802
4803 let mut req = req;
4806
4807 if let Some(schema) = routing_registry
4813 .get(&decision.model_id)
4814 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4815 {
4816 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
4817 }
4818
4819 let is_code_intent = is_explicit_code_intent(req.intent.as_ref());
4828 let is_complex = matches!(decision.complexity, TaskComplexity::Complex);
4829 if req.params.budget_tokens == 0 && (is_code_intent || is_complex) {
4830 let routed_schema = routing_registry
4834 .get(&decision.model_id)
4835 .or_else(|| routing_registry.find_by_name(&decision.model_id));
4836 let supports_thinking = routed_schema
4837 .map(|s| {
4838 s.supported_params
4839 .contains(&schema::GenerateParam::ExtendedThinking)
4840 })
4841 .unwrap_or(false);
4842 let fitted = auto_thinking_budget(is_code_intent, is_complex, supports_thinking)
4846 .and_then(|budget| match routed_schema.map(|s| (&s.source, &s.name)) {
4847 Some((ModelSource::RemoteApi { protocol, .. }, name)) => {
4848 protocol::fit_auto_thinking_budget(
4849 *protocol,
4850 name,
4851 req.params.max_tokens,
4852 budget,
4853 )
4854 }
4855 _ => Some(budget),
4856 });
4857 if let Some(budget) = fitted {
4858 req.params.budget_tokens = budget;
4859 tracing::info!(
4860 model = %decision.model_name,
4861 budget,
4862 code_intent = is_code_intent,
4863 "auto-enabled extended thinking"
4864 );
4865 }
4866 }
4867
4868 let mut models_to_try = vec![decision.model_id.clone()];
4870 models_to_try.extend(decision.fallbacks.iter().cloned());
4871
4872 let chain_has_local = models_to_try.iter().any(|m| {
4887 routing_registry
4888 .get(m)
4889 .or_else(|| routing_registry.find_by_name(m))
4890 .map(|s| s.is_local())
4891 .unwrap_or(false)
4892 });
4893 let mut local_last_resort_id = None;
4894 let mut adoptable_local = Vec::new();
4898 if should_append_local_last_resort(chain_has_local, req.params.strict_model) {
4899 if let Some(local) = self.first_installed_local_model(has_tools) {
4903 tracing::info!(
4904 local_model = %local,
4905 needs_tools = has_tools,
4906 "appended on-device model as last-resort fallback (chain was remote-only)"
4907 );
4908 models_to_try.push(local.clone());
4909 local_last_resort_id = Some(local);
4910 } else {
4911 adoptable_local = self.cache_only_local_models(has_tools);
4912 if !adoptable_local.is_empty() {
4913 tracing::warn!(
4914 models = ?adoptable_local,
4915 "no on-device last resort: ready local models are in the shared Hugging Face cache but not installed through CAR (`car models adopt <id>`)"
4916 );
4917 }
4918 }
4919 } else if !chain_has_local && req.params.strict_model {
4920 tracing::info!(
4921 model = %decision.model_id,
4922 "strict_model set — not degrading to on-device; a remote failure will surface as an error"
4923 );
4924 }
4925
4926 let mut last_error = None;
4927 let mut auth_dead_lane: Option<String> = None;
4933 let mut route_credential_failure = if chain_includes_parslee_route(
4943 |m| {
4944 routing_registry
4945 .get(m)
4946 .or_else(|| routing_registry.find_by_name(m))
4947 },
4948 &models_to_try,
4949 ) {
4950 initial_route_credential_failure
4951 } else {
4952 None
4953 };
4954 let mut fallback_hops: Vec<FallbackFrom> = Vec::new();
4958
4959 let mut candidate_queue: std::collections::VecDeque<String> =
4966 models_to_try.iter().cloned().collect();
4967 let mut is_primary_attempt = true;
4968 while let Some(candidate_owned) = candidate_queue.pop_front() {
4969 crate::offload::ensure_not_controlled_terminated()?;
4970 let was_primary = is_primary_attempt;
4971 is_primary_attempt = false;
4972 let candidate_id = &candidate_owned;
4973 #[allow(unused_mut)]
4976 let mut schema = routing_registry
4977 .get(candidate_id)
4978 .or_else(|| routing_registry.find_by_name(candidate_id))
4979 .cloned();
4980
4981 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4989 if requested_model_id.is_none() {
4990 if let Some(ref s) = schema {
4991 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
4992 tracing::info!(
4993 from = %s.id, to = %mlx_equiv.id,
4994 "redirecting GGUF model to MLX equivalent on Apple Silicon"
4995 );
4996 schema = Some(mlx_equiv.clone());
4997 }
4998 }
4999 }
5000
5001 let candidate_name = schema
5006 .as_ref()
5007 .map(|s| s.name.clone())
5008 .unwrap_or_else(|| candidate_id.clone());
5009
5010 if has_tools
5021 && schema
5022 .as_ref()
5023 .map(|s| !s.has_capability(ModelCapability::ToolUse))
5024 .unwrap_or(false)
5025 {
5026 let backend = schema
5027 .as_ref()
5028 .map(|s| if s.is_local() { "local" } else { "remote" })
5029 .unwrap_or("unknown");
5030 tracing::warn!(
5031 model = %candidate_id,
5032 backend,
5033 "tools requested but resolved model lacks ToolUse capability — skipping candidate"
5034 );
5035 let unsupported = InferenceError::UnsupportedMode {
5036 mode: "tool_use",
5037 backend,
5038 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
5039 };
5040 record_fallback_from(&mut fallback_hops, &candidate_name, &unsupported);
5046 last_error = Some(unsupported);
5047 continue;
5048 }
5049
5050 let resolved_id = schema
5061 .as_ref()
5062 .map(|s| s.id.clone())
5063 .unwrap_or_else(|| candidate_id.clone());
5064 let reported_model_used = if requested_model_id.is_some() {
5065 resolved_id.clone()
5066 } else {
5067 candidate_name.clone()
5068 };
5069 validate_expected_catalog_row(&req, catalog_snapshot, &resolved_id)?;
5070
5071 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5077 if requested_model_id.is_some()
5078 && schema
5079 .as_ref()
5080 .is_some_and(|schema| matches!(schema.source, ModelSource::Local { .. }))
5081 {
5082 let pinned = InferenceError::InferenceFailed(format!(
5083 "exact model_id `{resolved_id}` names a GGUF/Candle row, but Candle is disabled on Apple Silicon; MLX-equivalent substitution is disabled for exact pins"
5084 ));
5085 record_fallback_from(&mut fallback_hops, &candidate_name, &pinned);
5086 last_error = Some(pinned);
5087 continue;
5088 }
5089
5090 let is_remote = schema
5091 .as_ref()
5092 .map(|s| s.is_remote() || s.is_vllm_mlx())
5093 .unwrap_or(false);
5094 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
5095 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
5096
5097 let attempt_trace = {
5102 let mut tracker = self.outcome_tracker.write().await;
5103 tracker.record_start(&resolved_id, decision.task, &decision.reason)
5104 };
5105
5106 if is_delegated {
5112 let runner = match runner::current_inference_runner() {
5113 Some(r) => r,
5114 None => {
5115 let msg = "model declares ModelSource::Delegated but no inference runner is registered";
5116 self.outcome_tracker.write().await.record_not_model_fault(
5119 &attempt_trace,
5120 msg,
5121 outcome::UnattributedReason::Capability,
5122 );
5123 let unregistered = InferenceError::InferenceFailed(msg.into());
5124 record_fallback_from(&mut fallback_hops, &candidate_name, &unregistered);
5125 last_error = Some(unregistered);
5126 continue;
5127 }
5128 };
5129 let (tx, mut rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
5130 let emitter = runner::EventEmitter::new(tx);
5131 let runner_req = req.clone();
5132 let runner_handle = AbortOnDropTask(Some(tokio::spawn(async move {
5133 runner.run(runner_req, emitter).await
5134 })));
5135 let mut accumulator = stream::StreamAccumulator::default();
5136 while let Some(evt) = rx.recv().await {
5137 accumulator.push(&evt);
5138 }
5139 let (acc_text, acc_tool_calls, acc_usage, acc_stop_reason) =
5168 accumulator.finish_with_usage();
5169 match runner_handle.join().await {
5170 Ok(Ok(runner_result)) => {
5171 let elapsed = start.elapsed().as_millis() as u64;
5172 let acc_text = if acc_text.trim().is_empty() {
5173 runner_result.text
5174 } else {
5175 acc_text
5176 };
5177 let acc_tool_calls = if acc_tool_calls.is_empty() {
5178 runner_result.tool_calls
5179 } else {
5180 acc_tool_calls
5181 };
5182 let est_out =
5192 booked_output_tokens(acc_usage.as_ref(), &acc_text, &acc_tool_calls);
5193 {
5194 let mut tracker = self.outcome_tracker.write().await;
5195 tracker.record_complete(
5196 &attempt_trace,
5197 elapsed,
5198 estimated_input,
5199 est_out,
5200 );
5201 }
5202 let local_last_resort = report_local_last_resort_served(
5203 local_last_resort_id.as_deref(),
5204 candidate_id,
5205 &resolved_id,
5206 );
5207 record_generation_span(
5214 candidate_name.as_str(),
5215 elapsed,
5216 None,
5217 acc_usage.as_ref(),
5218 );
5219 return Ok(InferenceResult {
5220 text: acc_text,
5221 tool_calls: acc_tool_calls,
5222 bounding_boxes: vec![],
5223 trace_id: attempt_trace,
5224 model_used: reported_model_used,
5225 model_identity: bound_model_identity(
5226 catalog_snapshot,
5227 requested_model_id.as_deref(),
5228 &resolved_id,
5229 )?,
5230 latency_ms: elapsed,
5231 time_to_first_token_ms: None,
5232 usage: acc_usage,
5235 provider_output_items: vec![],
5236 thinking: vec![],
5239 stop_reason: acc_stop_reason,
5240 auth_fallback_from: auth_dead_lane.clone(),
5241 local_last_resort,
5242 fallback_from: fallback_hops.clone(),
5243 });
5244 }
5245 Ok(Err(e)) => {
5246 {
5247 let mut tracker = self.outcome_tracker.write().await;
5248 match &e {
5249 runner::RunnerError::Declined(_) => tracker.record_not_model_fault(
5252 &attempt_trace,
5253 &e.to_string(),
5254 outcome::UnattributedReason::Capability,
5255 ),
5256 runner::RunnerError::Failed(_) => {
5257 tracker.record_failure(&attempt_trace, &e.to_string())
5258 }
5259 }
5260 }
5261 let failed = InferenceError::InferenceFailed(e.to_string());
5268 record_auth_dead_lane(
5269 &mut auth_dead_lane,
5270 &candidate_name,
5271 &failed.to_string(),
5272 );
5273 record_route_credential_failure(
5274 &mut route_credential_failure,
5275 &candidate_name,
5276 &failed,
5277 was_primary && has_requested_route,
5278 );
5279 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
5280 last_error = Some(failed);
5281 continue;
5282 }
5283 Err(join_err) => {
5284 let msg = format!("runner task panicked: {join_err}");
5285 self.outcome_tracker
5286 .write()
5287 .await
5288 .record_failure(&attempt_trace, &msg);
5289 let failed = InferenceError::InferenceFailed(msg);
5295 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
5296 last_error = Some(failed);
5297 continue;
5298 }
5299 }
5300 }
5301
5302 let has_tools = Self::request_has_tools(&req);
5303
5304 let context = if has_tools
5306 && req.tools.as_ref().is_some_and(|t| {
5307 t.iter().any(|tool| {
5308 tool.get("function")
5309 .and_then(|f| f.get("name"))
5310 .and_then(|n| n.as_str())
5311 == Some("done")
5312 })
5313 }) {
5314 let base = req.context.as_deref().unwrap_or("");
5315 Some(format!(
5316 "{base}\n\nIMPORTANT: When calling the `done` tool, the `result` field MUST contain a DETAILED summary of everything you found and did. This is the ONLY output the user sees. Do NOT just say 'completed' — include specific findings, data, and conclusions."
5317 ))
5318 } else {
5319 req.context.clone()
5320 };
5321
5322 let mut captured_thinking: Vec<crate::tasks::generate::ThinkingBlock> = Vec::new();
5326 let mut captured_provider_output_items: Vec<serde_json::Value> = Vec::new();
5327 let mut local_reservation = if !is_delegated {
5333 match schema
5334 .as_ref()
5335 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
5336 {
5337 Some(local_schema) => {
5338 match self
5339 .reserve_local_request_with_worker_retry(
5340 local_schema,
5341 estimated_footprint,
5342 )
5343 .await
5344 {
5345 Ok(reservation) => Some(reservation),
5346 Err(error) if req.model.is_some() || req.params.strict_model => {
5347 self.outcome_tracker
5348 .write()
5349 .await
5350 .record_capability_rejection(
5351 &attempt_trace,
5352 &error.to_string(),
5353 );
5354 return Err(error);
5355 }
5356 Err(error) => {
5357 tracing::warn!(
5358 model = %local_schema.id,
5359 error = %error,
5360 "adaptive local candidate blocked by resource policy; trying next route"
5361 );
5362 self.outcome_tracker
5363 .write()
5364 .await
5365 .record_capability_rejection(
5366 &attempt_trace,
5367 &error.to_string(),
5368 );
5369 record_fallback_from(&mut fallback_hops, &candidate_name, &error);
5374 last_error = Some(error);
5375 continue;
5376 }
5377 }
5378 }
5379 None => None,
5380 }
5381 } else {
5382 None
5383 };
5384 let result = if is_codex_cli {
5385 let schema_ref = schema
5386 .as_ref()
5387 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5388 if req.response_format.is_some() {
5389 Err(InferenceError::UnsupportedMode {
5390 mode: "response-format",
5391 backend: "codex-cli",
5392 reason: "the subscription-backed Codex source returns plain text and does not expose provider-enforced schemas",
5393 })
5394 } else {
5395 let model = match &schema_ref.source {
5396 ModelSource::CodexCli { model } => model,
5397 _ => unreachable!("is_codex_cli matched a different source"),
5398 };
5399 crate::backend::codex_cli::generate_request(
5400 model,
5401 &req,
5402 context.as_deref(),
5403 schema_ref.context_length,
5404 )
5405 .await
5406 .map(|(output, calls)| (output.text, calls, Some(output.usage), None, None))
5407 }
5408 } else if is_remote {
5409 let (schema_val, _remote_request_reservation) = match self
5413 .vllm_live_schema(
5414 schema.unwrap(),
5415 local_reservation.take(),
5416 estimated_footprint,
5417 )
5418 .await
5419 {
5420 Ok(pair) => pair,
5421 Err(e) => {
5422 record_dispatch_failure(
5426 &mut *self.outcome_tracker.write().await,
5427 &attempt_trace,
5428 &e,
5429 );
5430 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5431 last_error = Some(e);
5432 continue;
5433 }
5434 };
5435 let _ctx_len = schema_val.context_length;
5436 let temperature = if !schema_val.supported_params.is_empty()
5439 && !schema_val
5440 .supported_params
5441 .contains(&crate::schema::GenerateParam::Temperature)
5442 {
5443 -1.0
5444 } else {
5445 req.params.temperature
5446 };
5447
5448 self.remote_backend
5454 .with_work_context(req.work_context.clone())
5455 .generate_with_tools_multi_observed(
5456 &schema_val,
5457 &req.prompt,
5458 context.as_deref(),
5459 temperature,
5460 req.params.max_tokens,
5461 req.tools.as_deref(),
5462 req.images.as_deref(),
5463 req.messages.as_deref(),
5464 req.params.tool_choice.as_deref(),
5465 req.params.parallel_tool_calls,
5466 req.params.budget_tokens,
5467 req.cache_control,
5468 req.params.cache_ttl,
5469 req.context_stable_prefix.as_deref(),
5470 req.response_format.as_ref(),
5471 retry_observer,
5472 )
5473 .await
5474 .map(|(t, c, thinking, provider_items, u, stop)| {
5480 captured_thinking = thinking;
5481 captured_provider_output_items = provider_items;
5482 (t, c, u, None::<u64>, stop)
5483 })
5484 } else if let Some(offload) = schema
5485 .as_ref()
5486 .filter(|schema| Self::supports_worker_offload(schema))
5487 .and_then(|_| crate::offload::current_local_offload())
5488 {
5489 let mut offload_req = req.clone();
5500 offload_req.model = Some(resolved_id.clone());
5501 let schema_ref = schema
5502 .as_ref()
5503 .ok_or_else(|| InferenceError::ModelNotFound(resolved_id.clone()))?;
5504 let reservation = local_reservation.as_mut().ok_or_else(|| {
5505 InferenceError::InferenceFailed(
5506 "local worker dispatch missing admission reservation".into(),
5507 )
5508 })?;
5509 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_id) {
5510 reservation.bind_allocation_id(&allocation_id);
5516 }
5517 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
5518 match offload.generate_admitted(offload_req, admission).await {
5519 Ok(outcome) => {
5520 Self::reconcile_worker_residency(
5521 offload.as_ref(),
5522 &resolved_id,
5523 &outcome.residency,
5524 outcome.retention,
5525 reservation,
5526 )
5527 .await?;
5528 let ir = outcome.result;
5529 captured_thinking = ir.thinking;
5536 captured_provider_output_items = ir.provider_output_items;
5537 Ok((
5538 ir.text,
5539 ir.tool_calls,
5540 ir.usage,
5541 ir.time_to_first_token_ms,
5542 ir.stop_reason,
5543 ))
5544 }
5545 Err(e) => Err(e),
5546 }
5547 } else {
5548 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5549 {
5550 let schema_ref = schema
5554 .as_ref()
5555 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5556
5557 if schema_ref.is_foundation_models() {
5567 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5568 if Self::request_has_video(&req) || Self::request_has_audio(&req) {
5569 Err(InferenceError::UnsupportedMode {
5570 mode: "multimodal-content",
5571 backend: "foundation-models",
5572 reason: "the FoundationModels bridge exposes text and image \
5573 input — route audio/video to a remote VL model",
5574 })
5575 } else if Self::request_has_message_images(&req) {
5576 Err(InferenceError::UnsupportedMode {
5579 mode: "multimodal-content",
5580 backend: "foundation-models",
5581 reason: "the FoundationModels bridge reads images only from \
5582 the request's `images` field — route images inside \
5583 `messages` to a VL model",
5584 })
5585 } else if has_images && has_tools {
5586 Err(InferenceError::UnsupportedMode {
5589 mode: "multimodal-content",
5590 backend: "foundation-models",
5591 reason: "the FoundationModels image path carries no tools — \
5592 route a request with both images and tools to a \
5593 tool-capable VL model",
5594 })
5595 } else if has_images
5596 && matches!(
5597 req.response_format,
5598 Some(crate::tasks::generate::ResponseFormat::JsonSchema { .. })
5599 )
5600 {
5601 Err(InferenceError::UnsupportedMode {
5604 mode: "structured-output",
5605 backend: "foundation-models",
5606 reason: "the FoundationModels image path has no schema-\
5607 constrained decoding — route an image request with a \
5608 JsonSchema response_format to a VL model",
5609 })
5610 } else if has_images {
5611 match foundation_models_inline_images(
5617 req.images.as_deref().unwrap_or_default(),
5618 ) {
5619 Err(refusal) => Err(refusal),
5620 Ok(images) => {
5621 let prompt = req.prompt.clone();
5622 let instructions = foundation_models_instructions(
5623 context.as_deref(),
5624 req.response_format.as_ref(),
5625 );
5626 let fm_prompt = prompt.clone();
5627 let fm_instructions = instructions.clone();
5628 let fm_ctx_window = schema_ref.context_length as u64;
5629 let max_tokens = req.params.max_tokens as u32;
5630 let temperature = req.params.temperature;
5631 tokio::task::spawn_blocking(move || {
5632 crate::backend::foundation_models::generate_with_images(
5633 &prompt,
5634 instructions.as_deref(),
5635 &images,
5636 max_tokens,
5637 temperature as f32,
5638 )
5639 })
5640 .await
5641 .map_err(|e| {
5642 InferenceError::InferenceFailed(format!(
5643 "FoundationModels task panicked: {e}"
5644 ))
5645 })
5646 .and_then(|r| r)
5647 .map(|text| {
5648 let usage = foundation_models_usage(
5649 fm_instructions.as_deref(),
5650 &fm_prompt,
5651 &text,
5652 fm_ctx_window,
5653 );
5654 (text, vec![], usage, None, None)
5655 })
5656 }
5657 }
5658 } else if has_tools {
5659 if req.response_format.is_some() {
5665 tracing::warn!(
5666 "FoundationModels: response_format is ignored when tools \
5667 are present — one turn is either tool-enabled or \
5668 schema-constrained"
5669 );
5670 }
5671 let prompt = req.prompt.clone();
5672 let instructions = context.clone();
5673 let fm_prompt = prompt.clone();
5676 let fm_instructions = instructions.clone();
5677 let fm_ctx_window = schema_ref.context_length as u64;
5678 let tools_defs = req.tools.clone().unwrap_or_default();
5679 let fm_tool_choice = req.params.tool_choice.clone();
5680 let max_tokens = req.params.max_tokens as u32;
5681 let temperature = req.params.temperature;
5682 tokio::task::spawn_blocking(move || {
5683 crate::backend::foundation_models::generate_with_tools(
5684 &prompt,
5685 instructions.as_deref(),
5686 &tools_defs,
5687 fm_tool_choice.as_deref(),
5688 max_tokens,
5689 temperature as f32,
5690 )
5691 })
5692 .await
5693 .map_err(|e| {
5694 InferenceError::InferenceFailed(format!(
5695 "FoundationModels task panicked: {e}"
5696 ))
5697 })
5698 .and_then(|r| r)
5699 .map(|(text, calls)| {
5700 let usage = foundation_models_usage(
5703 fm_instructions.as_deref(),
5704 &fm_prompt,
5705 &crate::backend::foundation_models::completion_text(
5706 &text, &calls,
5707 ),
5708 fm_ctx_window,
5709 );
5710 (text, calls, usage, None, None)
5711 })
5712 } else if let Some(crate::tasks::generate::ResponseFormat::JsonSchema {
5713 schema,
5714 ..
5715 }) = &req.response_format
5716 {
5717 let prompt = req.prompt.clone();
5721 let instructions = context.clone();
5722 let fm_prompt = prompt.clone();
5725 let fm_instructions = instructions.clone();
5726 let fm_ctx_window = schema_ref.context_length as u64;
5727 let schema_val = schema.clone();
5728 let max_tokens = req.params.max_tokens as u32;
5729 let temperature = req.params.temperature;
5730 tokio::task::spawn_blocking(move || {
5731 crate::backend::foundation_models::generate_structured(
5732 &prompt,
5733 instructions.as_deref(),
5734 &schema_val,
5735 max_tokens,
5736 temperature as f32,
5737 )
5738 })
5739 .await
5740 .map_err(|e| {
5741 InferenceError::InferenceFailed(format!(
5742 "FoundationModels task panicked: {e}"
5743 ))
5744 })
5745 .and_then(|r| r)
5746 .map(|text| {
5747 let usage = foundation_models_usage(
5748 fm_instructions.as_deref(),
5749 &fm_prompt,
5750 &text,
5751 fm_ctx_window,
5752 );
5753 (text, vec![], usage, None, None)
5754 })
5755 } else {
5756 let instructions = foundation_models_instructions(
5760 context.as_deref(),
5761 req.response_format.as_ref(),
5762 );
5763 let prompt = req.prompt.clone();
5764 let fm_prompt = prompt.clone();
5767 let fm_instructions = instructions.clone();
5768 let fm_ctx_window = schema_ref.context_length as u64;
5769 let max_tokens = req.params.max_tokens as u32;
5770 let temperature = req.params.temperature;
5771 tokio::task::spawn_blocking(move || {
5772 crate::backend::foundation_models::generate(
5773 &prompt,
5774 instructions.as_deref(),
5775 max_tokens,
5776 temperature as f32,
5777 )
5778 })
5779 .await
5780 .map_err(|e| {
5781 InferenceError::InferenceFailed(format!(
5782 "FoundationModels task panicked: {e}"
5783 ))
5784 })
5785 .and_then(|r| r)
5786 .map(|text| {
5787 let usage = foundation_models_usage(
5788 fm_instructions.as_deref(),
5789 &fm_prompt,
5790 &text,
5791 fm_ctx_window,
5792 );
5793 (text, vec![], usage, None, None)
5794 })
5795 }
5796 } else if !schema_ref.is_mlx() {
5797 Err(InferenceError::InferenceFailed(format!(
5798 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
5799 schema_ref.id
5800 )))
5801 } else if schema_ref.tags.iter().any(|t| t == "mlx-vlm-cli") {
5802 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5814 if !has_images {
5815 return Err(InferenceError::UnsupportedMode {
5816 mode: "text-only-on-mlx-vlm-id",
5817 backend: "mlx-vlm-cli",
5818 reason: "the `mlx-vlm/...` model IDs route exclusively \
5819 through the mlx-vlm CLI for image inference. \
5820 For text-only generation, route to a Qwen3 \
5821 text model (`mlx/qwen3-4b:4bit` etc.) — the \
5822 CLI shell-out has higher latency than the \
5823 in-process MLX text tower.",
5824 });
5825 }
5826 let vlm_status = crate::backend::mlx_vlm_cli::runtime_status();
5827 if !vlm_status.is_available() {
5828 return Err(InferenceError::InferenceFailed(vlm_status.user_message()));
5829 }
5830 let model_dir =
5831 self.unified_registry().ensure_local(&schema_ref.id).await?;
5832 let reservation = local_reservation
5833 .as_mut()
5834 .expect("local VLM branch has admission reservation");
5835 Self::reconcile_transient_local_allocation(
5836 reservation,
5837 backend_cache::estimate_model_size(&model_dir),
5838 )?;
5839 let detached_lease = reservation.detached_lease();
5840 let repo = match &schema_ref.source {
5841 crate::schema::ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
5842 _ => {
5843 return Err(InferenceError::InferenceFailed(format!(
5844 "model '{}' is tagged mlx-vlm-cli but its \
5845 source isn't ModelSource::Mlx — registry bug",
5846 schema_ref.id
5847 )));
5848 }
5849 };
5850 let imgs = req.images.clone().unwrap_or_default();
5851 let temp = req.params.temperature;
5852 let max_t = req.params.max_tokens;
5853 let prompt = req.prompt.clone();
5854 let (text, cli_usage) = run_admitted_blocking(detached_lease, move || {
5855 crate::backend::mlx_vlm_cli::generate(
5856 &repo, &prompt, &imgs, temp, max_t,
5857 )
5858 })
5859 .await
5860 .map_err(|e| {
5861 InferenceError::InferenceFailed(format!(
5862 "mlx_vlm CLI task panicked: {e}"
5863 ))
5864 })??;
5865 let bounding_boxes = parse_boxes(&text);
5866 let latency_ms = start.elapsed().as_millis() as u64;
5867 let usage = cli_usage.map(|u| TokenUsage {
5877 prompt_tokens: u.prompt_tokens,
5878 completion_tokens: u.completion_tokens,
5879 total_tokens: u.prompt_tokens + u.completion_tokens,
5880 context_window: schema_ref.context_length as u64,
5881 ..Default::default()
5883 });
5884 {
5885 let (in_tokens, out_tokens) = match &usage {
5891 Some(u) => (u.prompt_tokens as usize, u.completion_tokens as usize),
5892 None => (estimated_input, text.split_whitespace().count()),
5893 };
5894 let mut tracker = self.outcome_tracker.write().await;
5895 tracker.record_complete(
5896 &attempt_trace,
5897 latency_ms,
5898 in_tokens,
5899 out_tokens,
5900 );
5901 }
5902 let local_last_resort = report_local_last_resort_served(
5903 local_last_resort_id.as_deref(),
5904 candidate_id,
5905 &resolved_id,
5906 );
5907 record_generation_span(
5912 candidate_name.as_str(),
5913 latency_ms,
5914 None,
5915 usage.as_ref(),
5916 );
5917 return Ok(InferenceResult {
5918 text,
5919 tool_calls: vec![],
5920 bounding_boxes,
5921 trace_id: attempt_trace,
5922 model_used: schema_ref.id.clone(),
5923 model_identity: bound_model_identity(
5924 catalog_snapshot,
5925 requested_model_id.as_deref(),
5926 &resolved_id,
5927 )?,
5928 latency_ms,
5929 time_to_first_token_ms: None,
5930 usage,
5931 provider_output_items: Vec::new(),
5932 thinking: Vec::new(), stop_reason: None,
5934 auth_fallback_from: auth_dead_lane.clone(),
5935 local_last_resort,
5936 fallback_from: fallback_hops.clone(),
5937 });
5938 } else if !Self::supports_native_mlx(schema_ref) {
5939 if req.images.as_ref().is_some_and(|i| !i.is_empty())
5949 || Self::request_has_video(&req)
5950 || Self::request_has_audio(&req)
5951 {
5952 return Err(InferenceError::UnsupportedMode {
5953 mode: "multimodal-content-block",
5954 backend: "native-mlx-local",
5955 reason: "this in-process MLX backend is text-only; route \
5956 image/video/audio understanding to a vLLM-MLX or remote \
5957 multimodal model",
5958 });
5959 }
5960 self.generate_local(
5971 req.clone(),
5972 &schema_ref.id,
5973 local_reservation
5974 .as_mut()
5975 .expect("local branch has admission reservation"),
5976 )
5977 .await
5978 } else {
5979 if Self::request_has_video(&req) {
5985 return Err(InferenceError::UnsupportedMode {
5986 mode: "video-content-block",
5987 backend: "native-mlx-qwen25vl",
5988 reason: "Qwen2.5-VL video understanding is on the request surface \
5989 but the video-tokenization path (frame sampling + merger) \
5990 is not yet wired; route to a remote VL provider for now",
5991 });
5992 }
5993 if Self::request_has_audio(&req) {
5994 return Err(InferenceError::UnsupportedMode {
5995 mode: "audio-content-block",
5996 backend: "native-mlx-qwen25vl",
5997 reason: "audio understanding is on the request surface (Gemma 4 \
5998 E2B/E4B and Gemini accept it) but the native MLX path \
5999 for this model does not — route to Gemini or Gemma-4",
6000 });
6001 }
6002 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
6003 if has_images {
6004 let (handle, _retention) = self
6024 .ensure_text_backend(
6025 schema_ref,
6026 local_reservation
6027 .as_mut()
6028 .expect("local branch has admission reservation"),
6029 )
6030 .await?;
6031 let can_do_vision = {
6032 let guard = handle.lock().map_err(|_| {
6033 InferenceError::InferenceFailed(
6034 "MLX backend mutex poisoned".into(),
6035 )
6036 })?;
6037 guard.supports_capability(crate::schema::ModelCapability::Vision)
6038 };
6039 if !can_do_vision {
6040 return Err(InferenceError::UnsupportedMode {
6050 mode: "image-content-block",
6051 backend: "native-mlx-text",
6052 reason: "this MLX backend is a plain Qwen3 text tower. \
6053 For local image inference, route to \
6054 `mlx-vlm/qwen3-vl-2b:bf16` or another `mlx-vlm/...` \
6055 catalog ID so CAR shells out to `mlx_vlm.generate`. \
6056 Alternatives: a local vLLM-MLX VLM server, or a \
6057 remote VL model. (#115)",
6058 });
6059 }
6060 }
6061 self.generate_mlx(
6062 render_for_local_backend(req.clone()),
6063 &schema_ref.id,
6064 local_reservation
6065 .as_mut()
6066 .expect("local branch has admission reservation"),
6067 )
6068 .await
6069 .map(|(text, usage, ttft, stop)| (text, vec![], usage, ttft, stop))
6070 }
6071 }
6072
6073 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6074 {
6075 let schema_ref = schema
6076 .as_ref()
6077 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
6078 match self
6079 .ensure_backend(
6080 schema_ref,
6081 local_reservation
6082 .as_mut()
6083 .expect("local branch has admission reservation"),
6084 )
6085 .await
6086 {
6087 Ok(()) => {
6088 let mut write = self.backend.write().await;
6089 let backend = write.get_mut(&schema_ref.id).ok_or_else(|| {
6090 InferenceError::InferenceFailed(format!(
6091 "candle backend missing after ensure_backend for {}",
6092 schema_ref.id
6093 ))
6094 })?;
6095 let ctx_window = backend.context_length().unwrap_or(0) as u64;
6101 tasks::generate::generate(
6102 backend,
6103 render_for_local_backend(req.clone()),
6104 )
6105 .await
6106 .map(
6107 |(text, ttft, prompt_tokens, completion_tokens)| {
6108 let usage = TokenUsage {
6109 prompt_tokens: prompt_tokens as u64,
6110 completion_tokens: completion_tokens as u64,
6111 total_tokens: (prompt_tokens + completion_tokens) as u64,
6112 context_window: ctx_window,
6113 ..Default::default()
6116 };
6117 (text, vec![], Some(usage), ttft, None)
6118 },
6119 )
6120 }
6121 Err(e) => Err(e),
6122 }
6123 }
6124 };
6125
6126 match result {
6127 Ok((text, mut tool_calls, usage, time_to_first_token_ms, stop_reason)) => {
6128 let text = if !is_remote
6135 && !is_codex_cli
6136 && !is_delegated
6137 && req.tools.is_some()
6138 && tool_calls.is_empty()
6139 {
6140 let (clean, parsed) = tasks::generate::parse_tool_calls(&text);
6141 tool_calls = parsed;
6142 clean
6143 } else {
6144 text
6145 };
6146 let latency_ms = start.elapsed().as_millis() as u64;
6147 let estimated_tokens = booked_output_tokens(usage.as_ref(), &text, &tool_calls);
6151 let input_tokens = usage
6156 .as_ref()
6157 .map(|u| u.prompt_tokens as usize)
6158 .unwrap_or(estimated_input);
6159 let (cache_read, cache_creation) = usage
6164 .as_ref()
6165 .map(|u| {
6166 (
6167 u.cache_read_input_tokens as usize,
6168 u.cache_creation_input_tokens as usize,
6169 )
6170 })
6171 .unwrap_or((0, 0));
6172 {
6173 let mut tracker = self.outcome_tracker.write().await;
6174 tracker.record_complete_cached(
6175 &attempt_trace,
6176 latency_ms,
6177 input_tokens,
6178 estimated_tokens,
6179 cache_read,
6180 cache_creation,
6181 );
6182 }
6183 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
6189 cb.record_success(&resolved_id);
6190 }
6191 self.auto_save_outcomes().await;
6193
6194 record_generation_span(
6196 candidate_name.as_str(),
6197 latency_ms,
6198 time_to_first_token_ms,
6199 usage.as_ref(),
6200 );
6201
6202 let bounding_boxes = tasks::grounding::parse_boxes(&text);
6205 let local_last_resort = report_local_last_resort_served(
6206 local_last_resort_id.as_deref(),
6207 candidate_id,
6208 &resolved_id,
6209 );
6210 return Ok(InferenceResult {
6211 text,
6212 tool_calls,
6213 bounding_boxes,
6214 trace_id: attempt_trace,
6215 model_used: reported_model_used,
6216 model_identity: bound_model_identity(
6217 catalog_snapshot,
6218 requested_model_id.as_deref(),
6219 &resolved_id,
6220 )?,
6221 latency_ms,
6222 time_to_first_token_ms,
6223 usage,
6224 provider_output_items: captured_provider_output_items,
6225 thinking: captured_thinking,
6226 stop_reason,
6227 auth_fallback_from: auth_dead_lane.clone(),
6228 local_last_resort,
6229 fallback_from: fallback_hops.clone(),
6230 });
6231 }
6232 Err(e) => {
6233 if matches!(e, InferenceError::ControlledTermination) {
6234 self.outcome_tracker.write().await.record_not_model_fault(
6235 &attempt_trace,
6236 &e.to_string(),
6237 outcome::UnattributedReason::Cancelled,
6238 );
6239 return Err(e);
6240 }
6241 tracing::warn!(
6242 model = %candidate_name,
6243 error = %e,
6244 remaining = candidate_queue.len(),
6245 "model failed, trying next fallback immediately"
6246 );
6247 record_auth_dead_lane(&mut auth_dead_lane, &candidate_name, &e.to_string());
6254 record_route_credential_failure(
6255 &mut route_credential_failure,
6256 &candidate_name,
6257 &e,
6258 was_primary && has_requested_route,
6259 );
6260 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
6261 {
6266 let mut tracker = self.outcome_tracker.write().await;
6267 record_dispatch_failure(&mut tracker, &attempt_trace, &e);
6268 }
6269 if error_counts_against_circuit_breaker(&e) {
6279 let err_str = e.to_string();
6280 let is_client_error =
6281 err_str.contains("API returned 4") && !err_str.contains("429");
6282 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
6283 cb.record_failure(&resolved_id);
6285 if is_client_error {
6288 cb.record_failure(&resolved_id);
6289 }
6290 }
6291 }
6292 #[cfg(not(all(
6294 target_os = "macos",
6295 target_arch = "aarch64",
6296 not(car_skip_mlx)
6297 )))]
6298 {
6299 let mut write = self.backend.write().await;
6300 if write.remove(&resolved_id).is_some() {
6301 self.local_admission.mark_evicted(&resolved_id);
6302 }
6303 }
6304 if let InferenceError::GatewayUnconfigured { namespace, .. } = &e {
6332 let before = candidate_queue.len();
6333 candidate_queue.retain(|id| !id.starts_with(namespace.as_str()));
6334 let dropped = before - candidate_queue.len();
6335 if dropped > 0 {
6336 tracing::info!(
6337 %namespace,
6338 dropped,
6339 remaining = candidate_queue.len(),
6340 "gateway has no upstream for this namespace; dropping its \
6341 remaining candidates instead of replaying the same rejection"
6342 );
6343 }
6344 }
6345 if error_ends_fallback_chain(&e) {
6346 let dropped = candidate_queue.len();
6347 candidate_queue.clear();
6348 if dropped > 0 {
6349 tracing::info!(
6350 dropped,
6351 "content refused for this request; ending the fallback chain \
6352 rather than answering with a different model"
6353 );
6354 }
6355 }
6356 if let InferenceError::ProviderAccount { provider, .. } = &e {
6357 let mut rest: Vec<String> = candidate_queue.iter().cloned().collect();
6361 let demoted = routing_ext::demote_provider(provider, &mut rest, |id| {
6362 routing_registry
6363 .get(id)
6364 .or_else(|| routing_registry.find_by_name(id))
6365 .map(|s| s.provider.clone())
6366 });
6367 candidate_queue = rest.into();
6368 if demoted > 0 {
6369 tracing::info!(
6370 %provider,
6371 demoted,
6372 remaining = candidate_queue.len(),
6373 "account-level rejection; deferring this provider's \
6374 remaining candidates to the end of the chain"
6375 );
6376 }
6377 }
6378 if was_primary && !candidate_queue.is_empty() {
6379 let err_str = e.to_string();
6380 let status = parse_api_returned_status(&err_str);
6387 if routing_ext::is_provider_transient(status, &err_str) {
6388 let provider_of = |id: &str| {
6393 routing_registry
6394 .get(id)
6395 .or_else(|| routing_registry.find_by_name(id))
6396 .map(|s| s.provider.clone())
6397 };
6398 let primary = provider_of(candidate_id).unwrap_or_default();
6399 let queue_vec: Vec<String> = candidate_queue.iter().cloned().collect();
6400 if let Some(cross) =
6401 routing_ext::first_cross_provider(&primary, &queue_vec, provider_of)
6402 {
6403 let cross = cross.to_string();
6404 if let Some(pos) = candidate_queue.iter().position(|m| *m == cross)
6405 {
6406 if pos > 0 {
6407 if let Some(m) = candidate_queue.remove(pos) {
6408 tracing::info!(
6409 promoted = %m,
6410 "transient provider error on primary; promoting cross-provider fallback"
6411 );
6412 candidate_queue.push_front(m);
6413 }
6414 }
6415 }
6416 }
6417 }
6418 }
6419 last_error = Some(e);
6420 }
6421 }
6422 }
6423
6424 let underlying = last_error.unwrap_or(InferenceError::InferenceFailed(
6426 "no models available".into(),
6427 ));
6428
6429 let e = apply_route_failure_context(underlying, route_credential_failure.as_ref());
6443 let e = append_adopt_hint(e, &adoptable_local);
6444 self.auto_save_outcomes().await;
6448 Err(e)
6449 }
6450
6451 async fn generate_stream_raw(
6456 &self,
6457 req: GenerateRequest,
6458 ) -> Result<(String, tokio::sync::mpsc::Receiver<stream::StreamEvent>), InferenceError> {
6459 let mut routing_registry = if let Some(context) = &req.work_context {
6460 self.work_context_routing_snapshot(context, req.model.as_deref())
6461 .await
6462 .0
6463 } else {
6464 self.request_routing_registry_snapshot(req.model.as_deref())
6465 .await
6466 };
6467 let codex_cli_withheld =
6471 req.model.is_none() && Self::withhold_codex_cli_rows(&req, true, &mut routing_registry);
6472 if let Some(requested) = req.model.as_deref() {
6473 if routing_registry
6474 .get(requested)
6475 .or_else(|| routing_registry.find_by_name(requested))
6476 .is_none()
6477 {
6478 return Err(InferenceError::ModelNotFound(requested.to_string()));
6479 }
6480 }
6481 let has_tools = Self::request_has_tools(&req);
6482 let has_vision = Self::request_needs_vision(&req);
6483 let vision_stream_intent =
6484 Self::vision_stream_intent(req.intent.as_ref(), has_vision, &routing_registry);
6485 let route_intent = vision_stream_intent.as_ref().or(req.intent.as_ref());
6486 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6487 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
6488 let (estimated_cache_read, estimated_cache_write) =
6489 Self::routing_cache_estimates(&req, estimated_input);
6490 let preferred_model = self
6491 .preferred_model_for_capability(ModelCapability::Generate)
6492 .map(str::to_string);
6493 let exclude_set = self
6494 .adaptive_router
6495 .build_exclude_set(route_intent, &routing_registry);
6496 let unpinned_override = self
6497 .lane_pin_for(&req, &routing_registry)
6498 .or(preferred_model)
6499 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model))
6500 .filter(|model| {
6501 Self::override_still_listed(codex_cli_withheld, &routing_registry, model)
6502 });
6503 let decision = match req.model.clone().or(unpinned_override) {
6504 Some(m) => {
6505 let ctx_len = routing_registry
6506 .get(&m)
6507 .or_else(|| routing_registry.find_by_name(&m))
6508 .map(|s| s.context_length)
6509 .unwrap_or(0);
6510 AdaptiveRoutingDecision {
6511 model_id: m.clone(),
6512 model_name: m,
6513 task: InferenceTask::Generate,
6514 complexity: TaskComplexity::assess(&req.prompt),
6515 reason: "explicit model".into(),
6516 strategy: RoutingStrategy::Explicit,
6517 predicted_quality: 0.5,
6518 fallbacks: vec![],
6519 context_length: ctx_len,
6520 needs_compaction: false,
6521 candidates: vec![],
6522 }
6523 }
6524 None => {
6525 let tracker_read = self.outcome_tracker.read().await;
6526 self.adaptive_router
6527 .route_with(crate::adaptive_router::RouteRequest {
6528 estimated_total_tokens: estimated_footprint,
6529 estimated_input_tokens: estimated_input,
6530 estimated_output_tokens: req.params.max_tokens,
6531 estimated_cache_read_tokens: estimated_cache_read,
6532 estimated_cache_write_tokens: estimated_cache_write,
6533 has_tools,
6534 has_vision,
6535 workload: req.params.workload,
6536 intent: route_intent,
6537 ..crate::adaptive_router::RouteRequest::new(
6538 &req.prompt,
6539 &routing_registry,
6540 &tracker_read,
6541 )
6542 })
6543 }
6544 };
6545
6546 if decision.model_id.is_empty() {
6547 let excluded_models = req
6548 .intent
6549 .as_ref()
6550 .map(|hint| hint.exclude_models.join(", "))
6551 .unwrap_or_default();
6552 return Err(InferenceError::NoEligibleModel { excluded_models });
6553 }
6554
6555 #[allow(unused_mut)]
6558 let mut schema = routing_registry
6559 .get(&decision.model_id)
6560 .or_else(|| routing_registry.find_by_name(&decision.model_id))
6561 .cloned();
6562
6563 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6565 if let Some(ref s) = schema {
6566 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
6567 tracing::info!(
6568 from = %s.id, to = %mlx_equiv.id,
6569 "redirecting GGUF model to MLX equivalent on Apple Silicon (stream)"
6570 );
6571 schema = Some(mlx_equiv.clone());
6572 }
6573 }
6574
6575 if has_tools
6581 && schema
6582 .as_ref()
6583 .map(|s| !s.has_capability(ModelCapability::ToolUse))
6584 .unwrap_or(false)
6585 {
6586 let backend = schema
6587 .as_ref()
6588 .map(|s| if s.is_local() { "local" } else { "remote" })
6589 .unwrap_or("unknown");
6590 return Err(InferenceError::UnsupportedMode {
6591 mode: "tool_use",
6592 backend,
6593 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
6594 });
6595 }
6596
6597 let resolved_model_id = schema
6599 .as_ref()
6600 .map(|s| s.id.clone())
6601 .unwrap_or_else(|| decision.model_id.clone());
6602
6603 let mut req = req;
6610 if let Some(schema) = schema.as_ref() {
6611 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
6612 }
6613
6614 let is_remote = schema
6615 .as_ref()
6616 .map(|s| s.is_remote() || s.is_vllm_mlx())
6617 .unwrap_or(false);
6618
6619 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
6620 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
6621
6622 if is_codex_cli {
6623 return Err(InferenceError::UnsupportedMode {
6624 mode: "streaming",
6625 backend: "codex-cli",
6626 reason: "codex exec reports final answer items rather than token deltas; use non-streaming infer/generate_tracked",
6627 });
6628 }
6629
6630 if is_delegated {
6631 let runner = runner::current_inference_runner().ok_or_else(|| {
6635 InferenceError::InferenceFailed(
6636 "model declares ModelSource::Delegated but no inference runner is registered \
6637 (call set_inference_runner / registerInferenceRunner / register_inference_runner)"
6638 .into(),
6639 )
6640 })?;
6641 let (tx, rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6642 let emitter = runner::EventEmitter::new(tx);
6643 let request = req.clone();
6644 tokio::spawn(async move {
6645 if let Err(e) = runner.run(request, emitter).await {
6646 tracing::warn!(error = %e, "delegated inference runner failed");
6647 }
6648 });
6649 return Ok((resolved_model_id, rx));
6650 }
6651
6652 let mut local_reservation = if !is_remote
6654 || schema.as_ref().is_some_and(ModelSchema::is_vllm_mlx)
6655 {
6656 if let Some(local_schema) = schema
6657 .as_ref()
6658 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
6659 {
6660 Some(
6661 self.reserve_local_request_with_worker_retry(local_schema, estimated_footprint)
6662 .await?,
6663 )
6664 } else {
6665 None
6666 }
6667 } else {
6668 None
6669 };
6670
6671 if !is_remote && schema.as_ref().is_some_and(Self::supports_worker_offload) {
6678 if let Some(offload) = crate::offload::current_local_offload() {
6679 let mut offload_req = req.clone();
6680 offload_req.model = Some(resolved_model_id.clone());
6681 let schema_ref = schema
6682 .as_ref()
6683 .ok_or_else(|| InferenceError::ModelNotFound(resolved_model_id.clone()))?;
6684 let reservation = local_reservation.as_mut().ok_or_else(|| {
6685 InferenceError::InferenceFailed(
6686 "local worker stream missing admission reservation".into(),
6687 )
6688 })?;
6689 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_model_id) {
6690 reservation.bind_allocation_id(&allocation_id);
6691 }
6692 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
6693 let offload_stream = offload.stream_admitted(offload_req, admission).await?;
6694 Self::reconcile_worker_residency(
6695 offload.as_ref(),
6696 &resolved_model_id,
6697 &offload_stream.residency,
6698 offload_stream.retention,
6699 reservation,
6700 )
6701 .await?;
6702 let rx = offload_stream.events;
6703 let rx = match local_reservation {
6704 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6705 None => rx,
6706 };
6707 return Ok((resolved_model_id, rx));
6708 }
6709 }
6710
6711 if is_remote {
6712 let mut candidates = vec![schema.unwrap()];
6713 let mut local_fallback_ids = Vec::new();
6714 for fallback_id in &decision.fallbacks {
6715 if let Some(fallback) = routing_registry
6716 .get(fallback_id)
6717 .or_else(|| routing_registry.find_by_name(fallback_id))
6718 {
6719 if (fallback.is_remote() || fallback.is_vllm_mlx())
6720 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6721 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6722 && !candidates
6723 .iter()
6724 .any(|candidate| candidate.id == fallback.id)
6725 {
6726 candidates.push(fallback.clone());
6727 } else if fallback.is_local()
6728 && !fallback.is_vllm_mlx()
6729 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6730 && (!has_vision
6731 || (fallback.has_capability(ModelCapability::Vision)
6732 && !fallback.is_foundation_models()))
6733 {
6734 local_fallback_ids.push(fallback.id.clone());
6735 }
6736 }
6737 }
6738 let mut last_error = None;
6739 let mut fallback_hops: Vec<FallbackFrom> = Vec::new();
6742 for candidate in candidates {
6743 let candidate_id = candidate.id.clone();
6746 let (candidate, mut candidate_reservation) = match self
6747 .vllm_live_schema(candidate, local_reservation.take(), estimated_footprint)
6748 .await
6749 {
6750 Ok(candidate) => candidate,
6751 Err(error) => {
6752 record_fallback_from(&mut fallback_hops, &candidate_id, &error);
6753 last_error = Some(error);
6754 continue;
6755 }
6756 };
6757 self.remote_backend.register_model_keys(&candidate).await;
6758
6759 let spend_guard = self
6760 .spend_limits
6761 .read()
6762 .unwrap()
6763 .as_ref()
6764 .and_then(|limits| limits.per_request_usd)
6765 .map(|budget| {
6766 let mut prompt_tokens =
6767 routing_ext::MidStreamSpendGuard::estimate_tokens(&req.prompt)
6768 + req
6769 .context
6770 .as_deref()
6771 .map(routing_ext::MidStreamSpendGuard::estimate_tokens)
6772 .unwrap_or(0);
6773 prompt_tokens += media_tokens::request_media_and_history_tokens(
6774 req.images.as_deref(),
6775 req.messages.as_deref(),
6776 ) as u64;
6777 if let Some(tools) = &req.tools {
6778 prompt_tokens += routing_ext::MidStreamSpendGuard::estimate_tokens(
6779 &serde_json::to_string(tools).unwrap_or_default(),
6780 );
6781 }
6782 let prices = candidate.cost.prices_for(prompt_tokens as usize);
6783 let input_price = prices
6784 .input_per_mtok
6785 .map(|c| c / 1_000_000.0)
6786 .unwrap_or(0.0);
6787 let output_price = prices
6788 .output_per_mtok
6789 .map(|c| c / 1_000_000.0)
6790 .unwrap_or(0.0);
6791 routing_ext::MidStreamSpendGuard::new(
6792 Some(budget),
6793 prompt_tokens as f64 * input_price,
6794 input_price,
6795 output_price,
6796 )
6797 });
6798
6799 match self
6800 .remote_backend
6801 .with_work_context(req.work_context.clone())
6802 .generate_stream(
6803 &candidate,
6804 &req.prompt,
6805 req.messages.as_deref(),
6806 req.context.as_deref(),
6807 req.params.temperature,
6808 req.params.max_tokens,
6809 req.tools.as_deref(),
6810 req.images.as_deref(),
6811 req.params.tool_choice.as_deref(),
6812 req.params.parallel_tool_calls,
6813 req.response_format.as_ref(),
6814 spend_guard,
6815 )
6816 .await
6817 {
6818 Ok(receiver) => {
6819 let receiver = Self::hold_optional_reservation_for_stream(
6820 receiver,
6821 candidate_reservation.take(),
6822 );
6823 return Ok((candidate.id, receiver));
6824 }
6825 Err(error) => {
6826 tracing::warn!(
6827 model = %candidate.id,
6828 %error,
6829 "remote stream setup failed; trying routed fallback"
6830 );
6831 record_fallback_from(&mut fallback_hops, &candidate.id, &error);
6832 last_error = Some(error);
6833 }
6834 }
6835 }
6836
6837 let mut adoptable_local = Vec::new();
6842 if !req.params.strict_model {
6843 let last_resort = self.first_installed_local_model(has_tools);
6844 if last_resort.is_none() && local_fallback_ids.is_empty() {
6845 adoptable_local = self.cache_only_local_models(has_tools);
6846 }
6847 let mut last_resort_id = None;
6848 if let Some(local) = last_resort {
6849 let schema = routing_registry
6850 .get(&local)
6851 .or_else(|| routing_registry.find_by_name(&local));
6852 let supports_request = schema.is_some_and(|schema| {
6853 !has_vision
6854 || (schema.has_capability(ModelCapability::Vision)
6855 && !schema.is_foundation_models())
6856 });
6857 if supports_request
6858 && !local_fallback_ids.iter().any(|candidate| {
6859 routing_registry
6860 .get(candidate)
6861 .or_else(|| routing_registry.find_by_name(candidate))
6862 .is_some_and(|schema| schema.id == local || schema.name == local)
6863 })
6864 {
6865 last_resort_id = Some(local.clone());
6866 local_fallback_ids.push(local);
6867 }
6868 }
6869 for local_id in local_fallback_ids {
6870 let mut fallback_req = req.clone();
6871 fallback_req.model = Some(local_id.clone());
6872 fallback_req.params.strict_model = true;
6873 match Box::pin(self.generate_stream_raw(fallback_req)).await {
6874 Ok((served_id, rx)) => {
6875 let rx = lead_with_last_resort_substitution(
6876 rx,
6877 &served_id,
6878 last_resort_id.as_deref() == Some(local_id.as_str()),
6879 &fallback_hops,
6880 );
6881 return Ok((served_id, rx));
6882 }
6883 Err(error) => {
6884 tracing::warn!(%error, "local streaming fallback failed");
6885 last_error = Some(error);
6886 }
6887 }
6888 }
6889 }
6890
6891 Err(append_adopt_hint(
6892 last_error.unwrap_or_else(|| {
6893 InferenceError::InferenceFailed(
6894 "no compatible remote streaming model available".to_string(),
6895 )
6896 }),
6897 &adoptable_local,
6898 ))
6899 } else {
6900 let schema =
6901 schema.ok_or_else(|| InferenceError::ModelNotFound(decision.model_id.clone()))?;
6902 let (tx, rx) = tokio::sync::mpsc::channel(64);
6903
6904 #[cfg(any(
6910 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
6911 all(target_os = "ios", target_arch = "aarch64")
6912 ))]
6913 {
6914 if schema.is_foundation_models() {
6915 if Self::request_has_video(&req)
6921 || Self::request_has_audio(&req)
6922 || req.images.as_ref().is_some_and(|imgs| !imgs.is_empty())
6923 || Self::request_has_message_images(&req)
6924 {
6925 return Err(InferenceError::UnsupportedMode {
6926 mode: "multimodal-content",
6927 backend: "foundation-models",
6928 reason: "the FoundationModels stream path takes text only — \
6929 route image/audio/video to a VL model",
6930 });
6931 }
6932 if let Some(tools_defs) = req.tools.clone().filter(|t| !t.is_empty()) {
6938 if req.response_format.is_some() {
6939 tracing::warn!(
6940 "FoundationModels: response_format is ignored when tools \
6941 are present — one turn is either tool-enabled or \
6942 schema-constrained"
6943 );
6944 }
6945 let prompt = req.prompt.clone();
6946 let instructions = req.context.clone();
6947 let fm_tool_choice = req.params.tool_choice.clone();
6948 let max_tokens = req.params.max_tokens as u32;
6949 let temperature = req.params.temperature;
6950 tokio::task::spawn_blocking(move || {
6951 match crate::backend::foundation_models::generate_with_tools(
6952 &prompt,
6953 instructions.as_deref(),
6954 &tools_defs,
6955 fm_tool_choice.as_deref(),
6956 max_tokens,
6957 temperature as f32,
6958 ) {
6959 Ok((text, tool_calls)) => {
6960 if !text.is_empty() {
6961 let _ = tx.blocking_send(stream::StreamEvent::TextDelta(
6962 text.clone(),
6963 ));
6964 }
6965 send_foundation_models_usage(
6971 &tx,
6972 instructions.as_deref(),
6973 &prompt,
6974 &crate::backend::foundation_models::completion_text(
6975 &text,
6976 &tool_calls,
6977 ),
6978 );
6979 let _ = tx.blocking_send(stream::StreamEvent::Done {
6980 text,
6981 tool_calls,
6982 });
6983 }
6984 Err(e) => {
6985 tracing::warn!(
6986 error = %e,
6987 "FoundationModels tool-enabled stream failed"
6988 );
6989 send_stream_failure_blocking(&tx, &e);
6995 }
6996 }
6997 });
6998 return Ok((resolved_model_id, rx));
6999 }
7000 if let Some(crate::tasks::generate::ResponseFormat::JsonSchema {
7005 schema, ..
7006 }) = &req.response_format
7007 {
7008 let prompt = req.prompt.clone();
7009 let instructions = req.context.clone();
7010 let schema_val = schema.clone();
7011 let max_tokens = req.params.max_tokens as u32;
7012 let temperature = req.params.temperature;
7013 tokio::task::spawn_blocking(move || {
7014 match crate::backend::foundation_models::generate_structured(
7015 &prompt,
7016 instructions.as_deref(),
7017 &schema_val,
7018 max_tokens,
7019 temperature as f32,
7020 ) {
7021 Ok(text) => {
7022 if !text.is_empty() {
7023 let _ = tx.blocking_send(stream::StreamEvent::TextDelta(
7024 text.clone(),
7025 ));
7026 }
7027 let _ = tx.blocking_send(stream::StreamEvent::Done {
7028 text,
7029 tool_calls: vec![],
7030 });
7031 }
7032 Err(e) => {
7033 tracing::warn!(
7034 error = %e,
7035 "FoundationModels schema-constrained stream failed"
7036 );
7037 send_stream_failure_blocking(&tx, &e);
7038 }
7039 }
7040 });
7041 return Ok((resolved_model_id, rx));
7042 }
7043 let prompt = req.prompt.clone();
7044 let instructions = foundation_models_instructions(
7045 req.context.as_deref(),
7046 req.response_format.as_ref(),
7047 );
7048 let max_tokens = req.params.max_tokens as u32;
7049 let temperature = req.params.temperature;
7050 let tx_clone = tx.clone();
7051 tokio::task::spawn_blocking(move || {
7052 let accum = std::sync::Arc::new(std::sync::Mutex::new(String::new()));
7059 let accum_cb = accum.clone();
7060 let cb = crate::backend::foundation_models::StreamCallback::new(
7061 move |delta: &str| {
7062 if let Ok(mut g) = accum_cb.lock() {
7063 g.push_str(delta);
7064 }
7065 tx_clone
7066 .blocking_send(stream::StreamEvent::TextDelta(
7067 delta.to_string(),
7068 ))
7069 .is_ok()
7070 },
7071 );
7072 let result = crate::backend::foundation_models::stream(
7073 &prompt,
7074 instructions.as_deref(),
7075 max_tokens,
7076 temperature as f32,
7077 cb,
7078 );
7079 let final_text = accum.lock().map(|g| g.clone()).unwrap_or_default();
7080 match &result {
7084 Ok(()) => {
7085 send_foundation_models_usage(
7086 &tx,
7087 instructions.as_deref(),
7088 &prompt,
7089 &final_text,
7090 );
7091 let _ = tx.blocking_send(stream::StreamEvent::Done {
7092 text: final_text,
7093 tool_calls: vec![],
7094 });
7095 }
7096 Err(e) => send_stream_failure_blocking(&tx, e),
7097 }
7098 result
7099 });
7100 return Ok((resolved_model_id, rx));
7101 }
7102 }
7103
7104 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7106 {
7107 if !schema.is_mlx() {
7110 return Err(InferenceError::InferenceFailed(format!(
7111 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
7112 schema.id
7113 )));
7114 }
7115 let (backend, _retention) = self
7116 .ensure_text_backend(
7117 &schema,
7118 local_reservation
7119 .as_mut()
7120 .expect("local stream has admission reservation"),
7121 )
7122 .await?;
7123 let model_id = schema.id.clone();
7124 let cache = Arc::clone(&self.text_backends);
7125 let device_guard = Self::mlx_device_lock().lock_owned().await;
7132 tokio::task::spawn_blocking(move || {
7138 let _device_guard = device_guard;
7139 let _ = Self::stream_local_mlx(backend, cache, model_id, req, tx);
7140 });
7141 let rx = match local_reservation {
7142 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
7143 None => rx,
7144 };
7145 Ok((resolved_model_id, rx))
7146 }
7147
7148 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7149 {
7150 self.ensure_backend(
7151 &schema,
7152 local_reservation
7153 .as_mut()
7154 .expect("local stream has admission reservation"),
7155 )
7156 .await?;
7157 let backend = self.backend.clone();
7158 let model_id = schema.id.clone();
7159 tokio::spawn(async move {
7160 let _ = Self::stream_local_candle(backend, model_id, req, tx).await;
7161 });
7162 let rx = match local_reservation {
7163 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
7164 None => rx,
7165 };
7166 Ok((resolved_model_id, rx))
7167 }
7168 }
7169 }
7170
7171 pub async fn generate_tracked_stream(
7216 &self,
7217 req: GenerateRequest,
7218 ) -> Result<TrackedStream, InferenceError> {
7219 self.refuse_system_one_generate(&req)?;
7220 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
7224 let start = Instant::now();
7225
7226 let (model_id, mut producer_rx) = self.generate_stream_raw(req).await?;
7230
7231 let trace = {
7232 let mut t = self.outcome_tracker.write().await;
7233 t.record_start(&model_id, InferenceTask::Generate, "stream")
7234 };
7235 let trace_for_return = trace.clone();
7239
7240 let (out_tx, out_rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
7241 let tracker = Arc::clone(&self.outcome_tracker);
7242 tokio::spawn(async move {
7243 let mut acc = stream::StreamAccumulator::default();
7244 let mut stream_error: Option<String> = None;
7245 let mut attribution: Option<outcome::UnattributedReason> = None;
7246 let mut saw_done = false;
7247 let mut receiver_abandoned = false;
7248 while let Some(evt) = producer_rx.recv().await {
7249 if let stream::StreamEvent::Attribution(reason) = &evt {
7250 attribution = Some(*reason);
7252 continue;
7253 }
7254 if let stream::StreamEvent::Error(message) = &evt {
7255 stream_error = Some(message.clone());
7256 }
7257 if matches!(evt, stream::StreamEvent::Done { .. }) {
7258 saw_done = true;
7259 }
7260 acc.push(&evt);
7261 if out_tx.send(evt).await.is_err() {
7262 receiver_abandoned = true;
7263 break;
7264 }
7265 }
7266 if stream_error.is_none() && !saw_done && !receiver_abandoned {
7267 let error = "stream ended without positive provider completion".to_string();
7268 stream_error = Some(error.clone());
7269 let _ = out_tx.send(stream::StreamEvent::Error(error)).await;
7270 } else if stream_error.is_none() && receiver_abandoned {
7271 stream_error = Some("stream receiver was abandoned before completion".to_string());
7275 attribution = Some(outcome::UnattributedReason::Cancelled);
7276 }
7277 let (text, tool_calls, usage, _stop) = acc.finish_with_usage();
7278 let latency_ms = start.elapsed().as_millis() as u64;
7279 let input_tokens = usage
7280 .as_ref()
7281 .map(|u| u.prompt_tokens as usize)
7282 .unwrap_or(estimated_input);
7283 let output_tokens = booked_output_tokens(usage.as_ref(), &text, &tool_calls);
7289 let (cache_read, cache_creation) = usage
7294 .as_ref()
7295 .map(|u| {
7296 (
7297 u.cache_read_input_tokens as usize,
7298 u.cache_creation_input_tokens as usize,
7299 )
7300 })
7301 .unwrap_or((0, 0));
7302 let mut t = tracker.write().await;
7303 if let Some(error) = stream_error {
7304 let reason = attribution.or_else(|| {
7312 stream::content_refusal_tags(&error)
7313 .map(|_| outcome::UnattributedReason::ContentRefused)
7314 });
7315 t.note_elapsed(&trace, latency_ms);
7316 match reason {
7317 Some(reason) => t.record_not_model_fault(&trace, &error, reason),
7318 None => t.record_failure(&trace, &error),
7319 }
7320 } else {
7321 t.record_complete_cached(
7322 &trace,
7323 latency_ms,
7324 input_tokens,
7325 output_tokens,
7326 cache_read,
7327 cache_creation,
7328 );
7329 }
7330 });
7331
7332 Ok(TrackedStream {
7333 model_used: model_id,
7334 trace_id: trace_for_return,
7335 events: out_rx,
7336 })
7337 }
7338
7339 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7340 async fn stream_local_candle(
7341 backend_lock: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
7342 model_id: String,
7343 req: GenerateRequest,
7344 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
7345 ) -> Result<(), InferenceError> {
7346 let mut write = backend_lock.write().await;
7347 let backend = write.get_mut(&model_id).ok_or_else(|| {
7348 InferenceError::InferenceFailed(format!("backend not initialized for {model_id}"))
7349 })?;
7350 backend.clear_kv_cache();
7351
7352 let formatted = tasks::generate::render_chat_prompt(&req);
7353 let tokens = backend.encode(&formatted)?;
7354 let eos = backend.eos_token_id();
7355 let eos_alt = backend.token_id("<|im_end|>");
7356 let params = &req.params;
7357
7358 if tokens.is_empty() {
7359 let _ = tx
7360 .send(stream::StreamEvent::Done {
7361 text: String::new(),
7362 tool_calls: vec![],
7363 })
7364 .await;
7365 return Ok(());
7366 }
7367
7368 let max_ctx = backend.context_length().unwrap_or(32768);
7369 let headroom = params.max_tokens.min(max_ctx / 4);
7370 let max_prompt = max_ctx.saturating_sub(headroom);
7371 let tokens = if tokens.len() > max_prompt {
7372 tokens[tokens.len() - max_prompt..].to_vec()
7373 } else {
7374 tokens
7375 };
7376
7377 let mut generated = Vec::new();
7378 let logits = backend.forward(&tokens, 0)?;
7379 let mut next_token = tasks::generate::sample_token(&logits, params)?;
7380
7381 for _ in 0..params.max_tokens {
7382 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
7383 break;
7384 }
7385
7386 generated.push(next_token);
7387 let delta = backend.decode(&[next_token])?;
7388 if !delta.is_empty()
7389 && tx
7390 .send(stream::StreamEvent::TextDelta(delta))
7391 .await
7392 .is_err()
7393 {
7394 return Ok(());
7395 }
7396
7397 if !params.stop.is_empty() {
7398 let text_so_far = backend.decode(&generated)?;
7399 if params.stop.iter().any(|s| text_so_far.contains(s)) {
7400 break;
7401 }
7402 }
7403
7404 let pos = tokens.len() + generated.len() - 1;
7405 let logits = backend.forward(&[next_token], pos)?;
7406 next_token = tasks::generate::sample_token(&logits, params)?;
7407 }
7408
7409 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
7410 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
7411 let _ = tx
7415 .send(stream::StreamEvent::Usage {
7416 input_tokens: tokens.len() as u64,
7417 output_tokens: generated.len() as u64,
7418 cache_read_input_tokens: 0,
7419 cache_creation_input_tokens: 0,
7420 })
7421 .await;
7422 let _ = tx
7423 .send(stream::StreamEvent::Done {
7424 text,
7425 tool_calls: vec![],
7426 })
7427 .await;
7428 Ok(())
7429 }
7430
7431 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7435 fn stream_local_mlx(
7436 handle: backend_cache::CachedBackend<backend::SwiftLmBackend>,
7437 cache: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
7438 model_id: String,
7439 req: GenerateRequest,
7440 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
7441 ) -> Result<(), InferenceError> {
7442 let mut guard = handle.lock().map_err(|_| {
7443 InferenceError::InferenceFailed(format!("MLX backend mutex poisoned for {model_id}"))
7444 })?;
7445 let backend: &mut backend::SwiftLmBackend = &mut guard;
7446 backend.clear_kv_cache();
7447
7448 let formatted = tasks::generate::render_chat_prompt(&req);
7449 let tokens = backend.encode(&formatted)?;
7450 let eos = backend.eos_token_id();
7451 let eos_alt = backend.token_id("<|im_end|>");
7452 let params = &req.params;
7453
7454 if tokens.is_empty() {
7455 let _ = tx.blocking_send(stream::StreamEvent::Done {
7456 text: String::new(),
7457 tool_calls: vec![],
7458 });
7459 return Ok(());
7460 }
7461
7462 let max_ctx = backend.context_length();
7463 let headroom = params.max_tokens.min(max_ctx / 4);
7464 let max_prompt = max_ctx.saturating_sub(headroom);
7465 let tokens = if tokens.len() > max_prompt {
7466 tokens[tokens.len() - max_prompt..].to_vec()
7467 } else {
7468 tokens
7469 };
7470
7471 let mut generated = Vec::new();
7472
7473 let started = std::time::Instant::now();
7479 let timeout = local_decode_timeout();
7480 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
7481 let mut last_heartbeat = std::time::Duration::ZERO;
7482 tracing::info!(
7483 prompt_tokens = tokens.len(),
7484 max_tokens = params.max_tokens,
7485 timeout_secs = timeout.map(|t| t.as_secs()),
7486 "local stream prefill starting"
7487 );
7488
7489 let logits = match Self::catch_mlx("stream prefill", || backend.forward(&tokens, 0)) {
7494 Ok(v) => v,
7495 Err(e) => {
7496 cache.invalidate(&model_id);
7497 return Err(e);
7498 }
7499 };
7500 let mut next_token = Self::sample_from_logits(&logits, params)?;
7501
7502 for _ in 0..params.max_tokens {
7503 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
7504 break;
7505 }
7506
7507 generated.push(next_token);
7508 let delta = backend.decode(&[next_token])?;
7509 if !delta.is_empty()
7510 && tx
7511 .blocking_send(stream::StreamEvent::TextDelta(delta))
7512 .is_err()
7513 {
7514 return Ok(());
7515 }
7516
7517 if !params.stop.is_empty() {
7518 let text_so_far = backend.decode(&generated)?;
7519 if params.stop.iter().any(|s| text_so_far.contains(s)) {
7520 break;
7521 }
7522 }
7523
7524 let elapsed = started.elapsed();
7529 if deadline_exceeded(elapsed, timeout) {
7530 tracing::warn!(
7531 elapsed_secs = elapsed.as_secs_f64(),
7532 timeout_secs = timeout.map(|t| t.as_secs()),
7533 completion_tokens = generated.len(),
7534 max_tokens = params.max_tokens,
7535 "local stream hit its wall-clock ceiling and was cut short; \
7536 returning what was streamed so far. Raise or disable it with \
7537 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
7538 );
7539 let _ = tx.blocking_send(stream::StreamEvent::StopReason(
7544 LOCAL_DECODE_TIMEOUT_STOP_REASON.to_string(),
7545 ));
7546 break;
7547 }
7548 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7549 last_heartbeat = elapsed;
7550 tracing::info!(
7551 completion_tokens = generated.len(),
7552 max_tokens = params.max_tokens,
7553 elapsed_secs = elapsed.as_secs_f64(),
7554 tokens_per_sec =
7555 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7556 "local stream in progress"
7557 );
7558 }
7559
7560 let pos = tokens.len() + generated.len() - 1;
7561 let logits =
7562 match Self::catch_mlx("stream forward", || backend.forward(&[next_token], pos)) {
7563 Ok(v) => v,
7564 Err(e) => {
7565 cache.invalidate(&model_id);
7566 return Err(e);
7567 }
7568 };
7569 next_token = Self::sample_from_logits(&logits, params)?;
7570 }
7571
7572 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
7573 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
7574 let _ = tx.blocking_send(stream::StreamEvent::Usage {
7582 input_tokens: tokens.len() as u64,
7583 output_tokens: generated.len() as u64,
7584 cache_read_input_tokens: 0,
7586 cache_creation_input_tokens: 0,
7587 });
7588 let _ = tx.blocking_send(stream::StreamEvent::Done {
7589 text,
7590 tool_calls: vec![],
7591 });
7592 Ok(())
7593 }
7594
7595 pub async fn route_context_snapshot(
7597 &self,
7598 prompt: &str,
7599 workload: RoutingWorkload,
7600 has_tools: bool,
7601 has_vision: bool,
7602 ) -> AdaptiveRoutingDecision {
7603 let routing_registry = self.catalog_registry_snapshot();
7604 let tracker = self.outcome_tracker.read().await;
7605 self.adaptive_router.route_context_aware(
7606 prompt,
7607 0,
7608 &routing_registry,
7609 &tracker,
7610 has_tools,
7611 has_vision,
7612 workload,
7613 )
7614 }
7615
7616 pub async fn generate(&self, req: GenerateRequest) -> Result<String, InferenceError> {
7619 Ok(self.generate_tracked(req).await?.text)
7620 }
7621
7622 pub async fn tokenize(&self, model: &str, text: &str) -> Result<Vec<u32>, InferenceError> {
7634 self.assert_local_for_tokenize(model)?;
7635 let admission_schema = self
7636 .resolve_schema(model)
7637 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7638 .clone();
7639 let mut reservation =
7641 self.reserve_local_request(&admission_schema, text.len().div_ceil(4))?;
7642
7643 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7644 {
7645 let schema = admission_schema;
7646 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7647 let guard = handle.lock().map_err(|_| {
7648 InferenceError::InferenceFailed(format!(
7649 "MLX backend mutex poisoned for {}",
7650 schema.id
7651 ))
7652 })?;
7653 let tokens = guard.tokenize_raw(text)?;
7654 Ok(tokens)
7655 }
7656
7657 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7658 {
7659 self.ensure_backend(&admission_schema, &mut reservation)
7660 .await?;
7661 let read = self.backend.read().await;
7662 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7663 InferenceError::InferenceFailed(
7664 "candle backend missing after ensure_backend".to_string(),
7665 )
7666 })?;
7667 let tokens = backend.tokenize_raw(text)?;
7668 Ok(tokens)
7669 }
7670 }
7671
7672 pub async fn detokenize(&self, model: &str, tokens: &[u32]) -> Result<String, InferenceError> {
7674 self.assert_local_for_tokenize(model)?;
7675 let admission_schema = self
7676 .resolve_schema(model)
7677 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7678 .clone();
7679 let mut reservation = self.reserve_local_request(&admission_schema, tokens.len())?;
7681
7682 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7683 {
7684 let schema = admission_schema;
7685 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7686 let guard = handle.lock().map_err(|_| {
7687 InferenceError::InferenceFailed(format!(
7688 "MLX backend mutex poisoned for {}",
7689 schema.id
7690 ))
7691 })?;
7692 let text = guard.detokenize_raw(tokens)?;
7693 Ok(text)
7694 }
7695
7696 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7697 {
7698 self.ensure_backend(&admission_schema, &mut reservation)
7699 .await?;
7700 let read = self.backend.read().await;
7701 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7702 InferenceError::InferenceFailed(
7703 "candle backend missing after ensure_backend".to_string(),
7704 )
7705 })?;
7706 let text = backend.detokenize_raw(tokens)?;
7707 Ok(text)
7708 }
7709 }
7710
7711 fn assert_local_for_tokenize(&self, model: &str) -> Result<(), InferenceError> {
7715 if let Some(schema) = self.resolve_schema(model) {
7716 if !schema.is_local() {
7717 return Err(InferenceError::UnsupportedMode {
7718 mode: "tokenize/detokenize",
7719 backend: "remote",
7720 reason: "remote provider tokenizer is not exposed by the runtime; \
7721 use a local model (Qwen3 GGUF / MLX) for tokenizer-correctness checks",
7722 });
7723 }
7724 }
7725 Ok(())
7727 }
7728
7729 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7735 fn catch_mlx<F, T>(context: &str, f: F) -> Result<T, InferenceError>
7736 where
7737 F: FnOnce() -> Result<T, InferenceError>,
7738 {
7739 std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|e| {
7740 InferenceError::InferenceFailed(format!("MLX panicked during {context}: {e:?}"))
7741 })?
7742 }
7743
7744 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7754 fn drive_generation(
7755 backend: &mut dyn backend::local::TextDecoder,
7756 prompt: &str,
7757 params: &GenerateParams,
7758 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7759 Self::drive_generation_with_timeout(backend, prompt, params, local_decode_timeout())
7760 }
7761
7762 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7766 fn drive_generation_with_timeout(
7767 backend: &mut dyn backend::local::TextDecoder,
7768 prompt: &str,
7769 params: &GenerateParams,
7770 timeout: Option<std::time::Duration>,
7771 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7772 use backend::local::{DriveError, LocalGeneration};
7773
7774 let start = std::time::Instant::now();
7775
7776 let tokens = backend.encode(prompt).map_err(DriveError::Recoverable)?;
7777 let eos_ids = backend.eos_ids();
7778
7779 if tokens.is_empty() {
7780 backend.clear_kv_cache();
7781 return Ok(LocalGeneration {
7782 text: String::new(),
7783 ttft_ms: None,
7784 stop_reason: None,
7785 prompt_tokens: 0,
7786 completion_tokens: 0,
7787 });
7788 }
7789
7790 let max_ctx = backend.context_length();
7792 let headroom = params.max_tokens.min(max_ctx / 4);
7793 let max_prompt = max_ctx.saturating_sub(headroom);
7794 let tokens = if tokens.len() > max_prompt {
7795 tokens[tokens.len() - max_prompt..].to_vec()
7796 } else {
7797 tokens
7798 };
7799
7800 let mut generated = Vec::new();
7801
7802 tracing::info!(
7809 prompt_tokens = tokens.len(),
7810 max_tokens = params.max_tokens,
7811 timeout_secs = timeout.map(|t| t.as_secs()),
7812 "local prefill starting"
7813 );
7814
7815 let offset = backend.begin_prompt(&tokens);
7820 let logits = Self::catch_mlx("prefill", || backend.forward(&tokens[offset..], offset))
7821 .map_err(DriveError::BackendCorrupted)?;
7822 let mut next_token =
7823 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7824 let ttft_ms = Some(start.elapsed().as_millis() as u64);
7825
7826 tracing::info!(
7831 prompt_tokens = tokens.len(),
7832 max_tokens = params.max_tokens,
7833 prefill_ms = ttft_ms,
7834 timeout_secs = timeout.map(|t| t.as_secs()),
7835 "local decode starting"
7836 );
7837
7838 let mut natural_stop = false;
7841 let mut timed_out = false;
7842 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
7843 let mut last_heartbeat = std::time::Duration::ZERO;
7844 for _ in 0..params.max_tokens {
7845 if eos_ids.contains(&next_token) {
7846 natural_stop = true;
7847 break;
7848 }
7849
7850 generated.push(next_token);
7851
7852 if !params.stop.is_empty() {
7853 let text_so_far = backend
7854 .decode(&generated)
7855 .map_err(DriveError::Recoverable)?;
7856 if params.stop.iter().any(|s| text_so_far.contains(s)) {
7857 natural_stop = true;
7858 break;
7859 }
7860 }
7861
7862 let elapsed = start.elapsed();
7863 if deadline_exceeded(elapsed, timeout) {
7864 timed_out = true;
7865 break;
7866 }
7867 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7868 last_heartbeat = elapsed;
7869 tracing::info!(
7870 completion_tokens = generated.len(),
7871 max_tokens = params.max_tokens,
7872 elapsed_secs = elapsed.as_secs_f64(),
7873 tokens_per_sec =
7874 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7875 "local decode in progress"
7876 );
7877 }
7878
7879 let pos = tokens.len() + generated.len() - 1;
7880 let logits = Self::catch_mlx("forward", || backend.forward(&[next_token], pos))
7881 .map_err(DriveError::BackendCorrupted)?;
7882 next_token =
7883 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7884 }
7885
7886 let elapsed = start.elapsed();
7887 let tokens_per_sec = generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON);
7888 if timed_out {
7889 tracing::warn!(
7890 elapsed_secs = elapsed.as_secs_f64(),
7891 timeout_secs = timeout.map(|t| t.as_secs()),
7892 completion_tokens = generated.len(),
7893 max_tokens = params.max_tokens,
7894 tokens_per_sec,
7895 "local decode hit its wall-clock ceiling and was cut short; \
7896 returning the partial response. Raise or disable it with \
7897 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
7898 );
7899 } else {
7900 tracing::info!(
7901 elapsed_secs = elapsed.as_secs_f64(),
7902 completion_tokens = generated.len(),
7903 max_tokens = params.max_tokens,
7904 tokens_per_sec,
7905 natural_stop,
7906 "local decode finished"
7907 );
7908 }
7909
7910 let decoded = backend
7911 .decode(&generated)
7912 .map_err(DriveError::Recoverable)?;
7913 let text = tasks::generate::truncate_at_stop(&decoded, ¶ms.stop);
7914 let stop_reason = Some(
7915 if timed_out {
7916 LOCAL_DECODE_TIMEOUT_STOP_REASON
7917 } else if natural_stop {
7918 "stop"
7919 } else {
7920 "length"
7921 }
7922 .to_string(),
7923 );
7924 Ok(LocalGeneration {
7925 text: tasks::generate::strip_thinking(&text, params.thinking),
7926 ttft_ms,
7927 stop_reason,
7928 prompt_tokens: tokens.len(),
7929 completion_tokens: generated.len(),
7930 })
7931 }
7932
7933 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7936 async fn generate_mlx(
7946 &self,
7947 req: GenerateRequest,
7948 model_id: &str,
7949 reservation: &mut resource_policy::LocalLoadReservation,
7950 ) -> Result<(String, Option<TokenUsage>, Option<u64>, Option<String>), InferenceError> {
7951 let schema = self
7952 .unified_registry()
7953 .get(model_id)
7954 .cloned()
7955 .ok_or_else(|| {
7956 InferenceError::InferenceFailed(format!(
7957 "generate_mlx: unknown schema id {model_id}"
7958 ))
7959 })?;
7960 let (handle, _retention) = self.ensure_text_backend(&schema, reservation).await?;
7961 let formatted = tasks::generate::render_chat_prompt(&req);
7968 let params = req.params.clone();
7969 let model_id = model_id.to_string();
7970 let cache = self.text_backends.clone();
7971 let device_guard = Self::mlx_device_lock().lock_owned().await;
7972 let detached_lease = reservation.detached_lease();
7973
7974 run_admitted_blocking(detached_lease, move || {
7980 let _device_guard = device_guard;
7981 let mut guard = handle.lock().map_err(|_| {
7982 InferenceError::InferenceFailed(format!(
7983 "MLX backend mutex poisoned for {model_id}"
7984 ))
7985 })?;
7986 let backend: &mut backend::SwiftLmBackend = &mut guard;
7987 let ctx_window = backend.context_length() as u64;
7988 match Self::drive_generation(backend, &formatted, ¶ms) {
7989 Ok(gen) => {
7990 let usage = TokenUsage {
7991 prompt_tokens: gen.prompt_tokens as u64,
7992 completion_tokens: gen.completion_tokens as u64,
7993 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7994 context_window: ctx_window,
7995 ..Default::default()
7996 };
7997 Ok((gen.text, Some(usage), gen.ttft_ms, gen.stop_reason))
7998 }
7999 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
8000 Err(backend::local::DriveError::BackendCorrupted(error)) => {
8001 drop(guard);
8002 cache.invalidate(&model_id);
8003 Err(error)
8004 }
8005 }
8006 })
8007 .await
8008 .map_err(|error| {
8009 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
8010 })?
8011 }
8012
8013 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8018 async fn generate_local(
8019 &self,
8020 req: GenerateRequest,
8021 model_id: &str,
8022 reservation: &mut resource_policy::LocalLoadReservation,
8023 ) -> Result<
8024 (
8025 String,
8026 Vec<tasks::generate::ToolCall>,
8027 Option<TokenUsage>,
8028 Option<u64>,
8029 Option<String>,
8030 ),
8031 InferenceError,
8032 > {
8033 let schema = self
8034 .unified_registry()
8035 .get(model_id)
8036 .cloned()
8037 .ok_or_else(|| {
8038 InferenceError::InferenceFailed(format!(
8039 "generate_local: unknown schema id {model_id}"
8040 ))
8041 })?;
8042 let (handle, _retention) = self.ensure_local_backend(&schema, reservation).await?;
8043 let params = req.params.clone();
8044 let model_id = model_id.to_string();
8045 let cache = self.local_backends.clone();
8046 let device_guard = Self::mlx_device_lock().lock_owned().await;
8047 let detached_lease = reservation.detached_lease();
8048
8049 run_admitted_blocking(detached_lease, move || {
8050 let _device_guard = device_guard;
8051 let mut guard = handle.lock().map_err(|_| {
8052 InferenceError::InferenceFailed(format!(
8053 "local backend mutex poisoned for {model_id}"
8054 ))
8055 })?;
8056 let formatted = guard.render_prompt(&req)?;
8057 let backend: &mut dyn backend::local::TextDecoder = &mut **guard;
8058 let outcome = Self::drive_generation(backend, &formatted, ¶ms);
8059 match outcome {
8060 Ok(gen) => {
8061 let (clean, tool_calls) = guard.parse_tool_calls(&gen.text);
8062 let usage = TokenUsage {
8063 prompt_tokens: gen.prompt_tokens as u64,
8064 completion_tokens: gen.completion_tokens as u64,
8065 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
8066 context_window: guard.context_length() as u64,
8067 ..Default::default()
8068 };
8069 Ok((clean, tool_calls, Some(usage), gen.ttft_ms, gen.stop_reason))
8070 }
8071 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
8072 Err(backend::local::DriveError::BackendCorrupted(error)) => {
8073 drop(guard);
8074 cache.invalidate(&model_id);
8075 Err(error)
8076 }
8077 }
8078 })
8079 .await
8080 .map_err(|error| {
8081 InferenceError::InferenceFailed(format!("native local task panicked: {error}"))
8082 })?
8083 }
8084
8085 #[allow(dead_code)] fn apply_top_k_top_p(probs: &mut [f32], top_k: usize, top_p: f64) {
8093 if top_k > 0 && top_k < probs.len() {
8095 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
8096 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
8097 let allowed: std::collections::HashSet<usize> =
8098 indexed[..top_k].iter().map(|(i, _)| *i).collect();
8099 for (i, p) in probs.iter_mut().enumerate() {
8100 if !allowed.contains(&i) {
8101 *p = 0.0;
8102 }
8103 }
8104 let sum: f32 = probs.iter().sum();
8105 if sum > 0.0 {
8106 for p in probs.iter_mut() {
8107 *p /= sum;
8108 }
8109 }
8110 }
8111
8112 if top_p < 1.0 {
8114 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
8115 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
8116 let mut cumsum = 0.0f32;
8117 let mut cutoff_idx = indexed.len();
8118 for (i, &(_, p)) in indexed.iter().enumerate() {
8119 cumsum += p;
8120 if cumsum > top_p as f32 {
8121 cutoff_idx = i + 1;
8122 break;
8123 }
8124 }
8125 let allowed: std::collections::HashSet<usize> =
8126 indexed[..cutoff_idx].iter().map(|(i, _)| *i).collect();
8127 for (i, p) in probs.iter_mut().enumerate() {
8128 if !allowed.contains(&i) {
8129 *p = 0.0;
8130 }
8131 }
8132 let sum: f32 = probs.iter().sum();
8133 if sum > 0.0 {
8134 for p in probs.iter_mut() {
8135 *p /= sum;
8136 }
8137 }
8138 }
8139 }
8140
8141 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8143 fn sample_from_logits(logits: &[f32], params: &GenerateParams) -> Result<u32, InferenceError> {
8144 if params.temperature <= 0.0 {
8145 let (idx, _) = logits
8147 .iter()
8148 .enumerate()
8149 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
8150 .ok_or_else(|| InferenceError::InferenceFailed("empty logits".into()))?;
8151 return Ok(idx as u32);
8152 }
8153
8154 let temp = params.temperature as f32;
8156 let max_logit = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
8157 let mut probs: Vec<f32> = logits
8158 .iter()
8159 .map(|&l| ((l - max_logit) / temp).exp())
8160 .collect();
8161 let sum: f32 = probs.iter().sum();
8162 for p in &mut probs {
8163 *p /= sum;
8164 }
8165
8166 Self::apply_top_k_top_p(&mut probs, params.top_k, params.top_p);
8172
8173 use rand::Rng;
8175 let mut rng = rand::rng();
8176 let r: f32 = rng.random();
8177 let mut cumsum = 0.0;
8178 for (i, &p) in probs.iter().enumerate() {
8179 cumsum += p;
8180 if cumsum >= r {
8181 return Ok(i as u32);
8182 }
8183 }
8184 Ok((probs.len() - 1) as u32)
8185 }
8186
8187 pub async fn embed(&self, req: EmbedRequest) -> Result<Vec<Vec<f32>>, InferenceError> {
8190 let instruction = req
8191 .instruction
8192 .as_deref()
8193 .unwrap_or("Retrieve relevant memory facts");
8194 let embedding_model = self
8195 .preferred_model_for_capability(ModelCapability::Embed)
8196 .unwrap_or(&self.config.embedding_model);
8197 let admission_schema = self
8198 .resolve_schema(embedding_model)
8199 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model.to_string()))?
8200 .clone();
8201 let estimated_tokens = req.texts.iter().map(|text| text.len().div_ceil(4)).sum();
8202 let mut reservation = self.reserve_local_request(&admission_schema, estimated_tokens)?;
8204
8205 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8206 {
8207 let model_id = self.ensure_mlx_embedding_backend().await?;
8208 let schema = self
8209 .unified_registry()
8210 .get(&model_id)
8211 .cloned()
8212 .ok_or_else(|| {
8213 InferenceError::InferenceFailed(format!("embed: unknown schema id {model_id}"))
8214 })?;
8215 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
8216 let _device_guard = Self::mlx_device_lock().lock_owned().await;
8220 let mut guard = handle.lock().map_err(|_| {
8221 InferenceError::InferenceFailed(format!(
8222 "MLX embedding backend mutex poisoned for {model_id}"
8223 ))
8224 })?;
8225 let backend: &mut backend::SwiftLmBackend = &mut guard;
8226
8227 let mut results = Vec::with_capacity(req.texts.len());
8228 for text in &req.texts {
8229 let embedding = if req.is_query {
8230 backend.embed_query(text, instruction)?
8231 } else {
8232 backend.embed_one(text)?
8233 };
8234 results.push(embedding);
8235 }
8236 Ok(results)
8237 }
8238
8239 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8240 {
8241 self.ensure_embedding_backend(&mut reservation).await?;
8242 let mut write = self.embedding_backend.write().await;
8243 let backend = write.as_mut().unwrap();
8244
8245 let mut results = Vec::with_capacity(req.texts.len());
8246 for text in &req.texts {
8247 let embedding = if req.is_query {
8248 backend.embed_query(text, instruction)?
8249 } else {
8250 backend.embed_one(text)?
8251 };
8252 results.push(embedding);
8253 }
8254 Ok(results)
8255 }
8256 }
8257
8258 pub async fn rerank(&self, req: RerankRequest) -> Result<RerankResult, InferenceError> {
8292 if req.documents.is_empty() {
8293 return Ok(RerankResult {
8294 ranked: Vec::new(),
8295 model_used: None,
8296 });
8297 }
8298
8299 let model_name = match req.model.clone() {
8300 Some(m) => m,
8301 None => self
8302 .preferred_model_for_capability(ModelCapability::Rerank)
8303 .map(str::to_string)
8304 .ok_or_else(|| {
8305 InferenceError::InferenceFailed(
8306 "no reranker model available — pull a Qwen3-Reranker model first".into(),
8307 )
8308 })?,
8309 };
8310
8311 let schema = self.resolve_schema_name_first(&model_name).ok_or_else(|| {
8312 InferenceError::InferenceFailed(format!("rerank: unknown reranker model {model_name}"))
8313 })?;
8314 if !schema.has_capability(ModelCapability::Rerank) {
8315 return Err(InferenceError::InferenceFailed(format!(
8316 "model {} does not declare the Rerank capability",
8317 schema.name
8318 )));
8319 }
8320
8321 let instruction = req.instruction.as_deref().unwrap_or(
8322 "Given a web search query, retrieve relevant passages that answer the query",
8323 );
8324
8325 let mut scored: Vec<RerankedDocument> = Vec::with_capacity(req.documents.len());
8326 for (idx, doc) in req.documents.iter().enumerate() {
8327 let prompt = rerank_prompt(instruction, &req.query, doc);
8328 let gen_req = GenerateRequest {
8329 prompt,
8330 model: Some(schema.id.clone()),
8331 params: tasks::generate::GenerateParams {
8332 temperature: 0.0,
8333 max_tokens: 3,
8337 thinking: tasks::generate::ThinkingMode::Off,
8338 ..Default::default()
8339 },
8340 context: None,
8341 context_stable_prefix: None,
8342 tools: None,
8343 images: None,
8344 messages: None,
8345 cache_control: false,
8346 response_format: None,
8347 intent: None,
8348 client_ref: None,
8349 expected_row_digest: None,
8350 expected_catalog_revision: None,
8351 caller: None,
8352 work_context: None,
8353 };
8354 let out = self.generate(gen_req).await?;
8355 let score = score_from_rerank_output(&out, &schema.name);
8356 scored.push(RerankedDocument {
8357 index: idx,
8358 score,
8359 document: doc.clone(),
8360 });
8361 }
8362
8363 scored.sort_by(|a, b| {
8366 b.score
8367 .partial_cmp(&a.score)
8368 .unwrap_or(std::cmp::Ordering::Equal)
8369 .then_with(|| a.index.cmp(&b.index))
8370 });
8371 if let Some(n) = req.top_n {
8372 scored.truncate(n);
8373 }
8374
8375 Ok(RerankResult {
8376 ranked: scored,
8377 model_used: Some(schema.name),
8378 })
8379 }
8380
8381 pub async fn ground(&self, req: GroundRequest) -> Result<GroundResult, InferenceError> {
8391 let model_name = match req.model.clone() {
8392 Some(m) => m,
8393 None => self
8394 .preferred_model_for_capability(ModelCapability::Grounding)
8395 .map(str::to_string)
8396 .ok_or_else(|| {
8397 InferenceError::InferenceFailed(
8398 "no grounding-capable model available — pull a Qwen2.5-VL model first"
8399 .into(),
8400 )
8401 })?,
8402 };
8403
8404 let gen_req = GenerateRequest {
8405 prompt: req.prompt.clone(),
8406 model: Some(model_name),
8407 params: GenerateParams::default(),
8408 context: None,
8409 context_stable_prefix: None,
8410 tools: None,
8411 images: Some(vec![req.image.clone()]),
8412 messages: None,
8413 cache_control: false,
8414 response_format: None,
8415 intent: None,
8416 client_ref: None,
8417 expected_row_digest: None,
8418 expected_catalog_revision: None,
8419 caller: None,
8420 work_context: None,
8421 };
8422 let result = self.generate_tracked(gen_req).await?;
8423 Ok(GroundResult {
8424 boxes: result.bounding_boxes,
8425 raw_text: result.text,
8426 model_used: Some(result.model_used),
8427 })
8428 }
8429
8430 pub async fn classify(
8433 &self,
8434 req: ClassifyRequest,
8435 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8436 let registry = if let Some(context) = &req.work_context {
8437 Arc::new(
8438 self.work_context_routing_snapshot(context, req.model.as_deref())
8439 .await
8440 .0,
8441 )
8442 } else {
8443 self.unified_registry()
8444 };
8445 let model = match req.model.clone().or_else(|| {
8446 self.preferred_model_for_capability(ModelCapability::Classify)
8447 .map(str::to_string)
8448 }) {
8449 Some(m) => m,
8450 None => match default_system_one_classifier(registry.all()) {
8454 Some(m) => {
8455 debug!(model = %m, "classify defaulted to an available System One model");
8456 m
8457 }
8458 None => {
8459 let m = self.router.route_small(&self.registry);
8460 debug!(model = %m, "auto-routed classify request");
8461 m
8462 }
8463 },
8464 };
8465 let schema = self
8466 .resolve_schema(&model)
8467 .ok_or_else(|| InferenceError::ModelNotFound(model.clone()))?
8468 .clone();
8469
8470 if let ModelSource::Proprietary { protocol, .. } = &schema.source {
8471 if protocol.wire == ProprietaryWire::SystemOne {
8472 return self.classify_via_system_one(req, &schema).await;
8473 }
8474 }
8475 if !schema.is_local() {
8476 return self.classify_via_generate(req, &schema.id).await;
8477 }
8478
8479 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8482 {
8483 return self.classify_via_generate(req, &model).await;
8484 }
8485
8486 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8487 {
8488 let mut reservation =
8490 self.reserve_local_request(&schema, req.text.len().div_ceil(4))?;
8491 self.ensure_backend(&schema, &mut reservation).await?;
8492 let mut write = self.backend.write().await;
8493 let backend = write.get_mut(&schema.id).ok_or_else(|| {
8494 InferenceError::InferenceFailed(format!(
8495 "candle backend missing after ensure_backend for {}",
8496 schema.id
8497 ))
8498 })?;
8499 let result = tasks::classify::classify(backend, req).await?;
8500 Ok(result)
8501 }
8502 }
8503
8504 fn refuse_system_one_generate(&self, req: &GenerateRequest) -> Result<(), InferenceError> {
8513 let Some(model) = req.model.as_deref() else {
8514 return Ok(());
8515 };
8516 let model = model.strip_prefix(EXACT_MODEL_ID_PREFIX).unwrap_or(model);
8517 let schema = self.resolve_schema(model);
8518 match schema.as_ref().map(|s| &s.source) {
8519 Some(ModelSource::Proprietary { protocol, .. })
8520 if protocol.wire == ProprietaryWire::SystemOne =>
8521 {
8522 Err(InferenceError::UnsupportedMode {
8523 mode: "generate",
8524 backend: "parslee-system-one",
8525 reason: "a System One model returns typed decisions, not text; use classify",
8526 })
8527 }
8528 _ => Ok(()),
8529 }
8530 }
8531
8532 pub async fn option_probabilities(
8551 &self,
8552 model: &str,
8553 prompt: &str,
8554 options: &[String],
8555 ) -> Result<OptionProbabilities, InferenceError> {
8556 if options.is_empty() {
8557 return Err(InferenceError::InvalidClassifyLabels(
8558 "no options given".into(),
8559 ));
8560 }
8561 let schema = self
8562 .resolve_schema(model)
8563 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
8564 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8565 {
8566 let _ = (schema, prompt);
8567 Err(InferenceError::UnsupportedMode {
8568 mode: "option-logits",
8569 backend: "local",
8570 reason: "option probabilities read logits from the native MLX text backend, \
8571 which is Apple Silicon only",
8572 })
8573 }
8574 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8575 {
8576 if !schema.is_local() || !schema.family.to_lowercase().starts_with("qwen") {
8577 return Err(InferenceError::UnsupportedMode {
8578 mode: "option-logits",
8579 backend: "local",
8580 reason: "option probabilities need a local Qwen-family model: the prompt is \
8581 rendered with Qwen's chat template",
8582 });
8583 }
8584 let req = GenerateRequest {
8585 prompt: prompt.to_string(),
8586 model: Some(schema.id.clone()),
8587 params: tasks::generate::GenerateParams {
8588 thinking: tasks::generate::ThinkingMode::Off,
8589 ..Default::default()
8590 },
8591 ..Default::default()
8592 };
8593 let formatted = tasks::generate::render_chat_prompt(&req);
8594 let mut reservation =
8595 self.reserve_local_request(&schema, formatted.len().div_ceil(4))?;
8596 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
8597 let options = options.to_vec();
8598 let model_id = schema.id.clone();
8599 let device_guard = Self::mlx_device_lock().lock_owned().await;
8600 let detached_lease = reservation.detached_lease();
8601 run_admitted_blocking(detached_lease, move || {
8602 let _device_guard = device_guard;
8603 let mut guard = handle.lock().map_err(|_| {
8604 InferenceError::InferenceFailed(format!(
8605 "MLX backend mutex poisoned for {model_id}"
8606 ))
8607 })?;
8608 let backend: &mut backend::SwiftLmBackend = &mut guard;
8609 let result = tasks::classify::score_options(backend, &formatted, &options);
8610 backend.clear_kv_cache();
8611 result
8612 })
8613 .await
8614 .map_err(|error| {
8615 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
8616 })?
8617 }
8618 }
8619
8620 async fn classify_via_system_one(
8625 &self,
8626 req: ClassifyRequest,
8627 schema: &ModelSchema,
8628 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8629 let body = tasks::classify::system_one_request_body(&schema.name, &req.text, &req.labels)?;
8630 let trace = {
8633 let mut tracker = self.outcome_tracker.write().await;
8634 tracker.record_start(
8635 &schema.id,
8636 InferenceTask::Classify,
8637 "explicit system-one model",
8638 )
8639 };
8640 let start = Instant::now();
8641 let result = match self
8642 .remote_backend
8643 .with_work_context(req.work_context.clone())
8644 .parslee_system_one(schema, &body)
8645 .await
8646 {
8647 Ok(response) => tasks::classify::system_one_results(&response, &req.labels)
8648 .map(|results| (results, response)),
8649 Err(error) => Err(error),
8650 };
8651 let mut tracker = self.outcome_tracker.write().await;
8652 match result {
8653 Ok((results, response)) => {
8654 let usage = |key: &str| {
8655 response
8656 .pointer(&format!("/usage/{key}"))
8657 .and_then(|t| t.as_u64())
8658 .unwrap_or(0) as usize
8659 };
8660 tracker.record_complete(
8663 &trace,
8664 start.elapsed().as_millis() as u64,
8665 usage("input_tokens"),
8666 usage("output_tokens").max(1),
8667 );
8668 Ok(results)
8669 }
8670 Err(error) => {
8671 record_dispatch_failure(&mut tracker, &trace, &error);
8672 Err(error)
8673 }
8674 }
8675 }
8676
8677 async fn classify_via_generate(
8678 &self,
8679 req: ClassifyRequest,
8680 model: &str,
8681 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8682 let labels_str = req
8683 .labels
8684 .iter()
8685 .enumerate()
8686 .map(|(i, l)| format!("{}. {}", i + 1, l))
8687 .collect::<Vec<_>>()
8688 .join("\n");
8689
8690 let prompt = format!(
8691 "Classify the following text into one of these categories:\n\
8692 {labels_str}\n\n\
8693 Text: {}\n\n\
8694 Respond with ONLY the category name, nothing else.",
8695 req.text
8696 );
8697
8698 let gen_req = GenerateRequest {
8699 prompt,
8700 model: Some(model.to_string()),
8701 params: tasks::generate::GenerateParams {
8702 temperature: 0.0,
8703 max_tokens: 32,
8704 thinking: tasks::generate::ThinkingMode::Off,
8707 ..Default::default()
8708 },
8709 context: None,
8710 context_stable_prefix: None,
8711 tools: None,
8712 images: None,
8713 messages: None,
8714 cache_control: false,
8715 response_format: None,
8716 intent: None,
8717 client_ref: None,
8718 expected_row_digest: None,
8719 expected_catalog_revision: None,
8720 caller: None,
8721 work_context: req.work_context.clone(),
8722 };
8723
8724 let response = self.generate(gen_req).await?;
8725 tasks::classify::score_reply(&response, &req.labels)
8726 }
8727
8728 pub async fn transcribe(
8730 &self,
8731 req: TranscribeRequest,
8732 ) -> Result<TranscribeResult, InferenceError> {
8733 let candidates =
8734 self.speech_candidates(ModelCapability::SpeechToText, req.model.as_deref())?;
8735 let mut last_error = None;
8736 let mut local_resource_block = None;
8737
8738 for schema in candidates {
8739 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8741 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8742 SpeechCandidateAdmission::SkipBlocked(error) => {
8743 local_resource_block = Some(error.to_string());
8744 last_error = Some(error);
8745 continue;
8746 }
8747 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8748 };
8749 let result = match &schema.source {
8750 ModelSource::Mlx { .. } => {
8751 self.transcribe_local_mlx(&schema, &req, reservation.as_mut())
8752 .await
8753 }
8754 ModelSource::WhisperCpp { model } => {
8755 self.transcribe_whisper(&schema, model, &req, reservation.as_mut())
8756 .await
8757 }
8758 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8759 self.transcribe_elevenlabs(&schema, &req).await
8760 }
8761 _ => Err(InferenceError::InferenceFailed(format!(
8762 "speech-to-text not implemented for model source: {}",
8763 schema.id
8764 ))),
8765 };
8766
8767 match result {
8768 Ok(mut result) => {
8769 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8770 if let Some(reason) = local_resource_block.take() {
8771 result.routing_explanation = Some(format!(
8772 "Local speech recognition was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8773 ));
8774 }
8775 }
8776 return Ok(result);
8777 }
8778 Err(err) if req.model.is_some() => return Err(err),
8779 Err(err) => last_error = Some(err),
8780 }
8781 }
8782
8783 Err(last_error.unwrap_or_else(|| {
8784 InferenceError::InferenceFailed("no speech-to-text models available".into())
8785 }))
8786 }
8787
8788 pub async fn synthesize(
8790 &self,
8791 req: SynthesizeRequest,
8792 ) -> Result<SynthesizeResult, InferenceError> {
8793 let candidates =
8794 self.speech_candidates(ModelCapability::TextToSpeech, req.model.as_deref())?;
8795 let mut last_error = None;
8796 let mut local_resource_block = None;
8797
8798 for schema in candidates {
8799 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8801 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8802 SpeechCandidateAdmission::SkipBlocked(error) => {
8803 local_resource_block = Some(error.to_string());
8804 last_error = Some(error);
8805 continue;
8806 }
8807 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8808 };
8809 let result = match &schema.source {
8810 ModelSource::Mlx { .. } => {
8811 self.synthesize_local_mlx(&schema, &req, reservation.as_mut())
8812 .await
8813 }
8814 ModelSource::WindowsSpeech {} => {
8815 self.synthesize_windows_speech(&schema, &req).await
8816 }
8817 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8818 self.synthesize_elevenlabs(&schema, &req).await
8819 }
8820 _ => Err(InferenceError::InferenceFailed(format!(
8821 "text-to-speech not implemented for model source: {}",
8822 schema.id
8823 ))),
8824 };
8825
8826 match result {
8827 Ok(mut result) => {
8828 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8829 if let Some(reason) = local_resource_block.take() {
8830 result.routing_explanation = Some(format!(
8831 "Local speech synthesis was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8832 ));
8833 }
8834 }
8835 return Ok(result);
8836 }
8837 Err(err) if req.model.is_some() => return Err(err),
8838 Err(err) => last_error = Some(err),
8839 }
8840 }
8841
8842 Err(last_error.unwrap_or_else(|| {
8843 InferenceError::InferenceFailed("no text-to-speech models available".into())
8844 }))
8845 }
8846
8847 async fn synthesize_windows_speech(
8853 &self,
8854 schema: &ModelSchema,
8855 req: &SynthesizeRequest,
8856 ) -> Result<SynthesizeResult, InferenceError> {
8857 #[cfg(target_os = "windows")]
8858 {
8859 let text = req.text.clone();
8860 let voice = req.voice.clone().unwrap_or_default();
8861 let rate = req.speed.unwrap_or(1.0) as f64;
8862 let bytes =
8863 tokio::task::spawn_blocking(move || winrt_synthesize_wav(&text, &voice, rate))
8864 .await
8865 .map_err(|e| {
8866 InferenceError::InferenceFailed(format!("winrt tts join: {e}"))
8867 })??;
8868 let dest = requested_or_temp_output(req.output_path.as_deref(), "wav")?;
8869 ensure_parent_dir(&dest)?;
8870 std::fs::write(&dest, &bytes)?;
8871 Ok(SynthesizeResult {
8872 audio_path: dest.to_string_lossy().to_string(),
8873 media_type: "audio/wav".to_string(),
8874 model_used: Some(schema.name.clone()),
8875 voice_used: req.voice.clone(),
8876 routing_explanation: None,
8877 })
8878 }
8879 #[cfg(not(target_os = "windows"))]
8880 {
8881 let _ = (schema, req);
8882 Err(InferenceError::InferenceFailed(
8883 "Windows OS TTS is only available on Windows".into(),
8884 ))
8885 }
8886 }
8887
8888 pub async fn generate_image(
8890 &self,
8891 req: GenerateImageRequest,
8892 ) -> Result<GenerateImageResult, InferenceError> {
8893 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8910 {
8911 use crate::backend::external_flux;
8912 tracing::info!(
8913 model = req.model.as_deref().unwrap_or("<default>"),
8914 "routing image generation to mflux"
8915 );
8916 let schema = self
8917 .media_generation_candidates(
8918 ModelCapability::ImageGeneration,
8919 req.model.as_deref(),
8920 )?
8921 .into_iter()
8922 .next()
8923 .ok_or_else(|| {
8924 InferenceError::InferenceFailed("no image generation models available".into())
8925 })?;
8926 let mut reservation = self.reserve_local_request(&schema, 0)?;
8928 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8929 reservation
8930 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8931 .map_err(InferenceError::from)?;
8932 let mut req = req;
8933 req.model = self
8934 .resolve_external_hf_repo(req.model.as_deref(), ModelCapability::ImageGeneration);
8935 external_flux::generate_image(&req)
8936 }
8937
8938 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8943 {
8944 let _ = &req;
8945 Err(InferenceError::InferenceFailed(
8946 "local image generation requires Apple Silicon (mflux)".into(),
8947 ))
8948 }
8949 }
8950
8951 pub async fn generate_image_batch(
8967 &self,
8968 req: GenerateImageRequest,
8969 ) -> Result<Vec<GenerateImageResult>, InferenceError> {
8970 let count = req.variant_count.unwrap_or(1).max(1);
8971 if count == 1 {
8972 return self.generate_image(req).await.map(|r| vec![r]);
8973 }
8974 let base_seed = req.seed.unwrap_or(0);
8975 let mut results = Vec::with_capacity(count as usize);
8976 for i in 0..count {
8977 let mut variant_req = req.clone();
8982 variant_req.seed = Some(base_seed.wrapping_add(i as u64));
8983 variant_req.variant_count = Some(1);
8987 results.push(self.generate_image(variant_req).await?);
8988 }
8989 Ok(results)
8990 }
8991
8992 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
9004 fn mlx_device_lock() -> Arc<tokio::sync::Mutex<()>> {
9005 static MLX_DEVICE_LOCK: std::sync::OnceLock<Arc<tokio::sync::Mutex<()>>> =
9006 std::sync::OnceLock::new();
9007 MLX_DEVICE_LOCK
9008 .get_or_init(|| Arc::new(tokio::sync::Mutex::new(())))
9009 .clone()
9010 }
9011
9012 pub async fn generate_video(
9014 &self,
9015 req: GenerateVideoRequest,
9016 ) -> Result<GenerateVideoResult, InferenceError> {
9017 if let Err(msg) = req.validate() {
9020 return Err(InferenceError::InferenceFailed(format!(
9021 "invalid GenerateVideoRequest: {}",
9022 msg
9023 )));
9024 }
9025 #[allow(unused_variables)]
9027 let requires_audio_conditioning = req.requires_audio_passthrough_opt_in();
9028 let candidates = self
9031 .media_generation_candidates(ModelCapability::VideoGeneration, req.model.as_deref())?;
9032 let mut last_error = None;
9033
9034 for schema in candidates {
9035 let mut reservation = self.reserve_local_request(&schema, 0)?;
9037 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
9038 reservation
9039 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
9040 .map_err(InferenceError::from)?;
9041 let result = match &schema.source {
9042 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
9043 ModelSource::Mlx { hf_repo, .. } => {
9044 if !crate::backend::external_ltx::is_available() {
9058 return Err(InferenceError::InferenceFailed(
9059 "video generation requires the external `ltx-2-mlx` CLI on PATH"
9060 .to_string(),
9061 ));
9062 }
9063 let mut req = req.clone();
9064 req.model = Some(hf_repo.clone());
9065 crate::backend::external_ltx::generate_video(&req)
9066 }
9067 _ => Err(InferenceError::InferenceFailed(format!(
9068 "video generation not implemented for model source: {}",
9069 schema.id
9070 ))),
9071 };
9072
9073 match result {
9074 Ok(result) => return Ok(result),
9075 Err(err) if req.model.is_some() => return Err(err),
9076 Err(err) => last_error = Some(err),
9077 }
9078 }
9079
9080 Err(last_error.unwrap_or_else(|| {
9081 InferenceError::InferenceFailed("no video generation models available".into())
9082 }))
9083 }
9084
9085 pub fn response_format_rejection_reason(
9093 &self,
9094 model: &str,
9095 rf: &crate::tasks::generate::ResponseFormat,
9096 ) -> Option<String> {
9097 let registry = (*self.unified_registry()).clone();
9098 let schema = registry
9099 .list()
9100 .into_iter()
9101 .find(|s| s.id == model || s.name == model)?;
9102 match &schema.source {
9103 ModelSource::Proprietary { .. } => {
9104 Some("the Parslee gateway does not accept response_format".to_string())
9105 }
9106 ModelSource::RemoteApi { protocol, .. } => {
9107 let handler = crate::protocol::handler_for(*protocol);
9108 (!handler.supports_response_format(rf)).then(|| {
9109 format!(
9110 "the {} protocol rejects response_format",
9111 handler.protocol_name()
9112 )
9113 })
9114 }
9115 _ => None,
9116 }
9117 }
9118
9119 pub fn list_models_unified(&self) -> Vec<ModelInfo> {
9124 let hardware = HardwareInfo::detect();
9125 let policy = self.active_local_resource_policy();
9126 self.list_models_unified_for(&hardware, &policy.policy)
9127 }
9128
9129 pub fn model_fit(&self, schema: &ModelSchema) -> recommend::ModelFit {
9133 recommend::model_fit(
9134 schema,
9135 &HardwareInfo::detect(),
9136 Some(&self.active_local_resource_policy().policy),
9137 &self.checkpoint_geometry([schema]),
9138 )
9139 }
9140
9141 pub fn checkpoint_geometry<'a>(
9147 &self,
9148 models: impl IntoIterator<Item = &'a ModelSchema>,
9149 ) -> CheckpointGeometry {
9150 self.local_admission.checkpoint_geometry(models)
9151 }
9152
9153 pub fn list_models_unified_for(
9157 &self,
9158 hardware: &HardwareInfo,
9159 policy: &resource_policy::ResourcePolicy,
9160 ) -> Vec<ModelInfo> {
9161 self.list_models_unified_for_inner(hardware, policy, false)
9162 }
9163
9164 pub fn list_models_unified_for_explicit_status(&self) -> Vec<ModelInfo> {
9185 let hardware = HardwareInfo::detect();
9186 let policy = self.active_local_resource_policy();
9187 self.list_models_unified_for_inner(&hardware, &policy.policy, true)
9188 }
9189
9190 fn list_models_unified_for_inner(
9191 &self,
9192 hardware: &HardwareInfo,
9193 policy: &resource_policy::ResourcePolicy,
9194 authoritative_credentials: bool,
9195 ) -> Vec<ModelInfo> {
9196 let mut registry = (*self.unified_registry()).clone();
9197 registry.prune_missing_on_disk_models();
9198 if authoritative_credentials {
9199 registry.refresh_availability_for_explicit_status();
9200 } else {
9201 registry.refresh_availability();
9202 }
9203 let rows = registry.list();
9204 let geometry = self.checkpoint_geometry(rows.iter().copied());
9205 rows.iter()
9206 .map(|schema| {
9207 let mut info = ModelInfo::from(*schema).with_fit(recommend::model_fit(
9208 schema,
9209 hardware,
9210 Some(policy),
9211 &geometry,
9212 ));
9213 if !authoritative_credentials {
9214 info.credential_required = None;
9231 } else if let Some(env) = info.credential_required.as_deref() {
9232 if registry.credential_state_is_unknown(env) {
9243 info.credential_required = None;
9244 }
9245 }
9246 if schema.downloads_weights() {
9247 let enabled = self
9248 .model_management
9249 .car_enabled(&schema.id)
9250 .unwrap_or(false);
9251 if !enabled {
9252 info.available = false;
9256 }
9257 if enabled
9258 && self
9259 .model_management
9260 .load_receipt(&schema.id)
9261 .is_ok_and(|receipt| receipt.is_none())
9262 {
9263 if let Ok(_mutation) = self.model_management.begin_mutation(&schema.id) {
9270 if let Some(path) = registry.existing_local_artifact(&schema.id) {
9271 if std::fs::symlink_metadata(&path)
9272 .is_ok_and(|metadata| metadata.file_type().is_symlink())
9273 {
9274 let generation = self
9275 .resource_policy_generation
9276 .load(std::sync::atomic::Ordering::Acquire);
9277 let _ = self.model_management.record_managed_artifact(
9278 &schema.id,
9279 model_source_identity(schema),
9280 None,
9281 generation,
9282 true,
9283 path,
9284 );
9285 }
9286 }
9287 }
9288 }
9289 let receipt = self.model_management.load_receipt(&schema.id);
9290 let directory_cleanup_unsupported =
9291 !model_management::directory_removal_supported()
9292 && receipt.as_ref().is_ok_and(|receipt| {
9293 receipt.as_ref().is_some_and(|receipt| {
9294 receipt.artifact_kind
9295 == model_management::ManagedArtifactKind::Directory
9296 })
9297 });
9298 info.car_enabled = enabled;
9299 info.can_remove = enabled
9300 && self
9301 .model_management
9302 .can_remove(&schema.id)
9303 .unwrap_or(false);
9304 info.in_use = self.local_model_in_use(&schema.id);
9305 info.management_evidence = if !enabled {
9306 Some("disabled_tombstone".into())
9307 } else if directory_cleanup_unsupported {
9308 Some("install_receipt_directory_cleanup_unsupported".into())
9309 } else if receipt.is_ok_and(|receipt| receipt.is_some()) {
9310 Some("install_receipt".into())
9311 } else if info.weights_ready {
9312 Some("shared_or_hand_installed".into())
9313 } else {
9314 None
9315 };
9316 }
9317 info
9318 })
9319 .collect()
9320 }
9321
9322 fn local_model_in_use(&self, model_id: &str) -> bool {
9326 self.local_admission.active_request_count(model_id) > 0
9327 || self.local_admission.is_resident(model_id)
9328 || self.local_admission.teardown_pending(model_id)
9329 || self.model_management.model_in_use(model_id).unwrap_or(true)
9330 }
9331
9332 pub fn retire_plan(&self, model_id: &str) -> Result<retire::RetirePlan, InferenceError> {
9336 self.retire_plan_in(model_id, &hf_cache::hub_dir())
9337 }
9338
9339 pub fn retire_plan_in(
9341 &self,
9342 model_id: &str,
9343 hub: &Path,
9344 ) -> Result<retire::RetirePlan, InferenceError> {
9345 self.retire_plan_with(model_id, hub, None)
9346 .map(|(plan, _)| plan)
9347 }
9348
9349 fn retire_plan_with(
9354 &self,
9355 model_id: &str,
9356 hub: &Path,
9357 held: Option<&std::collections::BTreeSet<String>>,
9358 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
9359 if self.unified_registry().get(model_id).is_none() {
9360 return Err(InferenceError::ModelNotFound(model_id.to_string()));
9361 }
9362 let (refreshed, owned) = self.retire_snapshot(hub);
9363 self.retire_plan_on(&refreshed, &owned, model_id, hub, held)
9364 }
9365
9366 fn retire_snapshot(&self, hub: &Path) -> (UnifiedRegistry, retire::Ownership) {
9371 let mut refreshed = (*self.unified_registry()).clone();
9372 refreshed.refresh_availability();
9373 let owned = retire::ownership(
9374 retire::catalog_ownership(),
9375 retire::recorded_files(&self.config.state_root, hub),
9376 );
9377 (refreshed, owned)
9378 }
9379
9380 fn retire_plan_on(
9382 &self,
9383 refreshed: &UnifiedRegistry,
9384 owned: &retire::Ownership,
9385 model_id: &str,
9386 hub: &Path,
9387 held: Option<&std::collections::BTreeSet<String>>,
9388 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
9389 let schema = refreshed
9390 .get(model_id)
9391 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
9392 let registry: Vec<&ModelSchema> = refreshed.all().collect();
9393 let installed = |m: &ModelSchema| {
9394 m.downloads_weights()
9395 && m.weights_ready
9396 && self.model_management.car_enabled(&m.id).unwrap_or(true)
9397 };
9398 let primary = retire::primary_whole_repo(schema);
9404 let expands = |repo: &str| {
9408 hf_cache::checked_repo_dir_in(hub, repo).is_some()
9409 && owned.get(repo) == Some(&retire::Owned::Whole)
9410 };
9411 let mut targets: Vec<&ModelSchema> = vec![schema];
9412 targets.extend(registry.iter().copied().filter(|m| {
9413 m.id != schema.id
9414 && primary.is_some_and(|repo| {
9415 retire::primary_whole_repo(m) == Some(repo) && expands(repo)
9416 })
9417 }));
9418 let mut plan = retire::plan_hub_retirement(&retire::RetireInputs {
9419 targets: &targets,
9420 registry: ®istry,
9421 installed: &installed,
9422 owned,
9423 hub,
9424 });
9425 if plan.refusals.iter().any(|r| {
9426 matches!(
9427 r,
9428 retire::RetireRefusal::NothingToRetire | retire::RetireRefusal::Unsupported { .. }
9429 )
9430 }) {
9431 return Ok((plan, std::collections::BTreeSet::new()));
9432 }
9433
9434 let models_dir = self
9438 .config
9439 .models_dir
9440 .canonicalize()
9441 .unwrap_or_else(|_| self.config.models_dir.clone());
9442 for target in &targets {
9443 let mut projections = Vec::new();
9444 if retire::single_component(&target.name) {
9445 projections.push(models_dir.join(&target.name));
9446 } else {
9447 plan.refusals.push(retire::RetireRefusal::UnsafePath {
9448 path: self.config.models_dir.join(&target.name),
9449 why: "the model's name is not a single path component".into(),
9450 });
9451 }
9452 if let Ok(adopted) = self.model_management.adopted_projection_path(&target.id) {
9453 projections.push(adopted);
9454 }
9455 let receipt_path = self
9460 .model_management
9461 .load_receipt(&target.id)
9462 .ok()
9463 .flatten()
9464 .map(|r| r.managed_path);
9465 for path in projections {
9466 if std::fs::symlink_metadata(&path).is_err()
9467 || plan.projections.iter().any(|p| p.path == path)
9468 || plan.user_content.contains(&path)
9469 {
9470 continue;
9471 }
9472 let receipted = receipt_path
9477 .as_ref()
9478 .is_some_and(|managed| retire::same_entry(managed, &path));
9479 if receipted || retire::links_only(&path) {
9480 plan.freed_bytes += retire::tree_bytes(&path);
9481 plan.projections.push(retire::Projection {
9482 path,
9483 receipt_for: receipted.then(|| target.id.clone()),
9484 });
9485 } else {
9486 plan.user_content.push(path);
9487 }
9488 }
9489 }
9490
9491 if targets.iter().any(|t| self.local_model_in_use(&t.id)) {
9492 plan.refusals.push(retire::RetireRefusal::InUse);
9493 }
9494 let repos: std::collections::HashSet<String> = targets
9498 .iter()
9499 .flat_map(|t| retire::hf_references(t))
9500 .map(|r| r.repo)
9501 .collect();
9502 let lock_keys: std::collections::BTreeSet<String> = registry
9503 .iter()
9504 .filter(|m| {
9505 targets.iter().any(|t| t.id == m.id)
9506 || retire::hf_references(m)
9507 .iter()
9508 .any(|r| repos.contains(&r.repo))
9509 })
9510 .flat_map(|m| [m.id.clone(), m.name.clone()])
9511 .collect();
9512 match held {
9513 None => {
9514 if lock_keys
9515 .iter()
9516 .any(|key| crate::download::model_lock_held(key))
9517 {
9518 plan.refusals.push(retire::RetireRefusal::Downloading);
9519 }
9520 }
9521 Some(held) => {
9522 if !lock_keys.is_subset(held) {
9523 plan.refusals.push(retire::RetireRefusal::StateChanged);
9524 }
9525 }
9526 }
9527 for default in self.lane_defaults().defaults {
9528 if targets.iter().any(|t| t.id == default.model_id) {
9529 let use_case = serde_json::to_value(default.use_case)
9530 .ok()
9531 .and_then(|v| v.as_str().map(str::to_string))
9532 .unwrap_or_default();
9533 plan.refusals
9534 .push(retire::RetireRefusal::LaneDefault { use_case });
9535 }
9536 }
9537 let protected = self.protected_models(refreshed, false);
9541 for target in &targets {
9542 if let Some(why) = protected.get(&target.id) {
9543 plan.refusals
9544 .push(retire::RetireRefusal::Protected { why: why.clone() });
9545 }
9546 }
9547 if plan.deletions.is_empty() && plan.projections.is_empty() && plan.refusals.is_empty() {
9549 plan.refusals.push(retire::RetireRefusal::NothingToRetire);
9550 }
9551 plan.seal();
9552 Ok((plan, lock_keys))
9553 }
9554
9555 pub async fn retire_model(
9561 &self,
9562 model_id: &str,
9563 dry_run: bool,
9564 expect: Option<&str>,
9565 ) -> Result<retire::RetireOutcome, InferenceError> {
9566 self.retire_model_in(model_id, &hf_cache::hub_dir(), dry_run, expect)
9567 .await
9568 }
9569
9570 pub async fn retire_model_in(
9586 &self,
9587 model_id: &str,
9588 hub: &Path,
9589 dry_run: bool,
9590 expect: Option<&str>,
9591 ) -> Result<retire::RetireOutcome, InferenceError> {
9592 let not_executed = |plan| retire::RetireOutcome {
9593 plan,
9594 executed: false,
9595 removed: Vec::new(),
9596 errors: Vec::new(),
9597 };
9598 let (first, keys) = self.retire_plan_with(model_id, hub, None)?;
9599 if dry_run || first.is_refused() {
9600 return Ok(not_executed(first));
9601 }
9602 let _concierge = self.concierge_action_lock.lock().await;
9606 let mut locks = Vec::with_capacity(keys.len());
9607 for key in &keys {
9608 match crate::download::try_acquire_model_lock(key) {
9609 Some(guard) => locks.push(guard),
9610 None => {
9611 let mut refused = first;
9612 refused.refusals.push(retire::RetireRefusal::Downloading);
9613 return Ok(not_executed(refused));
9614 }
9615 }
9616 }
9617 let (mut plan, _) = self.retire_plan_with(model_id, hub, Some(&keys))?;
9618 if expect.is_some_and(|digest| digest != plan.digest) {
9619 plan.refusals.push(retire::RetireRefusal::StateChanged);
9620 }
9621 if plan.is_refused() {
9622 return Ok(not_executed(plan));
9623 }
9624 let targets: Vec<String> = std::iter::once(plan.model_id.clone())
9625 .chain(plan.also_retires.iter().cloned())
9626 .collect();
9627 let mut maintenance = Vec::with_capacity(targets.len());
9628 for id in &targets {
9629 match self.prepare_local_model_removal(id).await {
9630 Ok(guard) => maintenance.push(guard),
9631 Err(error) => {
9632 tracing::info!(model = %id, %error, "retirement refused: the model could not be released");
9633 plan.refusals.push(retire::RetireRefusal::InUse);
9634 return Ok(not_executed(plan));
9635 }
9636 }
9637 }
9638
9639 let started_at = std::time::SystemTime::now()
9642 .duration_since(std::time::UNIX_EPOCH)
9643 .map(|d| d.as_secs())
9644 .unwrap_or(0);
9645 for id in &targets {
9646 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9647 seq: 0,
9648 kind: action_ledger::ConciergeActionKind::RetireStarted,
9649 model_id: id.clone(),
9650 use_case: None,
9651 project: None,
9652 prior_model_id: None,
9653 detail: format!(
9654 "retiring with {}; plan {}; {} bytes planned",
9655 targets.join(", "),
9656 plan.digest,
9657 plan.freed_bytes
9658 ),
9659 timestamp: started_at,
9660 });
9661 }
9662
9663 let mut removed = Vec::new();
9664 let mut errors = Vec::new();
9665
9666 for projection in &plan.projections {
9669 let result = match &projection.receipt_for {
9670 Some(id) => {
9671 let result = async {
9674 self.model_management
9675 .note_retiring(id)
9676 .map_err(|e| e.to_string())?;
9677 self.remove_model_from_car_inner(id, true)
9678 .await
9679 .map_err(|e| e.to_string())?;
9680 self.model_management
9681 .clear_tombstone(id)
9682 .map_err(|e| e.to_string())?;
9683 self.model_management.retired(id);
9684 Ok::<_, String>(vec![projection.path.clone()])
9685 }
9686 .await;
9687 result
9688 }
9689 None => retire::remove_links_only_tree(&projection.path, &self.config.models_dir),
9690 };
9691 match result {
9692 Ok(paths) => removed.extend(paths),
9693 Err(why) => {
9694 errors.push(format!("{}: {why}", projection.path.display()));
9695 break;
9696 }
9697 }
9698 }
9699
9700 let hub_canonical = hub.canonicalize().unwrap_or_else(|_| hub.to_path_buf());
9701 if errors.is_empty() {
9702 'hub: for deletion in &plan.deletions {
9703 match deletion {
9704 retire::HubDeletion::Repo { path, .. } => {
9705 let quarantine = path.with_file_name(format!(
9706 "{}{:016x}",
9707 model_management::RETIRE_QUARANTINE_PREFIX,
9708 rand::random::<u64>()
9709 ));
9710 let verify = || retire::verify_repo_dir(path, hub);
9711 match self.model_management.detach_and_remove_tree(
9712 path,
9713 &quarantine,
9714 model_id,
9715 verify,
9716 ) {
9717 Ok(()) => removed.push(path.clone()),
9718 Err(failure) => {
9719 if failure.detached {
9720 removed.push(path.clone());
9723 }
9724 errors.push(format!("{}: {}", path.display(), failure.error));
9725 break 'hub;
9726 }
9727 }
9728 }
9729 retire::HubDeletion::Files {
9730 repo,
9731 entries,
9732 blobs,
9733 ..
9734 } => {
9735 let Some(dir) = hf_cache::checked_repo_dir_in(&hub_canonical, repo) else {
9736 errors.push(format!("{repo}: not a Hub repo id"));
9737 break 'hub;
9738 };
9739 for blob in blobs {
9743 let checked =
9744 retire::verify_blob(blob, &dir.join("blobs")).and_then(|()| {
9745 model_management::unlink_checked(
9746 blob,
9747 model_management::UnlinkKind::File,
9748 )
9749 .map_err(|e| e.to_string())
9750 });
9751 match checked {
9752 Ok(()) => removed.push(blob.clone()),
9753 Err(why) => {
9754 errors.push(format!("{}: {why}", blob.display()));
9755 break 'hub;
9756 }
9757 }
9758 }
9759 let snapshots = dir.join("snapshots");
9760 for entry in entries {
9761 let checked = retire::verify_entry(entry, &snapshots).and_then(|()| {
9762 model_management::unlink_checked(
9763 entry,
9764 model_management::UnlinkKind::FileOrLink,
9765 )
9766 .map_err(|e| e.to_string())
9767 });
9768 match checked {
9769 Ok(()) => removed.push(entry.clone()),
9770 Err(why) => {
9771 errors.push(format!("{}: {why}", entry.display()));
9772 break 'hub;
9773 }
9774 }
9775 }
9776 for entry in entries {
9779 let mut at = entry.parent();
9780 while let Some(parent) = at {
9781 if !parent.starts_with(&snapshots) || parent == snapshots {
9782 break;
9783 }
9784 if std::fs::remove_dir(parent).is_err() {
9785 break;
9786 }
9787 at = parent.parent();
9788 }
9789 }
9790 retire::prune_empty_repo(&dir);
9791 }
9792 }
9793 }
9794 }
9795 drop(maintenance);
9796 drop(locks);
9797
9798 if !removed.is_empty() {
9799 let now = std::time::SystemTime::now()
9800 .duration_since(std::time::UNIX_EPOCH)
9801 .map(|d| d.as_secs())
9802 .unwrap_or(0);
9803 for id in &targets {
9804 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9805 seq: 0,
9806 kind: action_ledger::ConciergeActionKind::Retire,
9807 model_id: id.clone(),
9808 use_case: None,
9809 project: None,
9810 prior_model_id: None,
9811 detail: format!(
9812 "retired with {}; plan {}; {} bytes planned; {} paths removed{}",
9813 targets.join(", "),
9814 plan.digest,
9815 plan.freed_bytes,
9816 removed.len(),
9817 if errors.is_empty() {
9818 String::new()
9819 } else {
9820 format!("; stopped: {}", errors.join("; "))
9821 }
9822 ),
9823 timestamp: now,
9824 });
9825 }
9826 }
9827 Ok(retire::RetireOutcome {
9828 executed: !removed.is_empty(),
9829 plan,
9830 removed,
9831 errors,
9832 })
9833 }
9834
9835 pub fn set_external_pins(&self, pins: std::collections::BTreeMap<String, Vec<String>>) {
9843 *self
9844 .external_pins
9845 .write()
9846 .unwrap_or_else(std::sync::PoisonError::into_inner) = pins;
9847 }
9848
9849 pub fn start_usage_tracking(&self) {
9853 self.model_management.start_usage_tracking();
9854 self.model_management.beat();
9855 }
9856
9857 fn recent_concierge_actions(&self, now: u64) -> Vec<portfolio::RecentAction> {
9861 use action_ledger::ConciergeActionKind as Kind;
9862 let since = now.saturating_sub(30 * 24 * 60 * 60);
9863 let actions = crate::action_ledger::read_actions(&self.action_ledger_path(), 0);
9864 let mut out: Vec<portfolio::RecentAction> = Vec::new();
9865 for (i, a) in actions.iter().enumerate() {
9866 if a.timestamp < since {
9867 continue;
9868 }
9869 let (kind, undo, interrupted) = match a.kind {
9870 Kind::Retire => (
9871 "retire",
9872 (!a.model_id.starts_with("repo:"))
9873 .then(|| format!("car models pull {}", a.model_id)),
9874 false,
9875 ),
9876 Kind::RetireStarted => {
9877 let finished = actions[i + 1..]
9878 .iter()
9879 .any(|later| later.kind == Kind::Retire && later.model_id == a.model_id);
9880 if finished {
9881 continue;
9882 }
9883 ("retire", None, true)
9884 }
9885 Kind::DiscardPartials => ("discard_partials", None, false),
9886 Kind::ResourcePolicy => (
9887 "resource_policy",
9888 a.prior_model_id
9889 .as_ref()
9890 .map(|prior| format!("car models resource-policy set {prior}")),
9891 false,
9892 ),
9893 Kind::AutoUpgrade => (
9894 "upgrade",
9895 match (a.use_case, &a.prior_model_id) {
9896 (Some(use_case), _) => serde_json::to_value(use_case)
9897 .ok()
9898 .and_then(|v| v.as_str().map(str::to_owned))
9899 .map(|lane| format!("car concierge rollback {lane}")),
9900 (None, Some(prior)) => {
9901 Some(format!("keep using {prior}; nothing was switched"))
9902 }
9903 (None, None) => None,
9904 },
9905 false,
9906 ),
9907 Kind::RevokedLaneCleared => ("lane_cleared", None, false),
9910 Kind::Install | Kind::SetDefault | Kind::ClearDefault | Kind::Rollback => continue,
9911 };
9912 out.push(portfolio::RecentAction {
9913 at: a.timestamp,
9914 kind: kind.to_string(),
9915 subject: a.model_id.clone(),
9916 detail: a.detail.clone(),
9917 undo,
9918 use_case: a.use_case.and_then(|use_case| {
9919 serde_json::to_value(use_case)
9920 .ok()
9921 .and_then(|v| v.as_str().map(str::to_owned))
9922 }),
9923 prior: a.prior_model_id.clone(),
9924 interrupted,
9925 });
9926 }
9927 out.reverse();
9928 out
9929 }
9930
9931 pub async fn set_resource_policy_by_user(
9936 &self,
9937 policy: resource_policy::ResourcePolicy,
9938 ) -> Result<(), resource_policy::ResourcePolicyError> {
9939 let _concierge = self.concierge_action_lock.lock().await;
9940 let repository =
9941 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9942 repository.clear_concierge_marker()?;
9943 resource_policy::ResourcePolicyRepository::save(&repository, &policy)?;
9944 self.apply_local_resource_policy(policy);
9945 Ok(())
9946 }
9947
9948 async fn apply_one_curated_upgrade(
9957 &self,
9958 hub: &Path,
9959 nudge: &crate::nudge::NudgeState,
9960 now: u64,
9961 ) -> Result<Option<maintenance::Upgraded>, String> {
9962 let findings = self.detect_upgrades().await;
9963 self.apply_one_curated_upgrade_from(findings, hub, nudge, now)
9964 .await
9965 }
9966
9967 async fn apply_one_curated_upgrade_from(
9969 &self,
9970 findings: Vec<crate::upgrade::UpgradeFinding>,
9971 hub: &Path,
9972 nudge: &crate::nudge::NudgeState,
9973 now: u64,
9974 ) -> Result<Option<maintenance::Upgraded>, String> {
9975 use crate::upgrade::UpgradeSource;
9976 for finding in findings {
9977 let label = match finding.source {
9978 UpgradeSource::Benchmark => "benchmark upgrade",
9979 UpgradeSource::Curated | UpgradeSource::Upstream => "curated upgrade",
9980 };
9981 let key = format!("{}=>{}", finding.from_id, finding.to_id);
9982 let backing_off = self
9983 .failed_upgrades
9984 .lock()
9985 .unwrap_or_else(std::sync::PoisonError::into_inner)
9986 .get(&key)
9987 .is_some_and(|at| now.saturating_sub(*at) < 24 * 60 * 60);
9988 if finding.source == UpgradeSource::Upstream
9989 || finding.trust_tier != crate::schema::TrustTier::Curated
9990 || !finding.target_pullable
9991 || nudge.dismissed.contains(&key)
9992 || backing_off
9993 {
9994 continue;
9995 }
9996 let Some(target) = self.unified_registry().get(&finding.to_id).cloned() else {
9997 continue;
9998 };
9999 let fit = self.model_fit(&target);
10000 if !fit.platform_compatible || fit.fit == recommend::ModelFitStatus::TooBig {
10001 continue;
10002 }
10003 let lanes: Vec<crate::lane_defaults::LaneDefault> = self
10004 .lane_defaults()
10005 .defaults
10006 .into_iter()
10007 .filter(|d| d.model_id == finding.from_id)
10008 .collect();
10009 if lanes.is_empty() {
10010 continue;
10011 }
10012 let present = self
10014 .unified_registry()
10015 .existing_local_artifact(&finding.to_id)
10016 .is_some();
10017 if !present && crate::download::check_disk_space(hub, target.size_mb()).is_err() {
10018 continue;
10019 }
10020 let mut switched = Vec::new();
10021 for lane in &lanes {
10022 if let Err(e) = self
10023 .apply_concierge(lane.use_case, &finding.to_id, lane.project.clone())
10024 .await
10025 {
10026 self.failed_upgrades
10027 .lock()
10028 .unwrap_or_else(std::sync::PoisonError::into_inner)
10029 .insert(key, now);
10030 if let Some(first) = lanes.first().filter(|_| !switched.is_empty()) {
10032 self.record_concierge_action(action_ledger::ConciergeActionEntry {
10033 seq: 0,
10034 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
10035 model_id: finding.to_id.clone(),
10036 use_case: Some(first.use_case),
10037 project: first.project.clone(),
10038 prior_model_id: Some(finding.from_id.clone()),
10039 detail: format!(
10040 "{label}, partial: switched {}; then {e}",
10041 switched.join(", ")
10042 ),
10043 timestamp: now,
10044 });
10045 }
10046 return Err(format!(
10047 "upgrading {} to {} (fetch, then switch the lane): {e}; retried in a day",
10048 finding.from_id, finding.to_id
10049 ));
10050 }
10051 switched.push(
10052 serde_json::to_value(lane.use_case)
10053 .ok()
10054 .and_then(|v| v.as_str().map(str::to_owned))
10055 .unwrap_or_default(),
10056 );
10057 }
10058 self.record_concierge_action(action_ledger::ConciergeActionEntry {
10059 seq: 0,
10060 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
10061 model_id: finding.to_id.clone(),
10062 use_case: lanes.first().map(|l| l.use_case),
10063 project: lanes.first().and_then(|l| l.project.clone()),
10064 prior_model_id: Some(finding.from_id.clone()),
10065 detail: format!("{label}: {}", finding.reason),
10066 timestamp: now,
10067 });
10068 return Ok(Some(maintenance::Upgraded {
10069 from: finding.from_id,
10070 to: finding.to_id,
10071 lanes: switched,
10072 }));
10073 }
10074 Ok(None)
10075 }
10076
10077 async fn size_memory_policy(
10086 &self,
10087 hw: &HardwareInfo,
10088 observable_since: Option<u64>,
10089 apply: bool,
10090 now: u64,
10091 ) -> Option<maintenance::PolicySizing> {
10092 use resource_policy::{ResourcePolicy, ResourcePolicyLoadSource};
10093 let since = observable_since?;
10094 let _concierge = self.concierge_action_lock.lock().await;
10095 let repository =
10096 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
10097 let loaded = repository.load_with_evidence().ok()?;
10098 let concierge = repository.concierge_choice();
10099 let user_chose = match loaded.source {
10100 ResourcePolicyLoadSource::MissingDefault => false,
10101 ResourcePolicyLoadSource::CorruptDefault => true,
10102 ResourcePolicyLoadSource::Loaded => {
10103 concierge.as_ref().map(|c| &c.policy) != Some(&loaded.policy)
10104 }
10105 };
10106 let everyday = ResourcePolicy::everyday();
10107 let local_focused = ResourcePolicy::local_focused();
10108 let fit = |m: &ModelSchema, policy: &ResourcePolicy| {
10109 recommend::model_fit(m, hw, Some(policy), &self.checkpoint_geometry([m])).fit
10110 };
10111 let raised_at = concierge
10112 .as_ref()
10113 .filter(|c| !c.reverted && c.policy == local_focused)
10114 .map(|c| c.at);
10115 let mut evidence = maintenance::PolicyEvidence::default();
10116 let mut days = std::collections::BTreeSet::new();
10117 for e in crate::outcome::read_ledger(
10118 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
10119 0,
10120 ) {
10121 if e.timestamp < since
10122 || e.attribution != Some(crate::outcome::UnattributedReason::ResourceBlocked)
10123 {
10124 continue;
10125 }
10126 let registry = self.unified_registry();
10127 let Some(m) = registry.get(&e.model_id).filter(|m| m.is_local()) else {
10128 continue;
10129 };
10130 if fit(m, &loaded.policy) == recommend::ModelFitStatus::TooBig
10135 && fit(m, &local_focused) == recommend::ModelFitStatus::Fits
10136 {
10137 evidence.ceiling_refusals += 1;
10138 days.insert(e.timestamp / (24 * 60 * 60));
10139 }
10140 if raised_at.is_some_and(|at| e.timestamp >= at)
10141 && fit(m, &everyday) == recommend::ModelFitStatus::Fits
10142 {
10143 evidence.live_refusals_since_raise += 1;
10144 }
10145 }
10146 if let Some(at) = raised_at {
10148 for e in crate::outcome::read_ledger(
10149 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
10150 0,
10151 ) {
10152 let admitted_by_raise = self.unified_registry().get(&e.model_id).is_some_and(|m| {
10153 m.is_local() && fit(m, &everyday) == recommend::ModelFitStatus::TooBig
10154 });
10155 if e.timestamp < at || !admitted_by_raise {
10156 continue;
10157 }
10158 match (e.success, e.attribution) {
10159 (Some(true), _) => evidence.admitted_successes_since_raise += 1,
10160 (Some(false), None) => evidence.admitted_failures_since_raise += 1,
10161 _ => {}
10162 }
10163 }
10164 }
10165 evidence.ceiling_refusal_days = days.len() as u64;
10166 let decision = maintenance::decide_resource_policy(
10167 &loaded.policy,
10168 user_chose,
10169 concierge.as_ref(),
10170 hw.total_ram_mb,
10171 &evidence,
10172 )?;
10173 let (target, reason, reverted) = match decision {
10174 maintenance::PolicyMove::Raise(reason) => (local_focused, reason, false),
10175 maintenance::PolicyMove::Revert(reason) => (everyday, reason, true),
10176 };
10177 let mut sizing = maintenance::PolicySizing {
10178 from: loaded.policy.profile.clone(),
10179 to: target.profile.clone(),
10180 reason,
10181 applied: false,
10182 };
10183 if apply {
10184 match repository.save_concierge_choice(&target, &sizing.reason, now, reverted) {
10185 Ok(()) => {
10186 self.apply_local_resource_policy(target);
10187 sizing.applied = true;
10188 self.record_concierge_action(action_ledger::ConciergeActionEntry {
10189 seq: 0,
10190 kind: action_ledger::ConciergeActionKind::ResourcePolicy,
10191 model_id: String::new(),
10192 use_case: None,
10193 project: None,
10194 prior_model_id: serde_json::to_value(&sizing.from)
10195 .ok()
10196 .and_then(|v| v.as_str().map(str::to_owned)),
10197 detail: format!("{:?} -> {:?}: {}", sizing.from, sizing.to, sizing.reason),
10198 timestamp: now,
10199 });
10200 }
10201 Err(error) => tracing::warn!(%error, "could not change the memory policy"),
10202 }
10203 }
10204 Some(sizing)
10205 }
10206
10207 async fn retire_orphan_repo(&self, repo: &str, hub: &Path) -> Result<u64, String> {
10214 let _concierge = self.concierge_action_lock.lock().await;
10215 let (refreshed, owned) = self.retire_snapshot(hub);
10216 let referenced: std::collections::HashSet<String> = refreshed
10217 .all()
10218 .flat_map(retire::hf_references)
10219 .map(|r| r.repo)
10220 .collect();
10221 let orphan = retire::unreferenced_repos(hub, &referenced, &owned)
10222 .into_iter()
10223 .find(|o| o.repo == repo && o.owner == retire::OrphanOwner::Car)
10224 .ok_or("no longer an orphan CAR owns")?;
10225 if retire::transfer_in_progress(&orphan.path)? {
10226 return Err("a transfer is writing into it".into());
10227 }
10228 let repo_dir = orphan
10233 .path
10234 .canonicalize()
10235 .map_err(|e| format!("cannot resolve the repo: {e}"))?;
10236 if let Some(user) = retire::path_resolving_into(&self.config.models_dir, &repo_dir) {
10237 return Err(format!("{} still resolves into it", user.display()));
10238 }
10239 for row in refreshed.all().filter(|m| m.is_local()) {
10240 let inside = refreshed
10241 .existing_local_artifact(&row.id)
10242 .and_then(|path| path.canonicalize().ok())
10243 .is_some_and(|path| path.starts_with(&repo_dir));
10244 if inside {
10245 return Err(format!("the model {} still loads from it", row.id));
10246 }
10247 }
10248 let label = format!("repo:{repo}");
10249 let now = std::time::SystemTime::now()
10250 .duration_since(std::time::UNIX_EPOCH)
10251 .map(|d| d.as_secs())
10252 .unwrap_or(0);
10253 let entry = |kind, detail: String| action_ledger::ConciergeActionEntry {
10254 seq: 0,
10255 kind,
10256 model_id: label.clone(),
10257 use_case: None,
10258 project: None,
10259 prior_model_id: None,
10260 detail,
10261 timestamp: now,
10262 };
10263 self.record_concierge_action(entry(
10264 action_ledger::ConciergeActionKind::RetireStarted,
10265 format!("orphaned repo; {} bytes planned", orphan.bytes),
10266 ));
10267 let quarantine = orphan.path.with_file_name(format!(
10268 "{}{:016x}",
10269 model_management::RETIRE_QUARANTINE_PREFIX,
10270 rand::random::<u64>()
10271 ));
10272 let verify = || retire::verify_repo_dir(&orphan.path, hub);
10273 self.model_management
10274 .detach_and_remove_tree(&orphan.path, &quarantine, &label, verify)
10275 .map_err(|failure| failure.error.to_string())?;
10276 self.record_concierge_action(entry(
10277 action_ledger::ConciergeActionKind::Retire,
10278 format!("orphaned repo no model uses; {} bytes", orphan.bytes),
10279 ));
10280 Ok(orphan.bytes)
10281 }
10282
10283 pub fn usage_heartbeat(&self) {
10287 self.model_management.beat();
10288 }
10289
10290 pub fn effective_update_policy(
10297 &self,
10298 prefs: &crate::update_prefs::UpdatePreferences,
10299 start_clock: bool,
10300 ) -> (crate::update_prefs::UpdatePolicy, Option<u64>) {
10301 use crate::update_prefs::UpdatePolicy;
10302 if prefs.policy != UpdatePolicy::Auto || prefs.policy_chosen {
10303 return (prefs.policy, None);
10304 }
10305 let marker = self.config.state_root.join(AUTO_GRACE_FILE);
10306 let now = std::time::SystemTime::now()
10307 .duration_since(std::time::UNIX_EPOCH)
10308 .map(|d| d.as_secs())
10309 .unwrap_or(0);
10310 let since = std::fs::read_to_string(&marker)
10311 .ok()
10312 .and_then(|text| text.trim().parse::<u64>().ok())
10313 .unwrap_or_else(|| {
10314 if start_clock {
10315 let _ = std::fs::create_dir_all(&self.config.state_root);
10316 let _ = std::fs::write(&marker, now.to_string());
10317 }
10318 now
10319 });
10320 match maintenance::auto_grace_until(since, now, AUTO_GRACE_SECS) {
10321 Some(until) => (UpdatePolicy::Notify, Some(until)),
10322 None => (UpdatePolicy::Auto, None),
10323 }
10324 }
10325
10326 pub async fn maintenance_pass(&self, dry_run: bool) -> maintenance::MaintenanceReport {
10335 self.maintenance_pass_in(&hf_cache::hub_dir(), &HardwareInfo::detect(), dry_run)
10336 .await
10337 }
10338
10339 pub async fn maintenance_pass_in(
10341 &self,
10342 hub: &Path,
10343 hw: &HardwareInfo,
10344 dry_run: bool,
10345 ) -> maintenance::MaintenanceReport {
10346 let prefs = self.update_prefs();
10347 let (policy, grace_until) = self.effective_update_policy(&prefs, !dry_run);
10348 let nudge = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10349 let disk = crate::download::disk_space(hub).map(|(free_bytes, total_bytes)| {
10350 maintenance::DiskSpace {
10351 free_bytes,
10352 total_bytes,
10353 }
10354 });
10355 let mut report = self
10356 .maintenance_pass_with(hub, hw, dry_run, policy, &nudge, disk)
10357 .await;
10358 report.auto_grace_until = grace_until;
10359 report
10360 }
10361
10362 pub async fn maintenance_pass_with(
10365 &self,
10366 hub: &Path,
10367 hw: &HardwareInfo,
10368 dry_run: bool,
10369 policy: crate::update_prefs::UpdatePolicy,
10370 nudge: &crate::nudge::NudgeState,
10371 disk: Option<maintenance::DiskSpace>,
10372 ) -> maintenance::MaintenanceReport {
10373 if !dry_run {
10374 self.enforce_catalog_revocations().await;
10375 }
10376 let mut report = maintenance::MaintenanceReport {
10377 policy,
10378 dry_run,
10379 ..Default::default()
10380 };
10381 if policy == crate::update_prefs::UpdatePolicy::Off {
10382 return report;
10383 }
10384 if !dry_run {
10385 report.errors.extend(self.resume_retirements().await);
10386 }
10387 let portfolio = self.portfolio_in(hub, hw).await;
10388 let now = portfolio.policy.now;
10389 report.disk = disk;
10390 let was_pressured = self
10391 .disk_pressured
10392 .load(std::sync::atomic::Ordering::Relaxed);
10393 report.pressured = maintenance::disk_pressure(report.disk, was_pressured);
10394 if !dry_run {
10395 self.disk_pressured
10396 .store(report.pressured, std::sync::atomic::Ordering::Relaxed);
10397 }
10398 let keep = maintenance::kept_models(nudge, now);
10399 let churned = crate::action_ledger::read_actions(&self.action_ledger_path(), 0)
10402 .into_iter()
10403 .filter(|a| {
10404 matches!(
10405 a.kind,
10406 action_ledger::ConciergeActionKind::Retire
10407 | action_ledger::ConciergeActionKind::RetireStarted
10408 )
10409 })
10410 .map(|a| a.model_id)
10411 .collect();
10412 let ctx = maintenance::MaintenanceContext {
10413 policy,
10414 pressured: report.pressured,
10415 keep,
10416 churned,
10417 superseded_unpressured_secs:
10418 maintenance::MaintenanceContext::DEFAULT_SUPERSEDED_UNPRESSURED_SECS,
10419 };
10420 report.plan = maintenance::decide(&portfolio, &ctx);
10421 report.portfolio = Some(portfolio.clone());
10422 report.resource_policy = self
10423 .size_memory_policy(
10424 hw,
10425 portfolio.policy.tracking_since,
10426 !dry_run && policy == crate::update_prefs::UpdatePolicy::Auto,
10427 now,
10428 )
10429 .await;
10430 if dry_run || policy != crate::update_prefs::UpdatePolicy::Auto {
10431 return report;
10432 }
10433 if let Some(act) = &report.plan.act {
10434 match self
10435 .retire_model_in(&act.model_id, hub, false, Some(&act.digest))
10436 .await
10437 {
10438 Ok(outcome) => {
10439 report.retired = outcome.executed.then(|| maintenance::Retired {
10440 model_id: act.model_id.clone(),
10441 also_retired: outcome.plan.also_retires.clone(),
10442 freed_bytes: outcome.plan.freed_bytes,
10443 restore: format!("car models pull {}", act.model_id),
10444 });
10445 if !outcome.executed {
10446 report.errors.push(format!(
10449 "{}: not retired this pass ({:?})",
10450 act.model_id, outcome.plan.refusals
10451 ));
10452 }
10453 report.errors.extend(outcome.errors);
10454 }
10455 Err(error) => report.errors.push(format!("{}: {error}", act.model_id)),
10456 }
10457 }
10458 if report.retired.is_none() && report.pressured {
10461 let mut orphans: Vec<_> = portfolio
10464 .orphans
10465 .iter()
10466 .filter(|o| o.owner == retire::OrphanOwner::Car)
10467 .collect();
10468 orphans.sort_by(|a, b| b.bytes.cmp(&a.bytes));
10469 for orphan in orphans {
10470 match self.retire_orphan_repo(&orphan.repo, hub).await {
10471 Ok(freed_bytes) => {
10472 report.orphan_retired = Some(maintenance::RetiredOrphan {
10473 repo: orphan.repo.clone(),
10474 freed_bytes,
10475 });
10476 break;
10477 }
10478 Err(error) => report.errors.push(format!("{}: {error}", orphan.repo)),
10479 }
10480 }
10481 }
10482 if report.retired.is_none() && report.orphan_retired.is_none() && !report.pressured {
10484 match self.apply_one_curated_upgrade(hub, nudge, now).await {
10485 Ok(upgraded) => report.upgraded = upgraded,
10486 Err(error) => report.errors.push(error),
10487 }
10488 }
10489 let owned = retire::ownership(
10490 retire::catalog_ownership(),
10491 retire::recorded_files(&self.config.state_root, hub),
10492 );
10493 let (discarded, errors) = retire::discard_abandoned_partials(hub, &owned);
10494 report.errors.extend(errors);
10495 if !discarded.is_empty() {
10496 let bytes: u64 = discarded.iter().map(|(_, b)| b).sum();
10497 self.record_concierge_action(action_ledger::ConciergeActionEntry {
10498 seq: 0,
10499 kind: action_ledger::ConciergeActionKind::DiscardPartials,
10500 model_id: String::new(),
10501 use_case: None,
10502 project: None,
10503 prior_model_id: None,
10504 detail: format!(
10505 "{} abandoned partial downloads, {bytes} bytes",
10506 discarded.len()
10507 ),
10508 timestamp: now,
10509 });
10510 report.partials_discarded = discarded.into_iter().map(|(path, _)| path).collect();
10511 }
10512 report
10513 }
10514
10515 pub async fn portfolio(&self) -> portfolio::Portfolio {
10518 self.portfolio_in(&hf_cache::hub_dir(), &HardwareInfo::detect())
10519 .await
10520 }
10521
10522 pub async fn portfolio_in(&self, hub: &Path, hw: &HardwareInfo) -> portfolio::Portfolio {
10524 let now = std::time::SystemTime::now()
10525 .duration_since(std::time::UNIX_EPOCH)
10526 .map(|d| d.as_secs())
10527 .unwrap_or(0);
10528 let observable_since = self
10531 .model_management
10532 .usage_tracking_since()
10533 .zip(self.model_management.usage_covered_since())
10534 .map(|(tracking, covered)| tracking.max(covered));
10535 let policy = portfolio::PortfolioPolicy::new(now, observable_since);
10536 let (refreshed, owned) = self.retire_snapshot(hub);
10537 let protected = self.protected_models(&refreshed, true);
10538 let mut ledger_last: std::collections::HashMap<String, u64> =
10539 std::collections::HashMap::new();
10540 let mut ledger_failures: std::collections::HashMap<String, Vec<u64>> =
10547 std::collections::HashMap::new();
10548 for entry in crate::outcome::read_ledger(
10549 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
10550 0,
10551 ) {
10552 if entry.counts_against_model() {
10553 ledger_failures
10554 .entry(entry.model_id.clone())
10555 .or_default()
10556 .push(entry.timestamp);
10557 }
10558 let last = ledger_last.entry(entry.model_id).or_default();
10559 *last = (*last).max(entry.timestamp);
10560 }
10561 let installed_at = |m: &ModelSchema| -> Option<u64> {
10564 let repo = retire::hf_references(m).into_iter().next()?.repo;
10565 let snapshots = hf_cache::checked_repo_dir_in(hub, &repo)?.join("snapshots");
10566 std::fs::read_dir(snapshots)
10567 .ok()?
10568 .filter_map(Result::ok)
10569 .filter_map(|e| e.metadata().ok()?.modified().ok())
10570 .filter_map(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
10571 .map(|d| d.as_secs())
10572 .max()
10573 };
10574 let installed = |m: &ModelSchema| {
10575 m.is_local()
10576 && m.downloads_weights()
10577 && m.weights_ready
10578 && self.model_management.car_enabled(&m.id).unwrap_or(true)
10579 };
10580 let geometry = self.checkpoint_geometry(refreshed.all());
10581 let runnable = |m: &ModelSchema| {
10582 let fit = recommend::model_fit(m, hw, None, &geometry);
10583 fit.platform_compatible && fit.fit != recommend::ModelFitStatus::TooBig
10584 };
10585 let outcomes: std::collections::HashMap<String, (u64, u64)> = {
10587 let tracker = self.outcome_tracker.read().await;
10588 refreshed
10589 .all()
10590 .filter(|m| installed(m))
10591 .filter_map(|m| {
10592 tracker
10593 .profile(&m.id)
10594 .map(|p| (m.id.clone(), (p.success_count, p.fail_count)))
10595 })
10596 .collect()
10597 };
10598 let has_run = |id: &str| outcomes.get(id).is_some_and(|(success, _)| *success > 0);
10606 let dismissed =
10608 crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path())
10609 .dismissed;
10610 let superseded: std::collections::HashMap<String, String> = refreshed
10611 .available_upgrades()
10612 .into_iter()
10613 .filter(|u| !dismissed.contains(&format!("{}=>{}", u.from_id, u.to_id)))
10614 .filter(|u| {
10615 refreshed
10616 .get(&u.to_id)
10617 .is_some_and(|to| installed(to) && runnable(to) && has_run(&to.id))
10618 })
10619 .map(|u| (u.from_id, u.to_id))
10620 .collect();
10621 let superseded: std::collections::HashMap<String, String> = superseded
10623 .iter()
10624 .filter(|(from, to)| superseded.get(*to) != Some(*from))
10625 .map(|(from, to)| (from.clone(), to.clone()))
10626 .collect();
10627 let budget = recommend::memory_budget_mb(hw);
10628 let keep = maintenance::kept_models(
10629 &crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path()),
10630 now,
10631 );
10632 let mut facts = Vec::new();
10633 for m in refreshed.all().filter(|m| installed(m)) {
10634 let fit = recommend::model_fit(m, hw, None, &geometry);
10635 let success_count = outcomes.get(&m.id).map_or(0, |(success, _)| *success);
10636 let since_install = installed_at(m)
10640 .unwrap_or(0)
10641 .max(policy.tracking_since.unwrap_or(u64::MAX));
10642 let fail_count = ledger_failures.get(&m.id).map_or(0, |times| {
10643 times.iter().filter(|t| **t >= since_install).count() as u64
10644 });
10645 let retire = self
10646 .retire_plan_on(&refreshed, &owned, &m.id, hub, None)
10647 .ok()
10648 .map(|(plan, _)| portfolio::RetireSummary {
10649 freed_bytes: plan.freed_bytes,
10650 refusals: plan
10651 .refusals
10652 .iter()
10653 .map(|r| {
10654 serde_json::to_value(r)
10655 .ok()
10656 .and_then(|v| {
10657 v.get("reason").and_then(|r| r.as_str()).map(str::to_owned)
10658 })
10659 .unwrap_or_else(|| format!("{r:?}"))
10660 })
10661 .collect(),
10662 also_retires: plan.also_retires.clone(),
10663 digest: plan.digest,
10664 });
10665 facts.push(portfolio::ModelFacts {
10666 model_id: m.id.clone(),
10667 name: m.name.clone(),
10668 generates_text: m.has_capability(ModelCapability::Generate),
10669 platform_compatible: fit.platform_compatible,
10670 never_fits: fit.fit == recommend::ModelFitStatus::TooBig,
10671 estimated_peak_mb: fit.estimated_peak_mb,
10672 hardware_budget_mb: budget,
10673 protected: protected.get(&m.id).cloned(),
10674 kept: keep.contains(&m.id),
10675 last_used: self.model_management.last_used(&m.id),
10676 ledger_last_used: ledger_last.get(&m.id).copied(),
10677 success_count,
10678 fail_count,
10679 superseded_by: superseded.get(&m.id).cloned(),
10680 retire,
10681 });
10682 }
10683 let referenced: std::collections::HashSet<String> = refreshed
10684 .all()
10685 .flat_map(retire::hf_references)
10686 .map(|r| r.repo)
10687 .collect();
10688 let mut assessed = portfolio::assess_all(&facts, &policy);
10689 assessed.orphans = retire::unreferenced_repos(hub, &referenced, &owned);
10690 assessed.recent_actions = self.recent_concierge_actions(now);
10691 assessed.revoked_pins = self.revoked_pins(&refreshed);
10692 assessed.revoked_copies =
10693 retire::revoked_copies(&self.config.state_root, &self.config.models_dir);
10694 assessed
10695 }
10696
10697 fn revoked_pins(&self, registry: &UnifiedRegistry) -> Vec<portfolio::RevokedPin> {
10705 let pins = self
10706 .external_pins
10707 .read()
10708 .unwrap_or_else(std::sync::PoisonError::into_inner)
10709 .clone();
10710 if pins.is_empty() {
10711 return Vec::new();
10712 }
10713 let names =
10714 catalog::load_revoked_names(&catalog::revoked_models_path(&self.config.state_root));
10715 pins.into_iter()
10716 .filter_map(|(pin, mut agents)| {
10717 let resolved = registry
10718 .get(&pin)
10719 .or_else(|| registry.find_by_name(&pin))
10720 .map(|schema| schema.id.clone());
10721 let model_id = if registry.is_revoked(&pin) {
10722 pin.clone()
10723 } else if let Some(id) = resolved {
10724 id
10725 } else {
10726 names.get(&pin)?.clone()
10727 };
10728 if !registry.is_revoked(&model_id) {
10729 return None;
10730 }
10731 agents.sort();
10732 agents.dedup();
10733 Some(portfolio::RevokedPin {
10734 pin,
10735 model_id,
10736 agents,
10737 })
10738 })
10739 .collect()
10740 }
10741
10742 fn protected_models(
10749 &self,
10750 registry: &UnifiedRegistry,
10751 include_lanes: bool,
10752 ) -> std::collections::BTreeMap<String, String> {
10753 let mut out = std::collections::BTreeMap::new();
10754 let mut protect = |key: &str, why: String| {
10755 let Some(schema) = registry.get(key).or_else(|| registry.find_by_name(key)) else {
10756 return;
10757 };
10758 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
10761 if let Some(twin) = registry.resolve_mlx_equivalent(schema) {
10762 out.entry(twin.id.clone()).or_insert_with(|| why.clone());
10763 }
10764 out.entry(schema.id.clone()).or_insert(why);
10765 };
10766 let lanes = if include_lanes {
10767 self.lane_defaults().defaults
10768 } else {
10769 Vec::new()
10770 };
10771 for lane in lanes {
10772 let use_case = serde_json::to_value(lane.use_case)
10773 .ok()
10774 .and_then(|v| v.as_str().map(str::to_owned))
10775 .unwrap_or_else(|| format!("{:?}", lane.use_case));
10776 protect(&lane.model_id, format!("the {use_case} lane default"));
10777 }
10778 let config = &self.config;
10779 for (key, what) in [
10780 (Some(&config.generation_model), "the generation model"),
10781 (
10782 config.preferred_generation_model.as_ref(),
10783 "the preferred generation model",
10784 ),
10785 (Some(&config.embedding_model), "the embedding model"),
10786 (
10787 config.preferred_embedding_model.as_ref(),
10788 "the preferred embedding model",
10789 ),
10790 (
10791 Some(&config.classification_model),
10792 "the classification model",
10793 ),
10794 (
10795 config.preferred_classification_model.as_ref(),
10796 "the preferred classification model",
10797 ),
10798 ] {
10799 if let Some(key) = key {
10800 protect(key, format!("{what} in CAR's configuration"));
10801 }
10802 }
10803 for (capability, what) in [
10804 (
10805 ModelCapability::SpeechToText,
10806 "the local speech-to-text default",
10807 ),
10808 (
10809 ModelCapability::TextToSpeech,
10810 "the local text-to-speech default",
10811 ),
10812 ] {
10813 if let Some(name) = self.speech_health_default_name(capability, true, false) {
10814 protect(&name, what.to_string());
10815 }
10816 }
10817 protect(
10818 LOCAL_TOOL_MODEL_PULL_ID,
10819 "the local tool model CAR's setup guidance installs".to_string(),
10820 );
10821 let pins = self
10822 .external_pins
10823 .read()
10824 .unwrap_or_else(std::sync::PoisonError::into_inner)
10825 .clone();
10826 for (key, agents) in pins {
10827 protect(&key, pinned_by(&agents));
10828 }
10829 out
10830 }
10831
10832 pub async fn resume_retirements(&self) -> Vec<String> {
10837 let _concierge = self.concierge_action_lock.lock().await;
10838 let mut errors: Vec<String> = self
10839 .model_management
10840 .resume_detached_trees()
10841 .into_iter()
10842 .map(|e| e.to_string())
10843 .collect();
10844 errors.extend(
10845 self.model_management
10846 .resume_retiring()
10847 .into_iter()
10848 .map(|e| e.to_string()),
10849 );
10850 errors
10851 }
10852
10853 pub fn model_management_store(&self) -> &model_management::ModelManagementStore {
10854 &self.model_management
10855 }
10856
10857 fn ensure_model_enabled(&self, model_id: &str) -> Result<(), InferenceError> {
10858 if self.model_management.car_enabled(model_id)? {
10859 return Ok(());
10860 }
10861 Err(InferenceError::InferenceFailed(format!(
10862 "local model {model_id} was removed from CAR; reinstall it before use"
10863 )))
10864 }
10865
10866 pub fn available_model_upgrades(&self) -> Vec<ModelUpgrade> {
10868 self.unified_registry().available_upgrades()
10869 }
10870
10871 pub async fn check_upgrade_nudge(
10877 &self,
10878 inference_active: bool,
10879 ) -> (crate::nudge::NudgeDecision, crate::nudge::NudgeState) {
10880 let findings = self.detect_upgrades().await;
10881 let mut prefs = self.update_prefs();
10882 prefs.policy = self.effective_update_policy(&prefs, false).0;
10883 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10884 let now = std::time::SystemTime::now()
10885 .duration_since(std::time::UNIX_EPOCH)
10886 .map(|d| d.as_secs())
10887 .unwrap_or(0);
10888 let decision = crate::nudge::decide_nudge(
10889 &findings,
10890 &prefs,
10891 &state,
10892 now,
10893 crate::nudge::DEFAULT_THROTTLE_SECS,
10894 inference_active,
10895 );
10896 (decision, state)
10897 }
10898
10899 pub fn dismiss_upgrade_nudge(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10902 let path = crate::nudge::NudgeState::default_path();
10903 let mut state = crate::nudge::NudgeState::load_from(&path);
10904 state.dismiss(dismiss_key);
10905 state
10906 .save_to(&path)
10907 .map_err(InferenceError::InferenceFailed)
10908 }
10909
10910 pub async fn check_concierge(
10917 &self,
10918 inference_active: bool,
10919 ) -> (
10920 Vec<crate::concierge::ConciergeSuggestion>,
10921 crate::nudge::NudgeState,
10922 ) {
10923 let prefs = self.update_prefs();
10924 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10925 let hw = crate::hardware::HardwareInfo::detect();
10926 let schemas = self.list_schemas();
10927 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10928 let now = std::time::SystemTime::now()
10929 .duration_since(std::time::UNIX_EPOCH)
10930 .map(|d| d.as_secs())
10931 .unwrap_or(0);
10932 let suggestions = crate::concierge::decide_concierge(
10933 &refs,
10934 &hw,
10935 crate::concierge::DEFAULT_WATCHED_USE_CASES,
10936 crate::intent::QualityTier::Balanced,
10937 &prefs,
10938 &state,
10939 now,
10940 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
10941 inference_active,
10942 &self.checkpoint_geometry(refs.iter().copied()),
10943 );
10944 (suggestions, state)
10945 }
10946
10947 pub fn dismiss_concierge_suggestion(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10952 self.dismiss_upgrade_nudge(dismiss_key)
10953 }
10954
10955 pub fn dismiss_concierge_labeled(
10959 &self,
10960 dismiss_key: &str,
10961 reason: crate::concierge::DismissReason,
10962 ) -> Result<(), String> {
10963 let path = crate::nudge::NudgeState::default_path();
10964 let mut state = crate::nudge::NudgeState::load_from(&path);
10965 let now = std::time::SystemTime::now()
10966 .duration_since(std::time::UNIX_EPOCH)
10967 .map(|d| d.as_secs())
10968 .unwrap_or(0);
10969 state.dismiss_labeled(dismiss_key, reason, now);
10970 state.save_to(&path).map_err(|e| e.to_string())
10971 }
10972
10973 pub async fn check_canaries(&self) -> Vec<crate::intent::UseCase> {
10981 use crate::action_ledger::ConciergeActionKind;
10982 use crate::concierge::{
10983 canary_verdict, CanaryVerdict, CANARY_MIN_SAMPLES, CANARY_REGRESSION_MARGIN,
10984 };
10985 use std::collections::BTreeMap;
10986
10987 let actions = self.concierge_actions(0);
10988 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
10989 let entries = crate::outcome::read_ledger(&ledger_path, 0);
10990
10991 let mut latest: BTreeMap<
10993 crate::intent::UseCase,
10994 &crate::action_ledger::ConciergeActionEntry,
10995 > = BTreeMap::new();
10996 for a in &actions {
10997 if a.project.is_some() {
10998 continue; }
11000 if let Some(uc) = a.use_case {
11001 latest.insert(uc, a);
11002 }
11003 }
11004
11005 let mut reverts: Vec<(crate::intent::UseCase, u64)> = Vec::new();
11010 let tracker = self.outcome_tracker.read().await;
11011 for (uc, a) in latest {
11012 if a.kind != ConciergeActionKind::SetDefault {
11015 continue;
11016 }
11017 let Some(prior) = a.prior_model_id.as_deref() else {
11018 continue; };
11020
11021 let (mut succ, mut total) = (0u64, 0u64);
11023 for e in &entries {
11024 if e.timestamp < a.timestamp
11025 || e.model_id != a.model_id
11026 || crate::usage_profile::use_case_for_task(e.task) != uc
11027 {
11028 continue;
11029 }
11030 match e.success {
11031 Some(true) => {
11032 succ += 1;
11033 total += 1;
11034 }
11035 Some(false) => total += 1,
11036 None => {}
11037 }
11038 }
11039 let new_rate = if total == 0 {
11040 None
11041 } else {
11042 Some(succ as f64 / total as f64)
11043 };
11044
11045 let (mut base_succ, mut base_total) = (0u64, 0u64);
11050 if let Some(profile) = tracker.profile(prior) {
11051 for t in [
11052 crate::outcome::InferenceTask::Generate,
11053 crate::outcome::InferenceTask::Embed,
11054 crate::outcome::InferenceTask::Classify,
11055 crate::outcome::InferenceTask::Code,
11056 crate::outcome::InferenceTask::Reasoning,
11057 ] {
11058 if crate::usage_profile::use_case_for_task(t) != uc {
11059 continue;
11060 }
11061 if let Some(ts) = profile.task_stats(t) {
11062 base_succ += ts.successes;
11063 base_total += ts.successes + ts.failures;
11064 }
11065 }
11066 }
11067 if base_total == 0 {
11070 continue;
11071 }
11072 let baseline = base_succ as f64 / base_total as f64;
11073
11074 if canary_verdict(
11075 new_rate,
11076 total,
11077 baseline,
11078 CANARY_MIN_SAMPLES,
11079 CANARY_REGRESSION_MARGIN,
11080 ) == CanaryVerdict::Revert
11081 {
11082 reverts.push((uc, a.seq));
11086 }
11087 }
11088 drop(tracker);
11089
11090 let mut reverted = Vec::new();
11093 for (uc, seq) in reverts {
11094 if self.rollback_lane_inner(uc, None, Some(seq)).await.is_ok() {
11095 tracing::info!(lane = ?uc, "concierge canary: auto-reverted a worse model switch");
11096 reverted.push(uc);
11097 }
11098 }
11099 reverted
11100 }
11101
11102 pub async fn concierge_ask(
11119 &self,
11120 question: Option<&str>,
11121 ) -> Result<concierge_answer::ConciergeAnswer, String> {
11122 let maintenance = self.maintenance_pass(true).await;
11123 let portfolio = match maintenance.portfolio.clone() {
11124 Some(portfolio) => portfolio,
11125 None => self.portfolio().await,
11126 };
11127 let memory_policy =
11128 serde_json::to_value(&self.active_local_resource_policy().policy.profile)
11129 .ok()
11130 .and_then(|v| v.as_str().map(str::to_owned))
11131 .unwrap_or_else(|| "unknown".into());
11132 let summary = concierge_answer::summarize(&portfolio, &maintenance, &memory_policy);
11133 let Some(question) = question else {
11134 return Ok(concierge_answer::ConciergeAnswer {
11135 answer: summary.clone(),
11136 summary,
11137 model_used: None,
11138 ungrounded: Vec::new(),
11139 });
11140 };
11141 use std::fmt::Write as _;
11142 let status = self.concierge_status(false).await;
11143 let hw = crate::hardware::HardwareInfo::detect();
11144 let schemas = self.list_schemas();
11145 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11146
11147 let mut evidence = String::new();
11148 evidence.push_str("OBSERVED USAGE (last 30 days):\n");
11149 if status.lanes.is_empty() {
11150 evidence.push_str(" (no usage recorded yet)\n");
11151 }
11152 for lane in &status.lanes {
11153 let rate = lane
11154 .success_rate()
11155 .map(|r| format!("{:.0}% success", r * 100.0))
11156 .unwrap_or_else(|| "no resolved signal".into());
11157 let _ = writeln!(
11158 evidence,
11159 " {:?}: {} calls, {}{}",
11160 lane.use_case,
11161 lane.calls,
11162 rate,
11163 if lane.failing_models.is_empty() {
11164 String::new()
11165 } else {
11166 format!(
11167 ", failing on {}",
11168 lane.failing_models
11169 .iter()
11170 .cloned()
11171 .collect::<Vec<_>>()
11172 .join(", ")
11173 )
11174 }
11175 );
11176 }
11177 evidence.push_str("\nMODEL HEALTH:\n");
11178 for m in &status.models {
11179 let success = match m.success_rate {
11180 Some(r) => format!("{:.0}% success", r * 100.0),
11181 None => "no resolved signal".to_string(),
11182 };
11183 let _ = writeln!(
11184 evidence,
11185 " {}: {} calls, {}, {:.0}ms avg{}",
11186 m.model_id,
11187 m.calls,
11188 success,
11189 m.avg_latency_ms,
11190 if m.excluded { " (excluded)" } else { "" }
11191 );
11192 }
11193 let mut menu_lanes: Vec<crate::intent::UseCase> =
11199 crate::concierge::DEFAULT_WATCHED_USE_CASES.to_vec();
11200 for lane in &status.lanes {
11201 if !menu_lanes.contains(&lane.use_case) {
11202 menu_lanes.push(lane.use_case);
11203 }
11204 }
11205 evidence.push_str("\nGROUNDED CANDIDATES (fit verified for this machine):\n");
11206 let geometry = self.checkpoint_geometry(refs.iter().copied());
11207 for uc in menu_lanes {
11208 let set = crate::recommend::recommend(
11209 &refs,
11210 &hw,
11211 uc,
11212 crate::intent::QualityTier::Balanced,
11213 crate::intent::Privacy::OnDevice,
11214 &geometry,
11215 );
11216 for p in set.picks.iter().take(3) {
11217 let _ = writeln!(
11218 evidence,
11219 " [{:?}] {} — {}{}",
11220 uc,
11221 p.display_name,
11222 if p.already_installed {
11223 "installed"
11224 } else {
11225 "available"
11226 },
11227 if p.fit == crate::recommend::FitStatus::Fits {
11228 ", fits"
11229 } else {
11230 ", does NOT fit"
11231 }
11232 );
11233 }
11234 }
11235 if let Some(s) = &status.decision.suggestion {
11236 let _ = writeln!(evidence, "\nCURRENT SUGGESTION: {}", s.message);
11237 }
11238
11239 evidence.push_str("\nLOCAL MODELS ON THIS MACHINE:\n");
11240 evidence.push_str(&summary);
11241
11242 let prompt = format!(
11243 "You are CAR's model concierge. Answer the user's question ONLY from the \
11244 EVIDENCE provided as context: this machine's local models and what each is \
11245 doing, what maintenance would do, observed usage and health, and the \
11246 grounded candidate menu (the only models you may recommend acquiring). \
11247 Name models by the ids the evidence uses. NEVER invent a model and NEVER \
11248 claim a model fits or is better than the evidence states. If the evidence \
11249 doesn't answer the question, say so. Answer in a few short sentences or \
11250 bullets.\n\nQUESTION: {question}"
11251 );
11252 let req = crate::tasks::generate::GenerateRequest {
11253 prompt,
11254 context: Some(evidence.clone()),
11255 intent: Some(crate::intent::IntentHint {
11256 task: Some(crate::intent::TaskHint::Chat),
11257 prefer_local: true,
11258 ..Default::default()
11259 }),
11260 params: crate::tasks::generate::GenerateParams {
11261 temperature: 0.2,
11262 max_tokens: 600,
11263 workload: crate::tasks::generate::RoutingWorkload::Background,
11264 ..Default::default()
11265 },
11266 ..Default::default()
11267 };
11268 match self.generate_tracked(req).await {
11269 Ok(result) => {
11270 let reply = concierge_answer::trim_repetition(&result.text);
11271 let ungrounded = concierge_answer::ungrounded_ids(&reply, &evidence);
11272 let answer = if ungrounded.is_empty() {
11273 reply
11274 } else {
11275 format!(
11276 "{reply}\n\n(Not verified for this machine: {}. Only the models listed \
11277 above are checked.)",
11278 ungrounded.join(", ")
11279 )
11280 };
11281 Ok(concierge_answer::ConciergeAnswer {
11282 answer,
11283 summary,
11284 model_used: Some(result.model_used),
11285 ungrounded,
11286 })
11287 }
11288 Err(error) => {
11289 tracing::warn!(%error, "concierge.ask: no model could answer; returning the facts");
11290 Ok(concierge_answer::ConciergeAnswer {
11291 answer: format!(
11292 "{summary}\n(No model could answer the question itself: {error}.)"
11293 ),
11294 summary,
11295 model_used: None,
11296 ungrounded: Vec::new(),
11297 })
11298 }
11299 }
11300 }
11301
11302 pub async fn refresh_catalog(&self) -> Result<usize, String> {
11312 let url = catalog::catalog_url();
11313 let pubkey = catalog::catalog_public_key();
11314 let verified = self.fetch_checked_catalog(&url, &pubkey).await?;
11319 let path = crate::catalog::cache_path(&self.config.state_root);
11320 self.adopt_verified_catalog(&path, &verified, &pubkey).await
11321 }
11322
11323 async fn fetch_checked_catalog(
11328 &self,
11329 url: &str,
11330 pubkey: &str,
11331 ) -> Result<catalog::VerifiedCatalog, String> {
11332 use catalog::CheckFailure::{Older, Rejected, Unreachable};
11333 let http = crate::tls_client::catalog_refresh_client();
11334 let fetched = match crate::catalog::fetch_signed(&http, url).await {
11337 Err(error) => Err((Unreachable, error)),
11338 Ok((bytes, sig)) => {
11339 crate::catalog::verify_fetched(bytes, sig, pubkey).map_err(|e| (Rejected, e))
11340 }
11341 };
11342 let cached = crate::catalog::load_doc(
11346 &crate::catalog::cache_path(&self.config.state_root),
11347 Some(pubkey),
11348 )
11349 .map(|doc| doc.version);
11350 let outcome = match &fetched {
11351 Ok(verified) => match cached {
11352 Some(cached) if verified.version() < cached => Err((
11353 Older,
11354 format!(
11355 "the source served catalog v{}, older than the cached v{cached}",
11356 verified.version()
11357 ),
11358 )),
11359 _ => Ok(verified.version()),
11360 },
11361 Err((failure, error)) => Err((*failure, error.clone())),
11362 };
11363 catalog::record_check(
11364 &catalog::status_path(&self.config.state_root),
11365 url,
11366 outcome
11367 .as_ref()
11368 .map(|v| *v)
11369 .map_err(|(f, e)| (*f, e.as_str())),
11370 now_unix(),
11371 );
11372 fetched.map_err(|(_, error)| error)
11373 }
11374
11375 pub fn mark_catalog_loop_alive(&self) {
11377 catalog::record_loop_alive(&catalog::status_path(&self.config.state_root), now_unix());
11378 }
11379
11380 pub fn mark_daemon_alive_for_catalog(&self) {
11383 catalog::record_daemon_alive(&catalog::status_path(&self.config.state_root), now_unix());
11384 }
11385
11386 pub fn catalog_freshness(&self) -> catalog::CatalogFreshness {
11389 catalog::catalog_freshness(&catalog::load_status(&catalog::status_path(
11390 &self.config.state_root,
11391 )))
11392 }
11393
11394 pub async fn refresh_catalog_on_schedule(&self) -> Result<Option<usize>, String> {
11399 if matches!(self.update_prefs().policy, update_prefs::UpdatePolicy::Off) {
11400 catalog::record_off(&catalog::status_path(&self.config.state_root), now_unix());
11401 return Ok(None);
11402 }
11403 self.refresh_catalog_from(&catalog::catalog_url(), &catalog::catalog_public_key())
11404 .await
11405 }
11406
11407 async fn refresh_catalog_from(&self, url: &str, pubkey: &str) -> Result<Option<usize>, String> {
11409 let verified = self.fetch_checked_catalog(url, pubkey).await?;
11410 let path = crate::catalog::cache_path(&self.config.state_root);
11411 let cached = crate::catalog::load_doc(&path, Some(pubkey)).map(|doc| doc.version);
11412 if cached.is_some_and(|version| verified.version() <= version) {
11413 return Ok(None);
11414 }
11415 self.adopt_verified_catalog(&path, &verified, pubkey)
11416 .await
11417 .map(Some)
11418 }
11419
11420 async fn adopt_verified_catalog(
11427 &self,
11428 path: &Path,
11429 verified: &catalog::VerifiedCatalog,
11430 pubkey: &str,
11431 ) -> Result<usize, String> {
11432 let count = crate::catalog::install_if_newer(path, verified, pubkey).await?;
11433 let newest = crate::catalog::load_verified(path, Some(pubkey))
11434 .ok_or_else(|| format!("re-reading the verified catalog at {}", path.display()))?;
11435 let revoked = newest.revoked().to_vec();
11436 self.unlink_revoked_signed_models(&revoked).await;
11437 self.swap_signed_catalog(newest);
11438 self.clear_revoked_lane_defaults(&revoked).await;
11439 Ok(count)
11440 }
11441
11442 pub async fn enforce_catalog_revocations(&self) {
11447 let revoked = self.current_registry().revoked_ids();
11448 self.clear_revoked_lane_defaults(&revoked).await;
11449 }
11450
11451 async fn clear_revoked_lane_defaults(&self, revoked: &[String]) {
11459 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11460 if revoked.is_empty() {
11461 return;
11462 }
11463 let _guard = self.concierge_action_lock.lock().await;
11464 let now = std::time::SystemTime::now()
11465 .duration_since(std::time::UNIX_EPOCH)
11466 .map(|d| d.as_secs())
11467 .unwrap_or(0);
11468 for lane in self.lane_defaults().defaults {
11469 if !revoked.contains(&lane.model_id) {
11470 continue;
11471 }
11472 match self.clear_lane_default(lane.project.as_deref(), lane.use_case) {
11473 Ok(true) => self.record_concierge_action(ConciergeActionEntry {
11474 seq: 0,
11475 kind: ConciergeActionKind::RevokedLaneCleared,
11476 model_id: lane.model_id.clone(),
11477 use_case: Some(lane.use_case),
11478 project: lane.project.clone(),
11479 prior_model_id: Some(lane.model_id.clone()),
11480 detail: "revoked by the signed model catalog".into(),
11481 timestamp: now,
11482 }),
11483 Ok(false) => {}
11484 Err(error) => {
11485 tracing::warn!(model = %lane.model_id, %error, "could not clear a revoked lane default")
11486 }
11487 }
11488 }
11489 for pin in self.revoked_pins(&self.current_registry()) {
11490 tracing::warn!(model = %pin.model_id, pin = %pin.pin, agents = ?pin.agents, "a pinned model was revoked by the signed catalog; re-point it");
11491 }
11492 }
11493
11494 async fn unlink_revoked_signed_models(&self, revoked: &[String]) {
11503 let registry = self.current_registry();
11504 let names_path = catalog::revoked_names_path(&self.config.state_root);
11505 let mut names = catalog::load_revoked_names(&names_path);
11506 let models_path = catalog::revoked_models_path(&self.config.state_root);
11507 let mut models = catalog::load_revoked_names(&models_path);
11508 names.retain(|_, id| revoked.contains(id));
11510 models.retain(|_, id| revoked.contains(id));
11511 let mut installed = Vec::new();
11512 for id in revoked {
11513 let Some(row) = registry.get(id) else {
11514 continue;
11515 };
11516 if registry.signed_origin_of(id).is_none() || registry.is_builtin(id) {
11517 continue;
11518 }
11519 if !retire::single_component(&row.name) {
11524 continue;
11525 }
11526 models.insert(row.name.clone(), id.clone());
11527 if row.is_local() && row.downloads_weights() {
11528 names.insert(row.name.clone(), id.clone());
11529 installed.push((id.clone(), row.name.clone()));
11530 }
11531 }
11532 for (path, record) in [(&names_path, &names), (&models_path, &models)] {
11533 if let Err(error) = catalog::save_revoked_names(path, record) {
11534 tracing::warn!(path = %path.display(), %error, "could not record revoked model names");
11535 }
11536 }
11537 let models_dir = self.config.models_dir.clone();
11538 for (id, name) in installed {
11539 let projection = models_dir.join(&name);
11540 if std::fs::symlink_metadata(&projection).is_err() || !retire::links_only(&projection) {
11541 continue;
11542 }
11543 match self.model_management.load_receipt(&id) {
11544 Ok(Some(receipt))
11545 if receipt.artifact_kind == model_management::ManagedArtifactKind::Symlink =>
11546 {
11547 if let Err(error) = self.remove_model_from_car(&id).await {
11548 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
11549 }
11550 }
11551 Ok(Some(_)) => {}
11552 Ok(None) => {
11553 if let Err(error) = retire::remove_links_only_tree(&projection, &models_dir) {
11554 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
11555 }
11556 }
11557 Err(error) => {
11558 tracing::warn!(%id, %error, "could not read a revoked model's receipt")
11559 }
11560 }
11561 }
11562 }
11563
11564 pub async fn discover_models(&self) -> Result<usize, String> {
11573 use crate::schema::{ModelSource, TrustTier};
11574 let template = self
11581 .unified_registry()
11582 .all()
11583 .filter(|m| {
11584 m.provider.eq_ignore_ascii_case("openai")
11585 && m.trust_tier == TrustTier::Curated
11586 && matches!(m.source, ModelSource::RemoteApi { .. })
11587 })
11588 .max_by(|a, b| {
11589 a.capabilities
11590 .len()
11591 .cmp(&b.capabilities.len())
11592 .then_with(|| a.id.cmp(&b.id))
11593 })
11594 .cloned();
11595 let Some(template) = template else {
11596 return Ok(0); };
11598 let (endpoint, api_key_env) = match &template.source {
11599 ModelSource::RemoteApi {
11600 endpoint,
11601 api_key_env,
11602 ..
11603 } => (endpoint.clone(), api_key_env.clone()),
11604 _ => return Ok(0),
11605 };
11606 let Some(models_url) = crate::discovery::models_url_from_endpoint(&endpoint) else {
11607 return Ok(0);
11608 };
11609 let key = match car_secrets::resolve_env_or_keychain(&api_key_env) {
11610 Some(k) if !k.is_empty() => k,
11611 _ => return Ok(0), };
11613
11614 let http = reqwest::Client::builder()
11618 .timeout(std::time::Duration::from_secs(20))
11619 .build()
11620 .map_err(|e| format!("discovery client: {e}"))?;
11621 let ids = crate::discovery::fetch_model_ids(&http, &models_url, &key).await?;
11622
11623 let cache = crate::discovery::cache_path(&self.config.state_models_dir());
11625 let mut cached = crate::discovery::load_cache(&cache);
11626 let mut have: std::collections::HashSet<String> =
11627 cached.iter().map(|m| m.id.clone()).collect();
11628 for id in ids {
11629 if !crate::discovery::is_chat_model(&id) {
11630 continue;
11631 }
11632 let schema = crate::discovery::discovered_schema("openai", &id, &template);
11633 if self.unified_registry().get(&schema.id).is_some() {
11639 continue;
11640 }
11641 if have.insert(schema.id.clone()) {
11642 cached.push(schema);
11643 }
11644 }
11645 let count = cached.len();
11646 crate::discovery::save_cache(&cache, &cached)?;
11647 Ok(count)
11648 }
11649
11650 pub fn lane_defaults(&self) -> crate::lane_defaults::LaneDefaults {
11652 self.lane_defaults_cache.read().unwrap().clone()
11653 }
11654
11655 pub fn lane_default(
11659 &self,
11660 project: Option<&str>,
11661 use_case: crate::intent::UseCase,
11662 ) -> Option<String> {
11663 self.lane_defaults_cache
11664 .read()
11665 .unwrap()
11666 .resolve(project, use_case)
11667 .map(str::to_string)
11668 }
11669
11670 fn model_is_excluded(
11671 exclude_set: &std::collections::HashSet<String>,
11672 registry: &UnifiedRegistry,
11673 model: &str,
11674 ) -> bool {
11675 registry
11676 .get(model)
11677 .or_else(|| registry.find_by_name(model))
11678 .map(|schema| exclude_set.contains(&schema.id))
11679 .unwrap_or_else(|| exclude_set.contains(model))
11680 }
11681
11682 fn lane_pin_for(
11688 &self,
11689 req: &GenerateRequest,
11690 routing_registry: &UnifiedRegistry,
11691 ) -> Option<String> {
11692 let task = req.intent.as_ref().and_then(|h| h.task)?;
11693 let use_case = crate::usage_profile::use_case_for_task_hint(task);
11694 let id = self.lane_default(None, use_case)?;
11695 let known = routing_registry
11700 .get(&id)
11701 .or_else(|| routing_registry.find_by_name(&id));
11702 match known {
11703 Some(s) if s.available_now() => Some(id),
11704 _ => None,
11705 }
11706 }
11707
11708 pub fn set_lane_default(
11711 &self,
11712 project: Option<String>,
11713 use_case: crate::intent::UseCase,
11714 model_id: &str,
11715 ) -> Result<(), String> {
11716 if self.current_registry().is_revoked(model_id) {
11719 return Err(format!(
11720 "{model_id} was revoked by the signed model catalog and cannot be a lane default"
11721 ));
11722 }
11723 let now = std::time::SystemTime::now()
11724 .duration_since(std::time::UNIX_EPOCH)
11725 .map(|d| d.as_secs())
11726 .unwrap_or(0);
11727 let mut defaults = self.lane_defaults_cache.write().unwrap();
11730 defaults.set(project, use_case, model_id.to_string(), now);
11731 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11732 .map_err(|e| e.to_string())
11733 }
11734
11735 pub fn clear_lane_default(
11738 &self,
11739 project: Option<&str>,
11740 use_case: crate::intent::UseCase,
11741 ) -> Result<bool, String> {
11742 let mut defaults = self.lane_defaults_cache.write().unwrap();
11743 let removed = defaults.clear(project, use_case);
11744 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11745 .map_err(|e| e.to_string())?;
11746 Ok(removed)
11747 }
11748
11749 pub async fn user_set_lane_default(
11757 &self,
11758 use_case: crate::intent::UseCase,
11759 model_id: &str,
11760 project: Option<String>,
11761 ) -> Result<(), String> {
11762 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11763 let _guard = self.concierge_action_lock.lock().await;
11764 let prior = self.lane_default(project.as_deref(), use_case);
11765 self.set_lane_default(project.clone(), use_case, model_id)?;
11766 self.record_concierge_action(ConciergeActionEntry {
11767 seq: 0,
11768 kind: ConciergeActionKind::SetDefault,
11769 model_id: model_id.to_string(),
11770 use_case: Some(use_case),
11771 project,
11772 prior_model_id: prior,
11773 detail: "user set lane default".into(),
11774 timestamp: now_unix(),
11775 });
11776 Ok(())
11777 }
11778
11779 pub async fn user_clear_lane_default(
11784 &self,
11785 use_case: crate::intent::UseCase,
11786 project: Option<String>,
11787 ) -> Result<bool, String> {
11788 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11789 let _guard = self.concierge_action_lock.lock().await;
11790 let prior = self.lane_default(project.as_deref(), use_case);
11791 let removed = self.clear_lane_default(project.as_deref(), use_case)?;
11792 if removed {
11793 self.record_concierge_action(ConciergeActionEntry {
11794 seq: 0,
11795 kind: ConciergeActionKind::ClearDefault,
11796 model_id: prior.clone().unwrap_or_default(),
11797 use_case: Some(use_case),
11798 project,
11799 prior_model_id: prior,
11800 detail: "user cleared lane default".into(),
11801 timestamp: now_unix(),
11802 });
11803 }
11804 Ok(removed)
11805 }
11806
11807 pub fn concierge_actions(
11809 &self,
11810 limit: usize,
11811 ) -> Vec<crate::action_ledger::ConciergeActionEntry> {
11812 crate::action_ledger::read_actions(&self.action_ledger_path(), limit)
11813 }
11814
11815 fn lane_defaults_path(&self) -> PathBuf {
11817 self.config
11818 .state_root
11819 .join(crate::lane_defaults::LANE_DEFAULTS_FILE)
11820 }
11821
11822 fn action_ledger_path(&self) -> PathBuf {
11825 self.config
11826 .state_root
11827 .join(crate::action_ledger::ACTION_LEDGER_FILE)
11828 }
11829
11830 fn record_concierge_action(&self, mut entry: crate::action_ledger::ConciergeActionEntry) {
11831 entry.seq = self
11832 .concierge_action_seq
11833 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
11834 if let Err(e) = crate::action_ledger::append_action(&self.action_ledger_path(), &entry) {
11835 tracing::debug!("record concierge action failed: {e}");
11836 }
11837 }
11838
11839 pub async fn apply_concierge(
11851 &self,
11852 use_case: crate::intent::UseCase,
11853 model_id: &str,
11854 project: Option<String>,
11855 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11856 self.with_pinned_registry(self.apply_concierge_unpinned(use_case, model_id, project))
11857 .await
11858 }
11859
11860 async fn apply_concierge_unpinned(
11861 &self,
11862 use_case: crate::intent::UseCase,
11863 model_id: &str,
11864 project: Option<String>,
11865 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11866 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11867 let now = || {
11868 std::time::SystemTime::now()
11869 .duration_since(std::time::UNIX_EPOCH)
11870 .map(|d| d.as_secs())
11871 .unwrap_or(0)
11872 };
11873 if let Some(schema) = self
11878 .list_schemas()
11879 .into_iter()
11880 .find(|s| s.id == model_id || s.name == model_id)
11881 {
11882 let serves = use_case
11883 .required_capabilities()
11884 .iter()
11885 .all(|c| schema.capabilities.contains(c));
11886 if !serves {
11887 return Err(format!(
11888 "model '{model_id}' does not serve the {use_case:?} lane"
11889 ));
11890 }
11891 }
11892
11893 if self
11901 .unified_registry()
11902 .existing_local_artifact(model_id)
11903 .is_none()
11904 {
11905 self.pull_model(model_id).await.map_err(|e| e.to_string())?;
11906 }
11907
11908 let _guard = self.concierge_action_lock.lock().await;
11911 let prior = self.lane_default(project.as_deref(), use_case);
11914 self.record_concierge_action(ConciergeActionEntry {
11915 seq: 0, kind: ConciergeActionKind::Install,
11917 model_id: model_id.to_string(),
11918 use_case: Some(use_case),
11919 project: project.clone(),
11920 prior_model_id: None,
11921 detail: "concierge apply: ensured model present".into(),
11922 timestamp: now(),
11923 });
11924
11925 self.set_lane_default(project.clone(), use_case, model_id)?;
11927 self.record_concierge_action(ConciergeActionEntry {
11928 seq: 0, kind: ConciergeActionKind::SetDefault,
11930 model_id: model_id.to_string(),
11931 use_case: Some(use_case),
11932 project: project.clone(),
11933 prior_model_id: prior.clone(),
11934 detail: "concierge apply: set lane default".into(),
11935 timestamp: now(),
11936 });
11937
11938 Ok(crate::action_ledger::ConciergeApplyResult {
11939 model_id: model_id.to_string(),
11940 use_case,
11941 installed: true,
11942 set_default: true,
11943 prior_model_id: prior,
11944 })
11945 }
11946
11947 pub async fn rollback_lane(
11952 &self,
11953 use_case: crate::intent::UseCase,
11954 project: Option<String>,
11955 ) -> Result<Option<String>, String> {
11956 self.rollback_lane_inner(use_case, project, None).await
11957 }
11958
11959 async fn rollback_lane_inner(
11965 &self,
11966 use_case: crate::intent::UseCase,
11967 project: Option<String>,
11968 expected_anchor_seq: Option<u64>,
11969 ) -> Result<Option<String>, String> {
11970 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11971 let _guard = self.concierge_action_lock.lock().await;
11972 let actions = self.concierge_actions(0);
11977 let anchor = actions.iter().rev().find(|a| {
11978 matches!(
11979 a.kind,
11980 ConciergeActionKind::SetDefault | ConciergeActionKind::Rollback
11981 ) && a.use_case == Some(use_case)
11982 && a.project == project
11983 });
11984 let set = match anchor {
11985 None => return Err("no prior set-default to roll back".into()),
11986 Some(a) if a.kind == ConciergeActionKind::Rollback => {
11987 return Err("already rolled back to the prior default; nothing to undo".into())
11988 }
11989 Some(a) => a,
11990 };
11991 if let Some(seq) = expected_anchor_seq {
11994 if set.seq != seq {
11995 return Err("lane default changed since the canary decision; not reverting".into());
11996 }
11997 }
11998 let prior = set.prior_model_id.clone();
11999 let now = std::time::SystemTime::now()
12000 .duration_since(std::time::UNIX_EPOCH)
12001 .map(|d| d.as_secs())
12002 .unwrap_or(0);
12003
12004 match &prior {
12005 Some(m) => self.set_lane_default(project.clone(), use_case, m)?,
12006 None => {
12007 self.clear_lane_default(project.as_deref(), use_case)?;
12008 }
12009 }
12010 let detail = match &prior {
12011 Some(_) => "concierge rollback: restored prior lane default",
12012 None => "concierge rollback: cleared lane default (no prior)",
12013 };
12014 self.record_concierge_action(ConciergeActionEntry {
12015 seq: 0, kind: ConciergeActionKind::Rollback,
12017 model_id: prior.clone().unwrap_or_default(),
12018 use_case: Some(use_case),
12019 project,
12020 prior_model_id: Some(set.model_id.clone()),
12021 detail: detail.into(),
12022 timestamp: now,
12023 });
12024 Ok(prior)
12025 }
12026
12027 pub async fn concierge_status(
12032 &self,
12033 inference_active: bool,
12034 ) -> crate::concierge::ConciergeStatus {
12035 const USAGE_WINDOW_SECS: u64 = 30 * 24 * 60 * 60;
12037
12038 let prefs = self.update_prefs();
12039 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
12040 let hw = crate::hardware::HardwareInfo::detect();
12041 let schemas = self.list_schemas();
12042 let refs: Vec<&ModelSchema> = schemas.iter().collect();
12043 let now = std::time::SystemTime::now()
12044 .duration_since(std::time::UNIX_EPOCH)
12045 .map(|d| d.as_secs())
12046 .unwrap_or(0);
12047
12048 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12049 let entries = crate::outcome::read_ledger(&ledger_path, 0);
12050 let usage =
12051 crate::usage_profile::UsageProfile::from_ledger(&entries, now, USAGE_WINDOW_SECS);
12052
12053 let decision = crate::concierge::evaluate_concierge(
12054 &refs,
12055 &hw,
12056 &usage,
12057 crate::intent::QualityTier::Balanced,
12058 &prefs,
12059 &state,
12060 now,
12061 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
12062 inference_active,
12063 &self.checkpoint_geometry(refs.iter().copied()),
12064 );
12065
12066 let tracker = self.outcome_tracker.read().await;
12067 let models = tracker
12068 .export_profiles()
12069 .iter()
12070 .map(|p| crate::concierge::ModelHealth {
12071 model_id: p.model_id.clone(),
12072 calls: p.total_calls,
12073 success_rate: p.success_rate_resolved(),
12077 avg_latency_ms: p.avg_latency_ms(),
12078 quality: p.ema_quality,
12079 excluded: tracker.is_excluded(&p.model_id),
12080 })
12081 .collect();
12082 drop(tracker);
12083
12084 const STALE_VERIFY_SECS: u64 = 14 * 24 * 60 * 60;
12089 let actions = self.concierge_actions(0);
12090 let mut latest: std::collections::BTreeMap<
12091 crate::intent::UseCase,
12092 &crate::action_ledger::ConciergeActionEntry,
12093 > = std::collections::BTreeMap::new();
12094 for a in &actions {
12095 if a.project.is_none() {
12096 if let Some(uc) = a.use_case {
12097 latest.insert(uc, a);
12098 }
12099 }
12100 }
12101 let mut pending_verification = Vec::new();
12102 for (uc, a) in latest {
12103 if a.kind != crate::action_ledger::ConciergeActionKind::SetDefault {
12104 continue;
12105 }
12106 if now.saturating_sub(a.timestamp) < STALE_VERIFY_SECS {
12107 continue; }
12109 let resolved = entries
12110 .iter()
12111 .filter(|e| {
12112 e.timestamp >= a.timestamp
12113 && e.model_id == a.model_id
12114 && crate::usage_profile::use_case_for_task(e.task) == uc
12115 && e.success.is_some()
12116 })
12117 .count() as u64;
12118 if resolved < crate::concierge::CANARY_MIN_SAMPLES {
12119 pending_verification.push(crate::concierge::PendingVerification {
12120 use_case: uc,
12121 model_id: a.model_id.clone(),
12122 set_at: a.timestamp,
12123 resolved_samples: resolved,
12124 needed: crate::concierge::CANARY_MIN_SAMPLES,
12125 });
12126 }
12127 }
12128
12129 crate::concierge::ConciergeStatus {
12130 lanes: usage.active_lanes().into_iter().cloned().collect(),
12131 decision,
12132 models,
12133 pending_verification,
12134 catalog: self.catalog_freshness(),
12135 revoked_pins: self.revoked_pins(&self.current_registry()),
12136 }
12137 }
12138
12139 pub async fn detect_upgrades(&self) -> Vec<crate::upgrade::UpgradeFinding> {
12143 let prefs = self.update_prefs();
12144 let curated = self.unified_registry().available_upgrades();
12145 let schemas = self.list_schemas();
12146 let refs: Vec<&ModelSchema> = schemas.iter().collect();
12147 let probe = crate::upgrade::HuggingFaceProbe::new();
12148 let now = std::time::SystemTime::now()
12149 .duration_since(std::time::UNIX_EPOCH)
12150 .map(|d| d.as_secs())
12151 .unwrap_or(0);
12152 let mut findings = crate::upgrade::detect_upgrades(
12153 curated,
12154 &refs,
12155 &prefs,
12156 &probe,
12157 &crate::upgrade::UpgradeCache::default_path(),
12158 now,
12159 crate::upgrade::DEFAULT_TTL_SECS,
12160 )
12161 .await;
12162 let lane_models: std::collections::HashSet<String> = self
12167 .lane_defaults()
12168 .defaults
12169 .into_iter()
12170 .map(|d| d.model_id)
12171 .collect();
12172 let in_use = |m: &ModelSchema| {
12173 lane_models.contains(&m.id)
12174 || self.model_management.last_used(&m.id).is_some_and(|at| {
12175 now.saturating_sub(at) < portfolio::PortfolioPolicy::DEFAULT_IDLE_AFTER_SECS
12176 })
12177 };
12178 let installed: Vec<&ModelSchema> = refs
12179 .iter()
12180 .copied()
12181 .filter(|m| {
12182 m.is_local()
12183 && m.downloads_weights()
12184 && m.weights_ready
12185 && self.model_management.car_enabled(&m.id).unwrap_or(true)
12186 && in_use(m)
12187 })
12188 .collect();
12189 let fit = |m: &ModelSchema| {
12190 let fit = self.model_fit(m);
12191 fit.platform_compatible.then_some(fit.fit)
12192 };
12193 let registry = self.unified_registry();
12194 let found = crate::upgrade::suggest_only_unless_builtin(
12195 crate::upgrade::benchmark_findings(&installed, &refs, &fit),
12196 &|id| registry.is_builtin(id),
12197 );
12198 for finding in found {
12199 let duplicate = findings
12200 .iter()
12201 .any(|f| f.from_id == finding.from_id && f.to_id == finding.to_id);
12202 if !duplicate {
12203 findings.push(finding);
12204 }
12205 }
12206 findings
12207 }
12208
12209 pub fn list_schemas(&self) -> Vec<ModelSchema> {
12212 self.catalog_registry_snapshot()
12213 .list()
12214 .into_iter()
12215 .cloned()
12216 .collect()
12217 }
12218
12219 pub fn catalog_snapshot(&self) -> Result<CatalogSnapshot, String> {
12223 CatalogSnapshot::new(self.unified_registry().list().into_iter().cloned())
12224 }
12225
12226 pub fn registered_schema(&self, id: &str) -> Option<ModelSchema> {
12231 self.unified_registry().registered_schema(id).cloned()
12232 }
12233
12234 pub fn list_models(&self) -> Vec<models::ModelInfo> {
12235 self.registry.list_models()
12236 }
12237
12238 pub fn knows_model(&self, name: &str) -> bool {
12251 self.model_schema(name).is_some()
12252 }
12253
12254 pub fn model_schema(&self, name: &str) -> Option<ModelSchema> {
12272 self.resolve_schema(name)
12273 }
12274
12275 fn first_installed_local_model(&self, needs_tools: bool) -> Option<String> {
12299 let registry = self.unified_registry();
12300 let mut candidates: Vec<&ModelSchema> = self
12301 .last_resort_pool(®istry, needs_tools)
12302 .into_iter()
12303 .filter(|s| self.schema_installed_through_car(s, ®istry))
12304 .collect();
12305 candidates.sort_by(|a, b| {
12306 b.has_capability(ModelCapability::ToolUse)
12307 .cmp(&a.has_capability(ModelCapability::ToolUse))
12308 .then(b.size_mb().cmp(&a.size_mb()))
12309 .then_with(|| a.id.cmp(&b.id))
12310 });
12311 candidates.first().map(|s| s.name.clone())
12312 }
12313
12314 fn cache_only_local_models(&self, needs_tools: bool) -> Vec<String> {
12321 let registry = self.unified_registry();
12322 let mut ids: Vec<String> = self
12323 .last_resort_pool(®istry, needs_tools)
12324 .into_iter()
12325 .filter(|s| !self.schema_installed_through_car(s, ®istry))
12326 .map(|s| s.id.clone())
12327 .collect();
12328 ids.sort();
12329 ids
12330 }
12331
12332 fn last_resort_pool<'r>(
12335 &self,
12336 registry: &'r UnifiedRegistry,
12337 needs_tools: bool,
12338 ) -> Vec<&'r ModelSchema> {
12339 let mut candidates: Vec<&ModelSchema> = registry
12347 .all()
12348 .filter(|s| s.is_local() && !s.is_vllm_mlx())
12355 .filter(|s| !s.is_foundation_models() || s.available)
12371 .filter(|s| s.has_capability(ModelCapability::Generate))
12372 .filter(|s| !needs_tools || s.has_capability(ModelCapability::ToolUse))
12376 .filter(|s| self.model_management.car_enabled(&s.id).unwrap_or(false))
12377 .filter(|s| registry.ready_without_download(&s.id) == Some(true))
12378 .collect();
12379
12380 if let Some(avail) = crate::hardware::available_ram_mb() {
12392 const WORKING_RESERVE_MB: u64 = 2048;
12393 candidates.retain(|s| s.size_mb().saturating_add(WORKING_RESERVE_MB) <= avail);
12394 }
12395 candidates
12396 }
12397
12398 pub fn installed_through_car(&self, id: &str) -> bool {
12405 let registry = self.unified_registry();
12406 registry
12407 .get(id)
12408 .or_else(|| registry.find_by_name(id))
12409 .is_some_and(|schema| self.schema_installed_through_car(schema, ®istry))
12410 }
12411
12412 fn schema_installed_through_car(
12426 &self,
12427 schema: &ModelSchema,
12428 registry: &UnifiedRegistry,
12429 ) -> bool {
12430 if !schema.downloads_weights() {
12431 return true;
12432 }
12433 if matches!(self.model_management.load_receipt(&schema.id), Ok(Some(_))) {
12434 return true;
12435 }
12436 registry
12437 .existing_local_artifact(&schema.id)
12438 .is_some_and(|artifact| artifact.starts_with(registry.models_dir()))
12439 }
12440
12441 pub async fn pull_model(&self, name: &str) -> Result<std::path::PathBuf, InferenceError> {
12443 self.pull_model_with_progress(name, &crate::download::ProgressSink::none())
12444 .await
12445 }
12446
12447 pub async fn pull_model_with_progress(
12452 &self,
12453 name: &str,
12454 sink: &crate::download::ProgressSink,
12455 ) -> Result<std::path::PathBuf, InferenceError> {
12456 self.with_pinned_registry(async {
12457 let pulled = self.pull_model_with_progress_unpinned(name, sink).await;
12458 if pulled.is_ok() {
12459 self.keep_pulled_row_if_withdrawn(name);
12460 }
12461 pulled
12462 })
12463 .await
12464 }
12465
12466 async fn pull_model_with_progress_unpinned(
12467 &self,
12468 name: &str,
12469 sink: &crate::download::ProgressSink,
12470 ) -> Result<std::path::PathBuf, InferenceError> {
12471 let schema = self
12472 .resolve_schema_name_first(name)
12473 .ok_or_else(|| InferenceError::ModelNotFound(name.to_string()))?;
12474 let _mutation = self.model_management.begin_mutation(&schema.id)?;
12475 if let Some(receipt) = self.model_management.load_receipt(&schema.id)? {
12476 let _ = self.model_management.can_remove(&schema.id)?;
12480 self.model_management.clear_tombstone(&schema.id)?;
12481 return Ok(receipt.managed_path);
12482 }
12483 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
12484 return Ok(receipt.managed_path);
12485 }
12486 let expected_managed = self.model_management.models_dir().join(&schema.name);
12487 if std::fs::symlink_metadata(&expected_managed).is_ok() {
12488 return Err(InferenceError::InferenceFailed(format!(
12489 "model {} has a pre-existing unreceipted artifact at {}; use models.adopt or move it before pulling",
12490 schema.id,
12491 expected_managed.display()
12492 )));
12493 }
12494 let staging = self.model_management.create_install_staging(&schema.id)?;
12495 let installed = match self
12496 .unified_registry()
12497 .ensure_local_with_progress_staged(&schema.id, sink, &staging)
12498 .await
12499 {
12500 Ok(installed) => installed,
12501 Err(error) => {
12502 self.model_management.discard_install_staging(&staging);
12503 return Err(error);
12504 }
12505 };
12506 let installed_is_staging = match (installed.canonicalize(), staging.canonicalize()) {
12507 (Ok(installed), Ok(staging)) => installed == staging,
12508 _ => false,
12509 };
12510 let generation = self
12511 .resource_policy_generation
12512 .load(std::sync::atomic::Ordering::Acquire);
12513 let managed_path = self.model_management.models_dir().join(&schema.name);
12514 let managed = if installed_is_staging {
12515 let receipt = self.model_management.install_receipt_for_publication(
12516 &schema.id,
12517 model_source_identity(&schema),
12518 generation,
12519 false,
12520 managed_path,
12521 model_management::ManagedArtifactKind::Directory,
12522 &staging,
12523 )?;
12524 self.model_management
12525 .begin_install_intent(receipt, Some(&staging))?;
12526 self.model_management
12527 .publish_install_staging(&schema.id, &staging, &schema.name)?
12528 } else {
12529 self.model_management.discard_install_staging(&staging);
12530 let receipt = self.model_management.install_receipt_for_publication(
12531 &schema.id,
12532 model_source_identity(&schema),
12533 generation,
12534 false,
12535 managed_path,
12536 model_management::ManagedArtifactKind::Symlink,
12537 &installed,
12538 )?;
12539 self.model_management.begin_install_intent(receipt, None)?;
12540 self.model_management
12541 .materialize_managed_projection(&schema.name, &installed)?
12542 };
12543 let receipt = self
12544 .model_management
12545 .resume_install_intent(&schema.id)?
12546 .ok_or_else(|| {
12547 InferenceError::InferenceFailed(format!(
12548 "model {} publication completed without a durable install intent",
12549 schema.id
12550 ))
12551 })?;
12552 debug_assert_eq!(receipt.managed_path, managed);
12553 Ok(receipt.managed_path)
12554 }
12555
12556 pub async fn adopt_model_into_car(
12560 &self,
12561 model_id: &str,
12562 ) -> Result<model_management::InstallReceipt, InferenceError> {
12563 let schema = self
12564 .resolve_schema(model_id)
12565 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
12566 if !schema.is_local() || !schema.downloads_weights() {
12567 return Err(InferenceError::InferenceFailed(format!(
12568 "model {} is not a CAR-manageable local artifact",
12569 schema.id
12570 )));
12571 }
12572 let existing = self
12573 .unified_registry()
12574 .existing_local_artifact(&schema.id)
12575 .ok_or_else(|| {
12576 InferenceError::InferenceFailed(format!(
12577 "model {} has no usable local artifact to adopt",
12578 schema.id
12579 ))
12580 })?;
12581 let _mutation = self.model_management.begin_mutation(&schema.id)?;
12582 let existing_is_managed_symlink = std::fs::symlink_metadata(&existing)
12583 .is_ok_and(|metadata| metadata.file_type().is_symlink())
12584 && existing.parent().is_some_and(|parent| {
12585 parent.canonicalize().ok() == self.model_management.models_dir().canonicalize().ok()
12586 });
12587 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
12588 return Ok(receipt);
12589 }
12590 let generation = self
12591 .resource_policy_generation
12592 .load(std::sync::atomic::Ordering::Acquire);
12593 let managed = if existing_is_managed_symlink {
12594 existing
12595 } else {
12596 let managed = self.model_management.adopted_projection_path(&schema.id)?;
12597 let receipt = self.model_management.install_receipt_for_publication(
12598 &schema.id,
12599 model_source_identity(&schema),
12600 generation,
12601 true,
12602 managed,
12603 model_management::ManagedArtifactKind::Symlink,
12604 &existing,
12605 )?;
12606 self.model_management.begin_install_intent(receipt, None)?;
12607 self.model_management
12608 .materialize_adopted_projection(&schema.id, &existing)?
12609 };
12610 if existing_is_managed_symlink {
12611 self.model_management
12612 .record_managed_artifact(
12613 &schema.id,
12614 model_source_identity(&schema),
12615 None,
12616 generation,
12617 true,
12618 managed,
12619 )
12620 .map_err(InferenceError::from)
12621 } else {
12622 self.model_management
12623 .resume_install_intent(&schema.id)?
12624 .ok_or_else(|| {
12625 InferenceError::InferenceFailed(format!(
12626 "model {} adoption completed without a durable install intent",
12627 schema.id
12628 ))
12629 })
12630 }
12631 }
12632
12633 pub async fn remove_model_from_car(
12636 &self,
12637 model_id: &str,
12638 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
12639 self.remove_model_from_car_inner(model_id, false).await
12640 }
12641
12642 async fn remove_model_from_car_inner(
12646 &self,
12647 model_id: &str,
12648 maintenance_held: bool,
12649 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
12650 let schema = self
12651 .resolve_schema(model_id)
12652 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
12653 if !schema.is_local() || !schema.downloads_weights() {
12654 return Err(InferenceError::InferenceFailed(format!(
12655 "model {} is not a CAR-owned local artifact",
12656 schema.id
12657 )));
12658 }
12659 if self.model_management.load_receipt(&schema.id)?.is_none() {
12660 if self.model_management.car_enabled(&schema.id)? {
12661 return Err(model_management::ModelManagementError::MissingReceipt {
12662 model_id: schema.id.clone(),
12663 }
12664 .into());
12665 }
12666 let generation = self
12670 .resource_policy_generation
12671 .load(std::sync::atomic::Ordering::Acquire)
12672 .saturating_add(1);
12673 return self
12674 .model_management
12675 .begin_mutation(&schema.id)?
12676 .remove(generation)
12677 .map_err(InferenceError::from);
12678 }
12679 let mutation = self.model_management.begin_mutation(&schema.id)?;
12680 let _maintenance = if maintenance_held {
12681 None
12682 } else {
12683 Some(
12684 self.prepare_local_model_removal(&schema.id)
12685 .await
12686 .map_err(|error| InferenceError::InferenceFailed(error.to_string()))?,
12687 )
12688 };
12689 let generation = self
12690 .resource_policy_generation
12691 .load(std::sync::atomic::Ordering::Acquire)
12692 .saturating_add(1);
12693 mutation.remove(generation).map_err(InferenceError::from)
12694 }
12695
12696 pub fn update_prefs(&self) -> crate::update_prefs::UpdatePreferences {
12701 let cwd = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("."));
12702 crate::update_prefs::UpdatePreferences::load_effective(&cwd).unwrap_or_else(|error| {
12703 tracing::warn!(%error, "update preferences unreadable; acting as notify");
12706 crate::update_prefs::UpdatePreferences {
12707 policy: crate::update_prefs::UpdatePolicy::Notify,
12708 policy_chosen: true,
12709 ..Default::default()
12710 }
12711 })
12712 }
12713
12714 pub fn set_update_prefs(
12716 &self,
12717 prefs: &crate::update_prefs::UpdatePreferences,
12718 ) -> Result<(), InferenceError> {
12719 prefs.save().map_err(InferenceError::InferenceFailed)
12720 }
12721
12722 #[deprecated(note = "use async remove_model_from_car for receipt-backed safe removal")]
12726 pub fn remove_model(&self, name: &str) -> Result<(), InferenceError> {
12727 Err(InferenceError::InferenceFailed(format!(
12728 "legacy removal for {name} is disabled; use async receipt-backed remove_model_from_car"
12729 )))
12730 }
12731
12732 pub fn register_model(&mut self, schema: ModelSchema) {
12738 self.unified_registry_mut().register(schema);
12739 }
12740
12741 pub fn register_user_model(&mut self, schema: ModelSchema) {
12743 self.unified_registry_mut().register_user_model(schema);
12744 }
12745
12746 pub async fn discover_vllm_mlx_models(&mut self) -> usize {
12749 let config = vllm_mlx::VllmMlxConfig::default();
12750 if !config.auto_discover {
12751 return 0;
12752 }
12753 vllm_mlx::discover_and_register(&config, self.unified_registry_mut()).await
12754 }
12755
12756 pub fn outcome_tracker(&self) -> Arc<RwLock<OutcomeTracker>> {
12758 self.outcome_tracker.clone()
12759 }
12760
12761 async fn auto_save_outcomes(&self) {
12769 const OUTCOME_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
12770
12771 if crate::offload::is_offload_worker() {
12779 return;
12780 }
12781
12782 let due = {
12784 let last = self.last_outcome_flush.lock().unwrap();
12785 last.is_none_or(|t| t.elapsed() >= OUTCOME_FLUSH_INTERVAL)
12786 };
12787 if due {
12788 match self.persist_outcomes().await {
12789 Ok(did) => {
12790 if did {
12791 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12792 }
12793 }
12794 Err(e) => tracing::debug!("auto-save outcomes failed: {}", e),
12795 }
12796 }
12797
12798 if let Err(e) = self.save_key_pool_stats().await {
12799 tracing::debug!("auto-save key pool stats failed: {}", e);
12800 }
12801 }
12802
12803 async fn persist_outcomes(&self) -> Result<bool, std::io::Error> {
12809 const PENDING_TTL_SECS: u64 = 300;
12810 let entries = {
12814 let mut tracker = self.outcome_tracker.write().await;
12815 tracker.sweep_pending(PENDING_TTL_SECS);
12816 tracker.drain_ledger()
12817 };
12818 let mut did = false;
12819 let ledger_disabled = std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some();
12824 if !entries.is_empty() && !ledger_disabled {
12825 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12826 let _guard = self.ledger_io_lock.lock().await;
12827 crate::outcome::append_ledger_entries(&ledger_path, &entries)?;
12828 did = true;
12829 }
12830 let profiles_path = self.config.state_models_dir().join("outcome_profiles.json");
12832 let wrote = {
12833 let mut tracker = self.outcome_tracker.write().await;
12834 tracker.save_if_dirty(&profiles_path)?
12835 };
12836 Ok(did || wrote)
12837 }
12838
12839 pub async fn save_outcomes(&self) -> Result<(), std::io::Error> {
12844 let tracker = self.outcome_tracker.read().await;
12845 let path = self.config.state_models_dir().join("outcome_profiles.json");
12846 tracker.save_to_file(&path)
12847 }
12848
12849 pub async fn flush_outcomes(&self) -> Result<bool, std::io::Error> {
12855 self.flush_outcomes_inner().await
12856 }
12857
12858 pub async fn reconcile_outcome_ledger(
12874 &self,
12875 ) -> Result<outcome::LedgerReconciliation, std::io::Error> {
12876 if crate::offload::is_offload_worker()
12877 || std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some()
12878 || !self.outcome_profiles_loaded
12879 {
12880 return Ok(outcome::LedgerReconciliation::default());
12881 }
12882 let models_dir = self.config.state_models_dir();
12883 let ledger = models_dir.join("outcome_ledger.jsonl");
12884 let _ = std::fs::remove_file(ledger.with_extension("jsonl.migrated"));
12887 self.flush_outcomes_inner().await?;
12888 let seeds = quality_seeds(&models_dir);
12889 let _io = self.ledger_io_lock.lock().await;
12890 let registry = &self.unified_registry();
12891 let is_registered = |id: &str| registry.get(id).is_some();
12892 let mut tracker = self.outcome_tracker.write().await;
12893 let plan = outcome::plan_reconcile(&ledger, &tracker, &is_registered, &seeds);
12894 if plan.ledger.is_none() && plan.profiles.is_none() {
12895 return Ok(plan.report);
12896 }
12897 if let Some(entries) = &plan.ledger {
12898 outcome::write_reconciled_ledger(&ledger, entries)?;
12899 }
12900 if let Some(profiles) = plan.profiles {
12901 tracker.commit_reconciled(profiles);
12902 tracker.save_to_file(&models_dir.join("outcome_profiles.json"))?;
12903 }
12904 tracing::info!(
12905 reclassified = plan.report.reclassified,
12906 unregistered = plan.report.unregistered,
12907 profiles_rebuilt = plan.report.profiles_rebuilt,
12908 profiles_removed = plan.report.profiles_removed,
12909 "reconciled the outcome ledger with model profiles"
12910 );
12911 Ok(plan.report)
12912 }
12913
12914 async fn flush_outcomes_inner(&self) -> Result<bool, std::io::Error> {
12915 let did = self.persist_outcomes().await?;
12916 if did {
12917 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12918 }
12919 Ok(did)
12920 }
12921
12922 pub async fn prune_outcome_ledger(&self, max_entries: usize) -> std::io::Result<()> {
12925 let path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12926 let _guard = self.ledger_io_lock.lock().await;
12927 crate::outcome::prune_ledger(&path, max_entries)
12928 }
12929
12930 pub async fn save_key_pool_stats(&self) -> Result<(), std::io::Error> {
12932 let path = self.config.state_models_dir().join("key_pool_stats.json");
12933 self.remote_backend.key_pool.save_stats(&path).await
12934 }
12935
12936 pub async fn key_pool_stats(
12938 &self,
12939 ) -> std::collections::HashMap<String, Vec<key_pool::KeyStats>> {
12940 self.remote_backend.key_pool.all_stats().await
12941 }
12942
12943 pub async fn export_profiles(&self) -> Vec<ModelProfile> {
12945 let tracker = self.outcome_tracker.read().await;
12946 tracker.export_profiles()
12947 }
12948
12949 pub fn outcome_scoreboard(&self) -> crate::scoreboard::Scoreboard {
12957 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12958 let entries = crate::outcome::read_ledger(&ledger_path, 0);
12959 crate::calibration::ShadowCalibration::from_ledger(&entries).emit();
12963 crate::scoreboard::Scoreboard::from_ledger(&entries, |id| {
12964 let s = self.resolve_schema(id)?;
12965 scoreboard_price_model(&s)
12966 })
12967 }
12968
12969 pub async fn import_profiles(&self, profiles: Vec<ModelProfile>) {
12971 let mut tracker = self.outcome_tracker.write().await;
12972 tracker.import_profiles(profiles);
12973 }
12974
12975 pub async fn prepare_speech_runtime(&self) -> Result<PathBuf, InferenceError> {
12998 self.ensure_speech_runtime().await.map(|r| r.root)
13004 }
13005
13006 pub fn set_speech_policy(&mut self, policy: SpeechPolicy) {
13008 self.speech_policy = policy;
13009 }
13010
13011 pub fn set_routing_config(&mut self, config: RoutingConfig) {
13012 self.adaptive_router.set_config(config);
13013 }
13014
13015 pub async fn install_curated_speech(
13017 &mut self,
13018 ) -> Result<Vec<SpeechInstallReport>, InferenceError> {
13019 if let Err(error) = self.prepare_speech_runtime().await {
13032 tracing::warn!(
13033 %error,
13034 "managed speech runtime could not be provisioned; installing the \
13035 models that do not depend on it"
13036 );
13037 }
13038 let schemas = self.list_schemas();
13039 let mut repos = Vec::new();
13040 for schema in &schemas {
13041 if !schema.is_mlx() || !schema.tags.iter().any(|tag| tag == "speech") {
13042 continue;
13043 }
13044 if !cfg!(all(
13059 target_os = "macos",
13060 target_arch = "aarch64",
13061 not(car_skip_mlx)
13062 )) {
13063 continue;
13064 }
13065 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
13066 if !repos.iter().any(|existing: &String| existing == hf_repo) {
13067 repos.push(hf_repo.clone());
13068 }
13069 }
13070 }
13071
13072 let mut installed = Vec::new();
13073 for repo in repos {
13074 let model_for_record = self
13075 .unified_registry()
13076 .all()
13077 .find(|m| matches!(&m.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo))
13078 .map(|m| m.id.clone())
13079 .unwrap_or_else(|| repo.clone());
13080 let _guard = crate::download::acquire_model_lock(&model_for_record).await;
13083 let (snapshot, fetched) =
13084 crate::retire::collecting_fetches(download_hf_repo_snapshot(&repo)).await;
13085 crate::retire::record_download(
13086 &self.config.state_root,
13087 &crate::hf_cache::hub_dir(),
13088 &model_for_record,
13089 &fetched,
13090 );
13091 let (snapshot_path, files_downloaded) = snapshot?;
13092 let name = schemas
13093 .iter()
13094 .find(|schema| {
13095 matches!(&schema.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo)
13096 })
13097 .map(|schema| schema.name.clone())
13098 .unwrap_or_else(|| repo.clone());
13099 installed.push(SpeechInstallReport {
13100 name,
13101 hf_repo: repo,
13102 snapshot_path,
13103 files_downloaded,
13104 });
13105 }
13106
13107 for schema in &schemas {
13111 if !schema.tags.iter().any(|tag| tag == "speech") {
13112 continue;
13113 }
13114 if let ModelSource::WhisperCpp { model } = &schema.source {
13115 let model = model.clone();
13116 let name = schema.name.clone();
13117 let path = tokio::task::spawn_blocking(move || car_whisper::ensure_model(&model))
13118 .await
13119 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))?
13120 .map_err(|e| {
13121 InferenceError::InferenceFailed(format!("whisper model fetch: {e}"))
13122 })?;
13123 installed.push(SpeechInstallReport {
13124 name,
13125 hf_repo: "ggerganov/whisper.cpp".to_string(),
13126 snapshot_path: path,
13127 files_downloaded: 1,
13128 });
13129 }
13130 }
13131
13132 self.unified_registry_mut().refresh_availability();
13133 Ok(installed)
13134 }
13135
13136 pub fn speech_health(&self) -> SpeechHealthReport {
13138 let local_stt_default =
13139 self.speech_health_default_name(ModelCapability::SpeechToText, true, false);
13140 let local_tts_default =
13141 self.speech_health_default_name(ModelCapability::TextToSpeech, true, false);
13142 let remote_stt_default =
13143 self.speech_health_default_name(ModelCapability::SpeechToText, false, true);
13144 let remote_tts_default =
13145 self.speech_health_default_name(ModelCapability::TextToSpeech, false, true);
13146
13147 let mut local_models = Vec::new();
13148 let mut remote_models = Vec::new();
13149 for schema in self.list_schemas() {
13150 let capability = if schema.has_capability(ModelCapability::SpeechToText) {
13151 Some(ModelCapability::SpeechToText)
13152 } else if schema.has_capability(ModelCapability::TextToSpeech) {
13153 Some(ModelCapability::TextToSpeech)
13154 } else {
13155 None
13156 };
13157 let Some(capability) = capability else {
13158 continue;
13159 };
13160
13161 let selected_by_default = local_stt_default
13162 .as_ref()
13163 .is_some_and(|name| name == &schema.name)
13164 || local_tts_default
13165 .as_ref()
13166 .is_some_and(|name| name == &schema.name)
13167 || remote_stt_default
13168 .as_ref()
13169 .is_some_and(|name| name == &schema.name)
13170 || remote_tts_default
13171 .as_ref()
13172 .is_some_and(|name| name == &schema.name);
13173
13174 let health = SpeechModelHealth {
13175 id: schema.id.clone(),
13176 name: schema.name.clone(),
13177 provider: schema.provider.clone(),
13178 capability,
13179 is_local: schema.is_local(),
13180 available: schema.available,
13181 cached: speech_model_cached(&schema),
13182 selected_by_default,
13183 source: speech_model_source_label(&schema),
13184 };
13185 if schema.is_local() {
13186 local_models.push(health);
13187 } else {
13188 remote_models.push(health);
13189 }
13190 }
13191
13192 let runtime = {
13199 let rt =
13200 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
13201 SpeechRuntimeHealth {
13202 root: rt.root.clone(),
13203 installed: rt.is_ready(),
13204 python: rt.python.clone(),
13205 stt_command: rt.stt_program.clone(),
13206 tts_command: rt.tts_program.clone(),
13207 configured_python: std::env::var("CAR_SPEECH_PYTHON")
13208 .ok()
13209 .filter(|value| !value.trim().is_empty()),
13210 detected_python: detect_speech_python(),
13211 }
13212 };
13213
13214 SpeechHealthReport {
13215 runtime,
13216 local_models,
13217 remote_models,
13218 elevenlabs_configured: crate::tasks::transcribe::provider_configured_for_passive_status(
13222 "ELEVENLABS_API_KEY",
13223 )
13224 || crate::tasks::synthesize::provider_configured_for_passive_status(
13225 "ELEVENLABS_API_KEY",
13226 ),
13227 prefer_local: self.speech_policy.prefer_local,
13228 allow_remote_fallback: self.speech_policy.allow_remote_fallback,
13229 preferred_local_stt: self.speech_policy.preferred_local_stt.clone(),
13230 preferred_local_tts: self.speech_policy.preferred_local_tts.clone(),
13231 preferred_remote_stt: self.speech_policy.preferred_remote_stt.clone(),
13232 preferred_remote_tts: self.speech_policy.preferred_remote_tts.clone(),
13233 local_stt_default,
13234 local_tts_default,
13235 remote_stt_default,
13236 remote_tts_default,
13237 }
13238 }
13239
13240 pub async fn model_health(&self) -> ModelHealthReport {
13243 self.model_health_with(false).await
13244 }
13245
13246 pub async fn model_health_for_explicit_status(&self) -> ModelHealthReport {
13256 self.model_health_with(true).await
13257 }
13258
13259 async fn model_health_with(&self, authoritative_credentials: bool) -> ModelHealthReport {
13260 let schemas: Vec<ModelSchema> = self
13261 .catalog_registry_snapshot_with(authoritative_credentials)
13262 .list()
13263 .into_iter()
13264 .cloned()
13265 .collect();
13266 let total_models = schemas.len();
13267 let available_models = schemas
13268 .iter()
13269 .filter(|schema| schema.available_now())
13270 .count();
13271 let local_models = schemas.iter().filter(|schema| schema.is_local()).count();
13272 let remote_models = total_models.saturating_sub(local_models);
13273
13274 let defaults = vec![
13275 self.model_default_health(
13276 ModelCapability::Generate,
13277 self.preferred_model_for_capability(ModelCapability::Generate)
13278 .unwrap_or(&self.config.generation_model),
13279 ),
13280 self.model_default_health(
13281 ModelCapability::Embed,
13282 self.preferred_model_for_capability(ModelCapability::Embed)
13283 .unwrap_or(&self.config.embedding_model),
13284 ),
13285 self.model_default_health(
13286 ModelCapability::Classify,
13287 self.preferred_model_for_capability(ModelCapability::Classify)
13288 .unwrap_or(&self.config.classification_model),
13289 ),
13290 ];
13291
13292 let credential_memo: std::sync::Mutex<std::collections::HashMap<String, bool>> =
13295 std::sync::Mutex::new(std::collections::HashMap::new());
13296 let credential = |env_var: &str| -> bool {
13297 if !authoritative_credentials {
13305 return std::env::var(env_var).is_ok_and(|value| !value.trim().is_empty());
13306 }
13307 let mut memo = credential_memo
13308 .lock()
13309 .unwrap_or_else(|poisoned| poisoned.into_inner());
13310 if let Some(known) = memo.get(env_var) {
13311 return *known;
13312 }
13313 let present = crate::registry::credential_present_without_reading_it(env_var);
13314 memo.insert(env_var.to_string(), present);
13315 present
13316 };
13317
13318 let mut providers = std::collections::BTreeMap::new();
13319 for schema in &schemas {
13320 let entry =
13321 providers
13322 .entry(schema.provider.clone())
13323 .or_insert_with(|| ProviderAccumulator {
13324 configured: false,
13325 local_models: 0,
13326 remote_models: 0,
13327 available_models: 0,
13328 capabilities: std::collections::HashSet::new(),
13329 });
13330
13331 entry.configured |= model_source_configured(schema, &credential);
13332 if schema.is_local() {
13333 entry.local_models += 1;
13334 } else {
13335 entry.remote_models += 1;
13336 }
13337 if schema.available_now() {
13338 entry.available_models += 1;
13339 }
13340 for capability in &schema.capabilities {
13341 entry.capabilities.insert(*capability);
13342 }
13343 }
13344
13345 let providers = providers
13346 .into_iter()
13347 .map(|(provider, acc)| ModelProviderHealth {
13348 provider,
13349 configured: acc.configured,
13350 local_models: acc.local_models,
13351 remote_models: acc.remote_models,
13352 available_models: acc.available_models,
13353 capabilities: sort_capabilities(acc.capabilities.into_iter().collect()),
13354 })
13355 .collect();
13356
13357 let capabilities = all_model_capabilities()
13358 .into_iter()
13359 .map(|capability| {
13360 let relevant: Vec<&ModelSchema> = schemas
13361 .iter()
13362 .filter(|schema| schema.has_capability(capability))
13363 .collect();
13364 let available: Vec<&ModelSchema> = relevant
13365 .iter()
13366 .copied()
13367 .filter(|schema| schema.available_now())
13368 .collect();
13369 ModelCapabilityHealth {
13370 capability,
13371 total_models: relevant.len(),
13372 available_models: available.len(),
13373 local_available_models: available
13374 .iter()
13375 .filter(|schema| schema.is_local())
13376 .count(),
13377 remote_available_models: available
13378 .iter()
13379 .filter(|schema| !schema.is_local())
13380 .count(),
13381 }
13382 })
13383 .collect();
13384
13385 let routing = self.routing_scenarios().await;
13386 let routing_config = self.adaptive_router.config().clone();
13387 let benchmark_priors =
13388 load_benchmark_prior_health(&self.config.state_models_dir(), &schemas);
13389
13390 ModelHealthReport {
13391 total_models,
13392 available_models,
13393 local_models,
13394 remote_models,
13395 defaults,
13396 providers,
13397 capabilities,
13398 routing_prefer_local: routing_config.prefer_local,
13399 routing_quality_first_cold_start: routing_config.quality_first_cold_start,
13400 routing_min_observations: routing_config.min_observations,
13401 routing_bootstrap_min_task_observations: routing_config.bootstrap_min_task_observations,
13402 routing_bootstrap_quality_floor: routing_config.bootstrap_quality_floor,
13403 routing_quality_weight: routing_config.quality_weight,
13404 routing_latency_weight: routing_config.latency_weight,
13405 routing_cost_weight: routing_config.cost_weight,
13406 routing_scenarios: routing,
13407 benchmark_priors,
13408 speech: self.speech_health(),
13409 }
13410 }
13411
13412 async fn routing_scenarios(&self) -> Vec<RoutingScenarioHealth> {
13413 let tracker = self.outcome_tracker.read().await;
13414 let config = self.adaptive_router.config().clone();
13415 let scenarios = [
13416 (
13417 "interactive_text",
13418 "Summarize the benefits of local-first AI routing in two sentences.",
13419 "text",
13420 RoutingWorkload::Interactive,
13421 false,
13422 false,
13423 ),
13424 (
13425 "background_code",
13426 "Write a Python function named fibonacci(n) that returns the nth Fibonacci number.",
13427 "code",
13428 RoutingWorkload::Background,
13429 false,
13430 false,
13431 ),
13432 (
13433 "interactive_tool_use",
13434 "Use the provided weather tool to get the weather for Boston.",
13435 "tool_use",
13436 RoutingWorkload::Interactive,
13437 true,
13438 false,
13439 ),
13440 (
13441 "interactive_vision",
13442 "What is in this image? Answer in one word.",
13443 "vision",
13444 RoutingWorkload::Interactive,
13445 false,
13446 true,
13447 ),
13448 ];
13449
13450 let routing_registry = self.catalog_registry_snapshot();
13457
13458 scenarios
13459 .into_iter()
13460 .map(
13461 |(name, prompt, task_family, workload, has_tools, has_vision)| {
13462 let decision = self.adaptive_router.route_context_aware(
13463 prompt,
13464 0,
13465 &routing_registry,
13466 &tracker,
13467 has_tools,
13468 has_vision,
13469 workload,
13470 );
13471 let quality_first_cold_start = if has_tools || has_vision {
13472 config.quality_first_cold_start
13473 } else if task_family == "code"
13474 && matches!(workload, RoutingWorkload::Background)
13475 {
13476 false
13477 } else {
13478 config.quality_first_cold_start
13479 };
13480 RoutingScenarioHealth {
13481 name: name.to_string(),
13482 task_family: task_family.to_string(),
13483 workload,
13484 has_tools,
13485 has_vision,
13486 prefer_local: if task_family == "speech" {
13487 self.speech_policy.prefer_local
13488 } else {
13489 config.prefer_local
13490 },
13491 quality_first_cold_start,
13492 bootstrap_min_task_observations: config.bootstrap_min_task_observations,
13493 bootstrap_quality_floor: config.bootstrap_quality_floor,
13494 model_id: decision.model_id,
13495 model_name: decision.model_name,
13496 reason: decision.reason,
13497 strategy: decision.strategy,
13498 }
13499 },
13500 )
13501 .collect()
13502 }
13503
13504 pub async fn smoke_test_speech(
13506 &self,
13507 local: bool,
13508 remote: bool,
13509 ) -> Result<SpeechSmokeReport, InferenceError> {
13510 let mut report = SpeechSmokeReport::default();
13511
13512 if local {
13513 let tts = self
13514 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
13515 .ok_or_else(|| {
13516 InferenceError::InferenceFailed(
13517 "no local text-to-speech model available".into(),
13518 )
13519 })?;
13520 let stt = self
13521 .preferred_speech_schema(ModelCapability::SpeechToText, true, false)
13522 .ok_or_else(|| {
13523 InferenceError::InferenceFailed(
13524 "no local speech-to-text model available".into(),
13525 )
13526 })?;
13527 report.local = Some(
13528 self.run_speech_smoke_path("local", &tts, &stt, "Testing CAR local speech path.")
13529 .await?,
13530 );
13531 } else {
13532 report.skipped.push("local".to_string());
13533 }
13534
13535 if remote {
13536 let tts = self
13537 .preferred_speech_schema(ModelCapability::TextToSpeech, false, true)
13538 .ok_or_else(|| {
13539 InferenceError::InferenceFailed(
13540 "no remote text-to-speech model available".into(),
13541 )
13542 })?;
13543 let stt = self
13544 .preferred_speech_schema(ModelCapability::SpeechToText, false, true)
13545 .ok_or_else(|| {
13546 InferenceError::InferenceFailed(
13547 "no remote speech-to-text model available".into(),
13548 )
13549 })?;
13550 report.remote = Some(
13551 self.run_speech_smoke_path("remote", &tts, &stt, "Testing CAR remote speech path.")
13552 .await?,
13553 );
13554 } else {
13555 report.skipped.push("remote".to_string());
13556 }
13557
13558 Ok(report)
13559 }
13560
13561 fn speech_candidates(
13562 &self,
13563 capability: ModelCapability,
13564 explicit: Option<&str>,
13565 ) -> Result<Vec<ModelSchema>, InferenceError> {
13566 if let Some(model) = explicit {
13567 let schema = self
13568 .resolve_schema(model)
13569 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
13570 if !schema.has_capability(capability) {
13571 return Err(InferenceError::InferenceFailed(format!(
13572 "model {} does not support {:?}",
13573 schema.name, capability
13574 )));
13575 }
13576 return Ok(vec![schema]);
13577 }
13578
13579 let mut candidates: Vec<ModelSchema> = self
13580 .unified_registry()
13581 .query(&ModelFilter {
13582 capabilities: vec![capability],
13583 ..Default::default()
13584 })
13585 .into_iter()
13586 .cloned()
13587 .collect();
13588
13589 if candidates.is_empty() {
13590 return Err(InferenceError::InferenceFailed(format!(
13591 "no models registered for capability {:?}",
13592 capability
13593 )));
13594 }
13595
13596 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
13597 if !self.speech_policy.allow_remote_fallback
13598 && candidates.iter().any(|model| model.is_local())
13599 {
13600 candidates.retain(|model| model.is_local());
13601 }
13602
13603 Ok(candidates)
13604 }
13605
13606 #[allow(dead_code)] fn resolve_external_hf_repo(
13612 &self,
13613 explicit: Option<&str>,
13614 capability: ModelCapability,
13615 ) -> Option<String> {
13616 let id = explicit?;
13617 let schema = self.resolve_schema(id)?;
13618 if !schema.has_capability(capability) {
13619 return Some(id.to_string());
13620 }
13621 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
13622 return Some(hf_repo.clone());
13623 }
13624 Some(id.to_string())
13625 }
13626
13627 fn media_generation_candidates(
13628 &self,
13629 capability: ModelCapability,
13630 explicit: Option<&str>,
13631 ) -> Result<Vec<ModelSchema>, InferenceError> {
13632 if let Some(model) = explicit {
13633 let schema = self
13634 .resolve_schema(model)
13635 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
13636 if !schema.has_capability(capability) {
13637 return Err(InferenceError::InferenceFailed(format!(
13638 "model {} does not support {:?}",
13639 schema.name, capability
13640 )));
13641 }
13642 return Ok(vec![schema]);
13643 }
13644
13645 let mut candidates: Vec<ModelSchema> = self
13646 .unified_registry()
13647 .query(&ModelFilter {
13648 capabilities: vec![capability],
13649 local_only: true,
13650 ..Default::default()
13651 })
13652 .into_iter()
13653 .cloned()
13654 .collect();
13655 candidates.sort_by_key(|schema| (!schema.available, schema.size_mb()));
13656 if candidates.is_empty() {
13657 return Err(InferenceError::InferenceFailed(format!(
13658 "no models registered for capability {:?}",
13659 capability
13660 )));
13661 }
13662 Ok(candidates)
13663 }
13664
13665 fn preferred_speech_schema(
13666 &self,
13667 capability: ModelCapability,
13668 local_only: bool,
13669 remote_only: bool,
13670 ) -> Option<ModelSchema> {
13671 let available_only = remote_only;
13672 let mut candidates: Vec<ModelSchema> = self
13673 .unified_registry()
13674 .query(&ModelFilter {
13675 capabilities: vec![capability],
13676 available_only,
13677 ..Default::default()
13678 })
13679 .into_iter()
13680 .filter(|schema| {
13681 (!local_only || schema.is_local()) && (!remote_only || schema.is_remote())
13682 })
13683 .cloned()
13684 .collect();
13685 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
13686 candidates.into_iter().next()
13687 }
13688
13689 fn speech_health_default_name(
13690 &self,
13691 capability: ModelCapability,
13692 local_only: bool,
13693 remote_only: bool,
13694 ) -> Option<String> {
13695 let preferred = match capability {
13696 ModelCapability::SpeechToText if local_only => {
13697 self.speech_policy.preferred_local_stt.as_ref()
13698 }
13699 ModelCapability::SpeechToText if remote_only => {
13700 self.speech_policy.preferred_remote_stt.as_ref()
13701 }
13702 ModelCapability::TextToSpeech if local_only => {
13703 self.speech_policy.preferred_local_tts.as_ref()
13704 }
13705 ModelCapability::TextToSpeech if remote_only => {
13706 self.speech_policy.preferred_remote_tts.as_ref()
13707 }
13708 _ => None,
13709 };
13710
13711 preferred
13712 .filter(|name| {
13713 self.unified_registry().list().iter().any(|schema| {
13714 schema.name == **name
13715 && schema.has_capability(capability)
13716 && (!local_only || schema.is_local())
13717 && (!remote_only || schema.is_remote())
13718 })
13719 })
13720 .cloned()
13721 .or_else(|| {
13722 self.preferred_speech_schema(capability, local_only, remote_only)
13723 .map(|schema| schema.name)
13724 })
13725 }
13726
13727 fn model_default_health(
13728 &self,
13729 capability: ModelCapability,
13730 configured_model: &str,
13731 ) -> ModelDefaultHealth {
13732 let schema = self.resolve_schema_name_first(configured_model);
13733
13734 ModelDefaultHealth {
13735 capability,
13736 configured_model: configured_model.to_string(),
13737 available: schema.as_ref().is_some_and(ModelSchema::available_now),
13738 is_local: schema.as_ref().is_some_and(ModelSchema::is_local),
13739 provider: schema.map(|model| model.provider),
13740 }
13741 }
13742
13743 fn speech_sort_key(
13744 &self,
13745 capability: ModelCapability,
13746 model: &ModelSchema,
13747 ) -> (u8, u8, u8, u8, u64, u64) {
13748 let policy_preference = match capability {
13749 ModelCapability::SpeechToText if model.is_local() => {
13750 self.speech_policy.preferred_local_stt.as_ref()
13751 }
13752 ModelCapability::SpeechToText => self.speech_policy.preferred_remote_stt.as_ref(),
13753 ModelCapability::TextToSpeech if model.is_local() => {
13754 self.speech_policy.preferred_local_tts.as_ref()
13755 }
13756 ModelCapability::TextToSpeech => self.speech_policy.preferred_remote_tts.as_ref(),
13757 _ => None,
13758 };
13759 let local_rank = if self.speech_policy.prefer_local {
13760 if model.is_local() {
13761 0
13762 } else {
13763 1
13764 }
13765 } else if model.is_remote() {
13766 0
13767 } else {
13768 1
13769 };
13770 let availability_rank = if model.available {
13771 0
13772 } else if model.is_local() {
13773 1
13774 } else {
13775 2
13776 };
13777 let policy_rank: u8 = if policy_preference.is_some_and(|preferred| preferred == &model.name)
13778 {
13779 0
13780 } else {
13781 1
13782 };
13783 let speech_rank = match capability {
13784 ModelCapability::TextToSpeech => {
13800 if model.name == "Kokoro-82M-bf16" {
13801 0
13802 } else if model.name == "Kokoro-82M-6bit" {
13803 1
13804 } else if model.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
13805 3
13808 } else {
13809 2
13810 }
13811 }
13812 ModelCapability::SpeechToText => {
13813 if model.name == "Parakeet-TDT-0.6B-v3-MLX" {
13814 0
13815 } else {
13816 1
13817 }
13818 }
13819 _ => 0,
13820 };
13821 let latency_rank = model.performance.latency_p50_ms.unwrap_or(u64::MAX);
13822 let size_rank = model.cost.size_mb.unwrap_or(u64::MAX);
13823 (
13824 local_rank,
13825 availability_rank,
13826 policy_rank,
13827 speech_rank,
13828 latency_rank,
13829 size_rank,
13830 )
13831 }
13832
13833 async fn run_speech_smoke_path(
13834 &self,
13835 path: &str,
13836 tts: &ModelSchema,
13837 stt: &ModelSchema,
13838 text: &str,
13839 ) -> Result<SpeechSmokePathReport, InferenceError> {
13840 let work_dir = temp_work_dir(&format!("speech-smoke-{path}"))?;
13841 let audio_path = work_dir.join(format!("{path}.wav"));
13842 let synth = self
13843 .synthesize(SynthesizeRequest {
13844 text: text.to_string(),
13845 model: Some(tts.name.clone()),
13846 voice: default_speech_voice(tts),
13847 language: Some("en".to_string()),
13848 output_path: Some(audio_path.display().to_string()),
13849 ..SynthesizeRequest::default()
13850 })
13851 .await?;
13852 let transcript = self
13853 .transcribe(TranscribeRequest {
13854 audio_path: synth.audio_path.clone(),
13855 model: Some(stt.name.clone()),
13856 language: Some("en".to_string()),
13857 prompt: None,
13858 timestamps: false,
13859 })
13860 .await?;
13861
13862 Ok(SpeechSmokePathReport {
13863 path: path.to_string(),
13864 tts_model: synth.model_used.unwrap_or_else(|| tts.name.clone()),
13865 stt_model: transcript.model_used.unwrap_or_else(|| stt.name.clone()),
13866 audio_path: PathBuf::from(synth.audio_path),
13867 transcript: transcript.text,
13868 })
13869 }
13870
13871 async fn ensure_speech_runtime(&self) -> Result<SpeechRuntime, InferenceError> {
13872 let mut guard = self.speech_runtime.lock().await;
13873 if let Some(runtime) = guard.as_ref() {
13874 if runtime.is_ready() {
13875 return Ok(runtime.clone());
13876 }
13877 }
13878
13879 let runtime =
13880 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
13881 if !runtime.is_ready() {
13882 bootstrap_speech_runtime(&runtime).await?;
13883 }
13884 if !runtime.is_ready() {
13885 return Err(InferenceError::InferenceFailed(format!(
13886 "managed speech runtime is not ready at {}",
13887 runtime.root.display()
13888 )));
13889 }
13890
13891 *guard = Some(runtime.clone());
13892 Ok(runtime)
13893 }
13894
13895 async fn transcribe_whisper(
13904 &self,
13905 schema: &ModelSchema,
13906 model: &str,
13907 req: &TranscribeRequest,
13908 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13909 ) -> Result<TranscribeResult, InferenceError> {
13910 let model = model.to_string();
13911 let model_path = car_whisper::ensure_model(&model)
13912 .map_err(|e| InferenceError::InferenceFailed(format!("whisper download: {e}")))?;
13913 let measured_bytes = backend_cache::estimate_model_size(&model_path);
13914 let reservation = reservation.ok_or_else(|| {
13915 InferenceError::InferenceFailed("local Whisper path missing admission".into())
13916 })?;
13917 reservation
13918 .reconcile_measured_weights(measured_bytes)
13919 .map_err(InferenceError::from)?;
13920 let detached_lease = reservation.detached_lease();
13921 let language = req.language.clone().unwrap_or_else(|| "auto".to_string());
13923 let audio_path = std::path::PathBuf::from(&req.audio_path);
13924 let name = schema.name.clone();
13925 let req_language = req.language.clone();
13926 let text =
13927 run_admitted_blocking(detached_lease, move || -> Result<String, InferenceError> {
13928 let stt = car_whisper::WhisperStt::load_from_path(&model_path, &language)
13930 .map_err(|e| InferenceError::InferenceFailed(format!("whisper load: {e}")))?;
13931 stt.transcribe_file(&audio_path).map_err(|e| {
13932 InferenceError::InferenceFailed(format!("whisper transcribe: {e}"))
13933 })
13934 })
13935 .await
13936 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))??;
13937 Ok(TranscribeResult::text_only(text, Some(name), req_language))
13938 }
13939
13940 async fn transcribe_local_mlx(
13941 &self,
13942 schema: &ModelSchema,
13943 req: &TranscribeRequest,
13944 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13945 ) -> Result<TranscribeResult, InferenceError> {
13946 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13947 let reservation = reservation.ok_or_else(|| {
13948 InferenceError::InferenceFailed("local STT path missing admission".into())
13949 })?;
13950 reservation
13951 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13952 .map_err(InferenceError::from)?;
13953 self.transcribe_via_speech_runtime(schema, req, reservation.detached_lease())
13962 .await
13963 }
13964
13965 async fn transcribe_via_speech_runtime(
13972 &self,
13973 schema: &ModelSchema,
13974 req: &TranscribeRequest,
13975 detached_lease: resource_policy::DetachedLocalLease,
13976 ) -> Result<TranscribeResult, InferenceError> {
13977 {
13978 let runtime = self.ensure_speech_runtime().await?;
13979 let hf_repo = match &schema.source {
13980 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13981 _ => {
13982 return Err(InferenceError::InferenceFailed(format!(
13983 "speech runtime needs an MLX model repo; {} is not one",
13984 schema.id
13985 )))
13986 }
13987 };
13988 let output_dir = temp_work_dir("stt")?;
13989 let output_prefix = output_dir.join("transcript");
13990 let mut args = vec![
13991 "--model".to_string(),
13992 hf_repo,
13993 "--audio".to_string(),
13994 req.audio_path.clone(),
13995 "--output-path".to_string(),
13996 output_prefix.display().to_string(),
13997 "--format".to_string(),
13998 "json".to_string(),
13999 ];
14000 if let Some(language) = &req.language {
14001 args.push("--language".to_string());
14002 args.push(normalize_lang_code(language));
14003 }
14004 if let Some(prompt) = &req.prompt {
14005 args.push("--context".to_string());
14006 args.push(prompt.clone());
14007 }
14008 if req.timestamps {
14009 args.push("--verbose".to_string());
14010 }
14011
14012 let output =
14013 run_mlx_audio_command(&runtime, "stt.generate", &args, detached_lease).await?;
14014 let text = read_transcription_result(&output_prefix)?
14015 .or_else(|| extract_text_from_payload(&output.stdout))
14016 .ok_or_else(|| {
14017 InferenceError::InferenceFailed(format!(
14018 "mlx-audio transcription returned no text: {}",
14019 output.stderr
14020 ))
14021 })?;
14022
14023 Ok(TranscribeResult {
14024 text,
14025 model_used: Some(schema.name.clone()),
14026 language: req.language.clone(),
14027 words: Vec::new(),
14028 routing_explanation: None,
14029 })
14030 }
14031 }
14032
14033 async fn synthesize_local_mlx(
14034 &self,
14035 schema: &ModelSchema,
14036 req: &SynthesizeRequest,
14037 reservation: Option<&mut resource_policy::LocalLoadReservation>,
14038 ) -> Result<SynthesizeResult, InferenceError> {
14039 let requested = req.requested_advanced_controls();
14044 let repo_supports_advanced = match &schema.source {
14045 ModelSource::Mlx { hf_repo, .. } => hf_repo.to_ascii_lowercase().contains("qwen3-tts"),
14046 _ => false,
14047 };
14048 if !requested.is_empty() && !repo_supports_advanced {
14049 if req.strict_capabilities {
14050 return Err(InferenceError::InferenceFailed(format!(
14051 "model {name} does not support Qwen3-TTS advanced controls {requested:?}; \
14052 route to a Qwen3-TTS model or set strict_capabilities = false to degrade",
14053 name = schema.name,
14054 )));
14055 }
14056 tracing::warn!(
14057 model = %schema.name,
14058 fields = ?requested,
14059 "Qwen3-TTS advanced controls set on non-Qwen3-TTS backend — ignored \
14060 (set strict_capabilities=true to error instead)"
14061 );
14062 }
14063
14064 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
14065 let reservation = reservation.ok_or_else(|| {
14066 InferenceError::InferenceFailed("local TTS path missing admission".into())
14067 })?;
14068 reservation
14069 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
14070 .map_err(InferenceError::from)?;
14071
14072 self.synthesize_via_speech_runtime(schema, req, reservation.detached_lease())
14082 .await
14083 }
14084
14085 async fn synthesize_via_speech_runtime(
14098 &self,
14099 schema: &ModelSchema,
14100 req: &SynthesizeRequest,
14101 detached_lease: resource_policy::DetachedLocalLease,
14102 ) -> Result<SynthesizeResult, InferenceError> {
14103 let runtime = self.ensure_speech_runtime().await?;
14104 let primary_hf_repo = match &schema.source {
14105 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
14106 _ => {
14107 return Err(InferenceError::InferenceFailed(format!(
14108 "speech runtime needs an MLX model repo; {} is not one",
14109 schema.id
14110 )))
14111 }
14112 };
14113 let (produced, model_used) = match self
14114 .synthesize_local_mlx_repo(
14115 &runtime,
14116 &primary_hf_repo,
14117 schema.name.as_str(),
14118 req,
14119 detached_lease.clone(),
14120 )
14121 .await
14122 {
14123 Ok(result) => result,
14124 Err(primary_err)
14125 if primary_hf_repo == "mlx-community/Kokoro-82M-6bit"
14126 && kokoro_runtime_fallback_enabled() =>
14127 {
14128 let fallback_repo = "mlx-community/Kokoro-82M-bf16";
14129 let fallback_name = "Kokoro-82M-bf16";
14130 match self
14131 .synthesize_local_mlx_repo(
14132 &runtime,
14133 fallback_repo,
14134 fallback_name,
14135 req,
14136 detached_lease.clone(),
14137 )
14138 .await
14139 {
14140 Ok(result) => result,
14141 Err(fallback_err) => {
14142 return Err(InferenceError::InferenceFailed(format!(
14143 "{primary_err}; fallback {fallback_name} also failed: {fallback_err}"
14144 )));
14145 }
14146 }
14147 }
14148 Err(err) => return Err(err),
14149 };
14150 let final_path =
14151 materialize_audio_output(&produced, req.output_path.as_deref(), &req.format)?;
14152
14153 Ok(SynthesizeResult {
14154 audio_path: final_path.display().to_string(),
14155 media_type: media_type_for_format(&req.format),
14156 model_used: Some(model_used),
14157 voice_used: req.voice.clone(),
14158 routing_explanation: None,
14159 })
14160 }
14161
14162 async fn synthesize_local_mlx_repo(
14163 &self,
14164 runtime: &SpeechRuntime,
14165 hf_repo: &str,
14166 model_name: &str,
14167 req: &SynthesizeRequest,
14168 detached_lease: resource_policy::DetachedLocalLease,
14169 ) -> Result<(PathBuf, String), InferenceError> {
14170 let output_dir = temp_work_dir("tts")?;
14171 let mut args = vec![
14172 "--model".to_string(),
14173 hf_repo.to_string(),
14174 "--text".to_string(),
14175 req.text.clone(),
14176 "--output_path".to_string(),
14177 output_dir.display().to_string(),
14178 ];
14179 if let Some(voice) = &req.voice {
14180 args.push("--voice".to_string());
14181 args.push(voice.clone());
14182 }
14183 if let Some(speed) = req.speed {
14184 args.push("--speed".to_string());
14185 args.push(speed.to_string());
14186 }
14187 let repo_lower = hf_repo.to_ascii_lowercase();
14188 if repo_lower.contains("kokoro") {
14189 args.push("--lang_code".to_string());
14190 args.push(kokoro_lang_code(req.language.as_deref()).to_string());
14191 } else if let Some(language) = &req.language {
14192 args.push("--lang_code".to_string());
14193 args.push(normalize_lang_code(language));
14194 }
14195
14196 if repo_lower.contains("qwen3-tts") {
14202 if let Some(ref_audio) = &req.reference_audio_path {
14203 args.push("--ref_audio".to_string());
14204 args.push(ref_audio.clone());
14205 }
14206 if let Some(ref_text) = &req.reference_text {
14207 args.push("--ref_text".to_string());
14208 args.push(ref_text.clone());
14209 }
14210 if let Some(instruct) = &req.voice_instruction {
14211 args.push("--instruct".to_string());
14212 args.push(instruct.clone());
14213 }
14214 }
14215
14216 let output = if repo_lower.contains("kokoro") {
14217 let device = std::env::var("CAR_SPEECH_KOKORO_DEVICE")
14218 .or_else(|_| std::env::var("CAR_SPEECH_MLX_DEVICE"))
14219 .unwrap_or_else(|_| "cpu".to_string());
14220 let extra_env = vec![
14221 ("MLX_DEVICE".to_string(), device),
14223 ("PYTORCH_ENABLE_MPS_FALLBACK".to_string(), "1".to_string()),
14225 ];
14226 run_mlx_audio_command_with_env(
14227 runtime,
14228 "tts.generate",
14229 &args,
14230 &extra_env,
14231 detached_lease,
14232 )
14233 .await?
14234 } else {
14235 run_mlx_audio_command(runtime, "tts.generate", &args, detached_lease).await?
14236 };
14237 let produced = find_audio_file(&output_dir)?.ok_or_else(|| {
14238 let hint = if repo_lower.contains("kokoro") {
14239 ". Kokoro models may crash on GPU — try CAR_SPEECH_KOKORO_DEVICE=cpu or use the default Qwen3-TTS model"
14240 } else {
14241 ""
14242 };
14243 InferenceError::InferenceFailed(format!(
14244 "mlx-audio synthesis produced no audio file: {}{}",
14245 output.stderr, hint
14246 ))
14247 })?;
14248 Ok((produced, model_name.to_string()))
14249 }
14250
14251 async fn transcribe_elevenlabs(
14252 &self,
14253 schema: &ModelSchema,
14254 req: &TranscribeRequest,
14255 ) -> Result<TranscribeResult, InferenceError> {
14256 let (endpoint, api_key) = elevenlabs_auth(
14257 schema,
14258 crate::tasks::transcribe::resolve_provider_credential_for_request,
14259 )?;
14260 let file_name = Path::new(&req.audio_path)
14261 .file_name()
14262 .and_then(|f| f.to_str())
14263 .unwrap_or("audio.wav")
14264 .to_string();
14265 let audio_bytes = tokio::fs::read(&req.audio_path).await?;
14266 let form = || {
14269 let file_part = Part::bytes(audio_bytes.clone()).file_name(file_name.clone());
14270 let form = Form::new()
14271 .text("model_id", schema.name.clone())
14272 .part("file", file_part);
14273 match &req.language {
14274 Some(language) => form.text("language_code", language.clone()),
14275 None => form,
14276 }
14277 };
14278 let url = format!("{}/v1/speech-to-text", endpoint.trim_end_matches('/'));
14279
14280 let resp = self
14281 .remote_backend
14282 .send_retrying_connect(
14283 &schema.id,
14284 &url,
14285 "ElevenLabs STT request failed",
14286 None,
14287 &mut 1,
14288 &mut |_| {},
14289 || {
14290 self.remote_backend
14291 .client
14292 .post(&url)
14293 .header("xi-api-key", &api_key)
14294 .multipart(form())
14295 .send()
14296 },
14297 )
14298 .await?;
14299 let status = resp.status();
14300 let body = resp.text().await.map_err(|e| {
14301 InferenceError::InferenceFailed(format!("read ElevenLabs STT body: {e}"))
14302 })?;
14303 if !status.is_success() {
14304 return Err(InferenceError::InferenceFailed(format!(
14305 "ElevenLabs STT returned {status}: {body}"
14306 )));
14307 }
14308 let payload: serde_json::Value = serde_json::from_str(&body).map_err(|e| {
14309 InferenceError::InferenceFailed(format!("parse ElevenLabs STT response: {e}"))
14310 })?;
14311 let text = payload
14312 .get("text")
14313 .and_then(|v| v.as_str())
14314 .map(str::to_string)
14315 .ok_or_else(|| {
14316 InferenceError::InferenceFailed("ElevenLabs STT response missing text".into())
14317 })?;
14318
14319 Ok(TranscribeResult {
14320 text,
14321 model_used: Some(schema.name.clone()),
14322 language: payload
14323 .get("language_code")
14324 .and_then(|v| v.as_str())
14325 .map(str::to_string),
14326 words: Vec::new(),
14327 routing_explanation: None,
14328 })
14329 }
14330
14331 async fn synthesize_elevenlabs(
14332 &self,
14333 schema: &ModelSchema,
14334 req: &SynthesizeRequest,
14335 ) -> Result<SynthesizeResult, InferenceError> {
14336 let requested = req.requested_advanced_controls();
14340 if !requested.is_empty() {
14341 if req.strict_capabilities {
14342 return Err(InferenceError::InferenceFailed(format!(
14343 "ElevenLabs backend does not support Qwen3-TTS advanced controls \
14344 {requested:?}; route to a Qwen3-TTS model or set strict_capabilities = false"
14345 )));
14346 }
14347 tracing::warn!(
14348 model = %schema.name,
14349 fields = ?requested,
14350 "Qwen3-TTS advanced controls ignored by ElevenLabs backend"
14351 );
14352 }
14353 let (endpoint, api_key) = elevenlabs_auth(
14354 schema,
14355 crate::tasks::synthesize::resolve_provider_credential_for_request,
14356 )?;
14357 let voice_id = req
14358 .voice
14359 .clone()
14360 .unwrap_or_else(|| "JBFqnCBsd6RMkjVDRZzb".to_string());
14361 let output_format = elevenlabs_output_format(&req.format);
14362 let url = format!(
14363 "{}/v1/text-to-speech/{}?output_format={}",
14364 endpoint.trim_end_matches('/'),
14365 voice_id,
14366 output_format
14367 );
14368
14369 let mut body = serde_json::json!({
14370 "text": req.text,
14371 "model_id": schema.name,
14372 });
14373 if let Some(language) = &req.language {
14374 body["language_code"] = serde_json::Value::String(language.clone());
14375 }
14376
14377 let resp = self
14378 .remote_backend
14379 .send_retrying_connect(
14380 &schema.id,
14381 &url,
14382 "ElevenLabs TTS request failed",
14383 None,
14384 &mut 1,
14385 &mut |_| {},
14386 || {
14387 self.remote_backend
14388 .client
14389 .post(&url)
14390 .header("xi-api-key", &api_key)
14391 .header("Content-Type", "application/json")
14392 .json(&body)
14393 .send()
14394 },
14395 )
14396 .await?;
14397 let status = resp.status();
14398 let audio = resp.bytes().await.map_err(|e| {
14399 InferenceError::InferenceFailed(format!("read ElevenLabs TTS body: {e}"))
14400 })?;
14401 if !status.is_success() {
14402 let err_body = String::from_utf8_lossy(&audio);
14403 return Err(InferenceError::InferenceFailed(format!(
14404 "ElevenLabs TTS returned {status}: {err_body}"
14405 )));
14406 }
14407
14408 let final_path = requested_or_temp_output(req.output_path.as_deref(), &req.format)?;
14409 ensure_parent_dir(&final_path)?;
14410 tokio::fs::write(&final_path, &audio).await?;
14411
14412 Ok(SynthesizeResult {
14413 audio_path: final_path.display().to_string(),
14414 media_type: media_type_for_format(&req.format),
14415 model_used: Some(schema.name.clone()),
14416 voice_used: Some(voice_id),
14417 routing_explanation: None,
14418 })
14419 }
14420}
14421
14422#[derive(Default)]
14423struct ProviderAccumulator {
14424 configured: bool,
14425 local_models: usize,
14426 remote_models: usize,
14427 available_models: usize,
14428 capabilities: std::collections::HashSet<ModelCapability>,
14429}
14430
14431struct CommandOutput {
14436 stdout: String,
14437 stderr: String,
14438}
14439
14440#[derive(Debug, Clone)]
14441struct SpeechRuntime {
14442 root: PathBuf,
14443 python: PathBuf,
14444 stt_program: PathBuf,
14445 tts_program: PathBuf,
14446}
14447
14448impl SpeechRuntime {
14449 fn new(root: PathBuf) -> Self {
14450 let python = managed_venv::interpreter(&root);
14451 let stt_program = managed_venv::venv_program(&root, "mlx_audio.stt.generate");
14452 let tts_program = managed_venv::venv_program(&root, "mlx_audio.tts.generate");
14453 Self {
14454 root,
14455 python,
14456 stt_program,
14457 tts_program,
14458 }
14459 }
14460
14461 fn is_ready(&self) -> bool {
14462 managed_venv::interpreter_healthy(&self.root)
14467 && self.stt_program.exists()
14468 && self.tts_program.exists()
14469 }
14470
14471 fn command_for(&self, subcommand: &str) -> Result<&Path, InferenceError> {
14472 match subcommand {
14473 "stt.generate" => Ok(&self.stt_program),
14474 "tts.generate" => Ok(&self.tts_program),
14475 _ => Err(InferenceError::InferenceFailed(format!(
14476 "unknown speech subcommand: {subcommand}"
14477 ))),
14478 }
14479 }
14480}
14481
14482async fn run_mlx_audio_command(
14483 runtime: &SpeechRuntime,
14484 subcommand: &str,
14485 args: &[String],
14486 detached_lease: resource_policy::DetachedLocalLease,
14487) -> Result<CommandOutput, InferenceError> {
14488 run_mlx_audio_command_with_env(runtime, subcommand, args, &[], detached_lease).await
14489}
14490
14491async fn run_admitted_blocking<F, T>(
14495 detached_lease: resource_policy::DetachedLocalLease,
14496 operation: F,
14497) -> Result<T, tokio::task::JoinError>
14498where
14499 F: FnOnce() -> T + Send + 'static,
14500 T: Send + 'static,
14501{
14502 tokio::task::spawn_blocking(move || {
14503 let _detached_lease = detached_lease;
14504 operation()
14505 })
14506 .await
14507}
14508
14509struct DetachedSpeechProcess {
14510 child: Option<tokio::process::Child>,
14511 lease: Option<resource_policy::DetachedLocalLease>,
14512}
14513
14514impl Drop for DetachedSpeechProcess {
14515 fn drop(&mut self) {
14516 let Some(mut child) = self.child.take() else {
14517 return;
14518 };
14519 let lease = self.lease.take();
14520 let _ = child.start_kill();
14521 if tokio::runtime::Handle::try_current().is_ok() {
14522 tokio::spawn(async move {
14523 let _ = child.wait().await;
14524 drop(lease);
14525 });
14526 } else {
14527 std::mem::forget((child, lease));
14530 }
14531 }
14532}
14533
14534async fn run_mlx_audio_command_with_env(
14535 runtime: &SpeechRuntime,
14536 subcommand: &str,
14537 args: &[String],
14538 envs: &[(String, String)],
14539 detached_lease: resource_policy::DetachedLocalLease,
14540) -> Result<CommandOutput, InferenceError> {
14541 let program = runtime.command_for(subcommand)?;
14542 let mut command = Command::new(program);
14543 command.args(args);
14544 crate::hf_cache::apply_to(&mut command);
14545 for (key, value) in envs {
14546 command.env(key, value);
14547 }
14548 command
14549 .stdout(std::process::Stdio::piped())
14550 .stderr(std::process::Stdio::piped())
14551 .kill_on_drop(true);
14552 let child = command
14553 .spawn()
14554 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
14555 let mut owned = DetachedSpeechProcess {
14556 child: Some(child),
14557 lease: Some(detached_lease),
14558 };
14559 let mut stdout = owned
14560 .child
14561 .as_mut()
14562 .expect("speech child owned")
14563 .stdout
14564 .take()
14565 .ok_or_else(|| {
14566 InferenceError::InferenceFailed(format!("{} stdout unavailable", program.display()))
14567 })?;
14568 let mut stderr = owned
14569 .child
14570 .as_mut()
14571 .expect("speech child owned")
14572 .stderr
14573 .take()
14574 .ok_or_else(|| {
14575 InferenceError::InferenceFailed(format!("{} stderr unavailable", program.display()))
14576 })?;
14577 let stdout_reader = tokio::spawn(async move {
14578 let mut bytes = Vec::new();
14579 stdout.read_to_end(&mut bytes).await.map(|_| bytes)
14580 });
14581 let stderr_reader = tokio::spawn(async move {
14582 let mut bytes = Vec::new();
14583 stderr.read_to_end(&mut bytes).await.map(|_| bytes)
14584 });
14585 let status = owned
14586 .child
14587 .as_mut()
14588 .expect("speech child owned")
14589 .wait()
14590 .await
14591 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
14592 owned.child.take();
14593 owned.lease.take();
14594 let stdout = stdout_reader
14595 .await
14596 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout join: {err}")))?
14597 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout: {err}")))?;
14598 let stderr = stderr_reader
14599 .await
14600 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr join: {err}")))?
14601 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr: {err}")))?;
14602
14603 if status.success() {
14604 Ok(CommandOutput {
14605 stdout: String::from_utf8_lossy(&stdout).to_string(),
14606 stderr: String::from_utf8_lossy(&stderr).to_string(),
14607 })
14608 } else {
14609 Err(InferenceError::InferenceFailed(format!(
14610 "{} exited with {}: {}",
14611 program.display(),
14612 status,
14613 String::from_utf8_lossy(&stderr)
14614 )))
14615 }
14616}
14617
14618async fn bootstrap_speech_runtime(runtime: &SpeechRuntime) -> Result<(), InferenceError> {
14619 let python = select_speech_python()?;
14620
14621 let outcome = managed_venv::ensure_venv(&runtime.root, &python)
14627 .await
14628 .map_err(|err| InferenceError::InferenceFailed(err.to_string()))?;
14629 if outcome == managed_venv::VenvOutcome::Recreated {
14630 tracing::warn!(
14631 root = %runtime.root.display(),
14632 "managed speech runtime had an unusable interpreter; rebuilt it \
14633 (installed packages were discarded and are being reinstalled)"
14634 );
14635 }
14636
14637 run_command(
14638 "uv",
14639 &[
14640 "pip".to_string(),
14641 "install".to_string(),
14642 "--python".to_string(),
14643 runtime.python.display().to_string(),
14644 speech_runtime_mlx_audio_spec(),
14645 "misaki[en]".to_string(),
14646 speech_runtime_spacy_model_spec(),
14647 ],
14648 )
14649 .await?;
14650
14651 Ok(())
14652}
14653
14654async fn run_command(program: &str, args: &[String]) -> Result<(), InferenceError> {
14655 let output = Command::new(program)
14656 .args(args)
14657 .output()
14658 .await
14659 .map_err(|err| InferenceError::InferenceFailed(format!("{program}: {err}")))?;
14660
14661 if output.status.success() {
14662 Ok(())
14663 } else {
14664 Err(InferenceError::InferenceFailed(format!(
14665 "{} exited with {}: {}",
14666 program,
14667 output.status,
14668 String::from_utf8_lossy(&output.stderr)
14669 )))
14670 }
14671}
14672
14673fn select_speech_python() -> Result<String, InferenceError> {
14674 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14675 if !path.trim().is_empty() {
14676 return Ok(path);
14677 }
14678 }
14679
14680 for candidate in ["python3.13", "python3.12", "python3.11"] {
14681 if command_in_path(candidate) {
14682 return Ok(candidate.to_string());
14683 }
14684 }
14685
14686 Ok(SPEECH_RUNTIME_FALLBACK_PYTHON.to_string())
14696}
14697
14698const SPEECH_RUNTIME_FALLBACK_PYTHON: &str = "3.12";
14704
14705fn detect_speech_python() -> Option<String> {
14706 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14707 if !path.trim().is_empty() {
14708 return Some(path);
14709 }
14710 }
14711
14712 ["python3.13", "python3.12", "python3.11"]
14713 .into_iter()
14714 .find(|candidate| command_in_path(candidate))
14715 .map(str::to_string)
14716}
14717
14718fn speech_runtime_root_from_models_dir(_models_dir: &Path) -> PathBuf {
14719 if let Ok(path) = std::env::var("CAR_SPEECH_RUNTIME_DIR") {
14720 if !path.trim().is_empty() {
14721 return PathBuf::from(path);
14722 }
14723 }
14724
14725 std::env::var_os("HOME")
14726 .or_else(|| std::env::var_os("USERPROFILE"))
14727 .map(PathBuf::from)
14728 .unwrap_or_else(|| PathBuf::from("."))
14729 .join(".car")
14730 .join("speech-runtime")
14731}
14732
14733fn command_in_path(name: &str) -> bool {
14734 std::env::var_os("PATH")
14735 .map(|paths| {
14736 std::env::split_paths(&paths).any(|dir| {
14737 let path = dir.join(name);
14738 path.exists() && path.is_file()
14739 })
14740 })
14741 .unwrap_or(false)
14742}
14743
14744fn speech_model_cached(schema: &ModelSchema) -> bool {
14745 match &schema.source {
14746 ModelSource::Mlx { hf_repo, .. } => huggingface_repo_has_snapshot(hf_repo),
14747 ModelSource::WhisperCpp { model } => car_whisper::model_cached(model),
14748 ModelSource::WindowsSpeech {} => cfg!(target_os = "windows"),
14750 ModelSource::Proprietary { .. } => false,
14753 _ => false,
14754 }
14755}
14756
14757fn model_source_configured(schema: &ModelSchema, credential: &dyn Fn(&str) -> bool) -> bool {
14777 match &schema.source {
14778 ModelSource::RemoteApi {
14779 protocol: ApiProtocol::OpenRouter,
14780 ..
14781 } => crate::openrouter::credential_source().is_some(),
14782 ModelSource::RemoteApi {
14783 api_key_env,
14784 api_key_envs,
14785 ..
14786 } => credential(api_key_env) || api_key_envs.iter().any(|env_var| credential(env_var)),
14787 ModelSource::Proprietary { auth, .. } => match auth {
14788 ProprietaryAuth::ApiKeyEnv { env_var } => credential(env_var),
14789 ProprietaryAuth::BearerTokenEnv { env_var } => credential(env_var),
14790 ProprietaryAuth::OAuth2Pkce { .. } => matches!(
14791 car_auth::credential_authority_hint().state,
14792 car_auth::CredentialAuthorityState::Configured
14793 ),
14794 ProprietaryAuth::ChatGptSubscription {} => {
14795 credential(car_auth::CODEX_AUTH_RECORD_KEY)
14796 || crate::registry::passive_codex_oauth_available()
14797 }
14798 },
14799 ModelSource::VllmMlx { .. } => {
14800 std::env::var("VLLM_MLX_ENDPOINT").is_ok() || schema.available
14801 }
14802 ModelSource::Ollama { .. } => schema.available,
14803 ModelSource::CodexCli { .. } => crate::backend::codex_cli::is_available(),
14804 ModelSource::Mlx { .. }
14805 | ModelSource::ManagedVllmMlx { .. }
14806 | ModelSource::Local { .. }
14807 | ModelSource::WhisperCpp { .. } => true,
14808 ModelSource::WindowsSpeech {} => schema.available,
14810 ModelSource::AppleFoundationModels { .. } => schema.available,
14811 ModelSource::Delegated { .. } => true,
14816 }
14817}
14818
14819fn scoreboard_price_model(schema: &ModelSchema) -> Option<crate::scoreboard::PriceModel> {
14820 if schema.is_subscription_billed() {
14821 return None;
14822 }
14823 match (schema.cost.input_per_mtok, schema.cost.output_per_mtok) {
14824 (Some(input_per_mtok), Some(output_per_mtok)) => Some(crate::scoreboard::PriceModel {
14828 input_per_mtok,
14829 output_per_mtok,
14830 cache: schema.cache_rates(),
14831 is_estimate: !schema.cost.pricing_tiers.is_empty(),
14832 }),
14833 _ => None,
14834 }
14835}
14836
14837fn all_model_capabilities() -> [ModelCapability; 13] {
14838 [
14839 ModelCapability::Generate,
14840 ModelCapability::Embed,
14841 ModelCapability::Classify,
14842 ModelCapability::Code,
14843 ModelCapability::Reasoning,
14844 ModelCapability::Summarize,
14845 ModelCapability::ToolUse,
14846 ModelCapability::MultiToolCall,
14847 ModelCapability::Vision,
14848 ModelCapability::SpeechToText,
14849 ModelCapability::TextToSpeech,
14850 ModelCapability::ImageGeneration,
14851 ModelCapability::VideoGeneration,
14852 ]
14853}
14854
14855fn sort_capabilities(mut capabilities: Vec<ModelCapability>) -> Vec<ModelCapability> {
14856 capabilities.sort_by_key(|capability| {
14857 all_model_capabilities()
14858 .iter()
14859 .position(|candidate| candidate == capability)
14860 .unwrap_or(usize::MAX)
14861 });
14862 capabilities
14863}
14864
14865fn speech_model_source_label(schema: &ModelSchema) -> String {
14866 match &schema.source {
14867 ModelSource::Mlx { hf_repo, .. } => format!("mlx:{hf_repo}"),
14868 ModelSource::ManagedVllmMlx { hf_repo, .. } => {
14869 format!("managed-vllm-mlx:{hf_repo}")
14870 }
14871 ModelSource::WhisperCpp { model } => format!("whisper:{model}"),
14872 ModelSource::WindowsSpeech {} => "windows-speech".to_string(),
14873 ModelSource::Proprietary {
14874 provider, endpoint, ..
14875 } => format!("proprietary:{provider}:{endpoint}"),
14876 ModelSource::RemoteApi { endpoint, .. } => format!("remote:{endpoint}"),
14877 ModelSource::CodexCli { model } => format!("codex-cli:{model}"),
14878 ModelSource::Local { hf_repo, .. } => format!("local:{hf_repo}"),
14879 ModelSource::VllmMlx {
14880 endpoint,
14881 model_name,
14882 } => format!("vllm-mlx:{endpoint}:{model_name}"),
14883 ModelSource::Ollama { model_tag, host } => format!("ollama:{host}:{model_tag}"),
14884 ModelSource::AppleFoundationModels { use_case } => {
14885 format!(
14886 "apple-foundation:{}",
14887 use_case.as_deref().unwrap_or("default")
14888 )
14889 }
14890 ModelSource::Delegated { hint } => {
14891 format!("delegated:{}", hint.as_deref().unwrap_or("(none)"))
14892 }
14893 }
14894}
14895
14896fn rerank_prompt(instruction: &str, query: &str, document: &str) -> String {
14904 const SYSTEM: &str = "Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".";
14905 format!(
14906 "<|im_start|>system\n{SYSTEM}<|im_end|>\n\
14907 <|im_start|>user\n<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {document}<|im_end|>\n\
14908 <|im_start|>assistant\n<think>\n\n</think>\n\n"
14909 )
14910}
14911
14912fn parse_api_returned_status(err: &str) -> Option<u16> {
14936 let lower = err.to_ascii_lowercase();
14937 let idx = lower.find("api returned ")?;
14938 let digits: String = lower[idx + "api returned ".len()..]
14939 .chars()
14940 .take_while(|c| c.is_ascii_digit())
14941 .take(3)
14942 .collect();
14943 digits.parse().ok()
14944}
14945
14946fn parse_org_lookup_status(err: &str) -> Option<u16> {
14950 const ANCHOR: &str = "org lookup failed: http ";
14951 let lower = err.to_ascii_lowercase();
14952 let idx = lower.find(ANCHOR)?;
14953 let digits: String = lower[idx + ANCHOR.len()..]
14954 .chars()
14955 .take_while(|c| c.is_ascii_digit())
14956 .take(3)
14957 .collect();
14958 digits.parse().ok()
14959}
14960
14961pub const NO_BACKEND_RECOVERY_MARKER: &str = "no inference backend is available.";
14974
14975const AUTO_GRACE_FILE: &str = "concierge-auto-since";
14978const AUTO_GRACE_SECS: u64 = 7 * 24 * 60 * 60;
14980
14981#[doc(hidden)]
14985pub const LOCAL_TOOL_MODEL_PULL_ID: &str =
14986 if cfg!(all(
14989 target_os = "macos",
14990 target_arch = "aarch64",
14991 not(car_skip_mlx)
14992 )) {
14993 "mlx/qwen3-4b:4bit"
14994 } else {
14995 "qwen/qwen3-4b:q4_k_m"
14996 };
14997
14998#[doc(hidden)]
15005pub fn no_backend_recovery_hint(underlying: &str) -> Option<String> {
15006 let is_no_backend = underlying.contains("no credential")
15007 || underlying.contains("model not found")
15008 || underlying.contains("no models available")
15009 || underlying.contains("no inference runner");
15010 if !is_no_backend {
15011 return None;
15012 }
15013 Some(format!(
15019 "{NO_BACKEND_RECOVERY_MARKER} To use Parslee's hosted models, run:\n \
15020 car auth login\n\
15021 Or, for a model you name yourself with --model, install a local \
15022 tool-capable one:\n \
15023 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
15024 (underlying error: {underlying})"
15025 ))
15026}
15027
15028pub fn is_auth_rejection_message(underlying: &str) -> bool {
15059 if let Some(status) = parse_api_returned_status(underlying) {
15060 return status == 401;
15061 }
15062 let l = underlying.to_ascii_lowercase();
15063 if l.contains("org lookup failed") {
15064 return parse_org_lookup_status(underlying) == Some(401);
15065 }
15066 l.contains("authentication required")
15067 || (l.contains("401") && l.contains("unauthorized"))
15068 || l.contains("invalid_grant")
15069 || l.contains("token expired")
15070}
15071
15072pub fn is_auth_failure_message(message: &str) -> bool {
15079 let lower = message.to_ascii_lowercase();
15080 AUTH_FAILURE_MESSAGE_MARKERS
15081 .iter()
15082 .any(|marker| lower.contains(marker))
15083 || is_auth_rejection_message(message)
15084}
15085
15086fn record_auth_dead_lane(slot: &mut Option<String>, candidate: &str, error: &str) {
15095 if slot.is_none() && is_auth_rejection_message(error) {
15096 *slot = Some(candidate.to_string());
15097 }
15098}
15099
15100fn is_local_last_resort(appended_id: Option<&str>, candidate_id: &str) -> bool {
15105 appended_id == Some(candidate_id)
15106}
15107
15108fn record_generation_span(
15124 model: &str,
15125 latency_ms: u64,
15126 ttft_ms: Option<u64>,
15127 usage: Option<&TokenUsage>,
15128) {
15129 let span = tracing::Span::current();
15130 span.record("model", model);
15131 span.record("gen_ai.response.model", model);
15132 span.record("latency_ms", latency_ms);
15133 if let Some(ttft) = ttft_ms {
15134 span.record("ttft_ms", ttft);
15135 }
15136 if let Some(u) = usage {
15137 span.record("prompt_tokens", u.prompt_tokens);
15138 span.record("completion_tokens", u.completion_tokens);
15139 span.record("gen_ai.usage.input_tokens", u.prompt_tokens);
15140 span.record("gen_ai.usage.output_tokens", u.completion_tokens);
15141 }
15142}
15143
15144fn lead_with_last_resort_substitution(
15151 rx: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
15152 served_id: &str,
15153 is_last_resort: bool,
15154 hops: &[FallbackFrom],
15155) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
15156 if !report_local_last_resort_served(is_last_resort.then_some(served_id), served_id, served_id) {
15157 return rx;
15158 }
15159 let substitution = stream::StreamSubstitution {
15160 model_id: served_id.to_string(),
15161 substituted_from: hops.first().map(|hop| hop.candidate.clone()),
15162 substitution_reason: hops.first().map(|hop| hop.reason),
15163 };
15164 let (tx, out) = tokio::sync::mpsc::channel(64);
15165 let mut rx = rx;
15166 tokio::spawn(async move {
15167 if tx
15168 .send(stream::StreamEvent::Substituted(substitution))
15169 .await
15170 .is_err()
15171 {
15172 return;
15173 }
15174 while let Some(event) = rx.recv().await {
15175 if tx.send(event).await.is_err() {
15176 break;
15177 }
15178 }
15179 });
15180 out
15181}
15182
15183fn report_local_last_resort_served(
15184 appended_id: Option<&str>,
15185 candidate_id: &str,
15186 resolved_id: &str,
15187) -> bool {
15188 let served = is_local_last_resort(appended_id, candidate_id);
15189 if served {
15190 tracing::warn!(
15191 local_model = %resolved_id,
15192 "on-device last-resort model served inference turn"
15193 );
15194 }
15195 served
15196}
15197
15198fn auth_expired_recovery_hint(underlying: &str) -> Option<String> {
15209 if !is_auth_rejection_message(underlying) {
15210 return None;
15211 }
15212 Some(format!(
15213 "your Parslee session has expired or was rejected — re-authenticate:\n \
15214 car auth login\n\
15215 Or, for a model you name yourself with --model, install a local \
15216 tool-capable one:\n \
15217 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
15218 (underlying error: {underlying})"
15219 ))
15220}
15221
15222fn score_from_rerank_output(text: &str, model_name: &str) -> f32 {
15223 let normalized: String = text
15228 .to_ascii_lowercase()
15229 .chars()
15230 .map(|c| if c.is_ascii_alphanumeric() { c } else { ' ' })
15231 .collect();
15232 for tok in normalized.split_ascii_whitespace().take(5) {
15233 match tok {
15234 "yes" => return 1.0,
15235 "no" => return 0.0,
15236 _ => continue,
15237 }
15238 }
15239 tracing::warn!(
15240 model = %model_name,
15241 output = %text,
15242 "rerank: first tokens contain neither `yes` nor `no`; returning neutral 0.5"
15243 );
15244 0.5
15245}
15246
15247fn default_speech_voice(schema: &ModelSchema) -> Option<String> {
15248 if schema.provider == "elevenlabs" {
15249 Some("JBFqnCBsd6RMkjVDRZzb".to_string())
15250 } else if schema.name == "Kokoro-82M-6bit" || schema.name == "Kokoro-82M-bf16" {
15251 Some("af_heart".to_string())
15252 } else if schema.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
15253 Some("Chelsie".to_string())
15254 } else {
15255 None
15256 }
15257}
15258
15259#[allow(dead_code)] fn huggingface_repo_has_snapshot(repo_id: &str) -> bool {
15261 crate::registry::latest_huggingface_repo_snapshot(repo_id).is_some()
15262}
15263
15264#[allow(dead_code)] fn count_files_recursive(root: &Path) -> usize {
15266 let Ok(entries) = std::fs::read_dir(root) else {
15267 return 0;
15268 };
15269 entries
15270 .filter_map(Result::ok)
15271 .map(|entry| entry.path())
15272 .map(|path| {
15273 if path.is_dir() {
15274 count_files_recursive(&path)
15275 } else if path.is_file() {
15276 1
15277 } else {
15278 0
15279 }
15280 })
15281 .sum()
15282}
15283
15284async fn download_hf_repo_snapshot(repo_id: &str) -> Result<(PathBuf, usize), InferenceError> {
15285 let api = crate::hf_cache::api()
15286 .with_progress(false)
15287 .build()
15288 .map_err(|e| InferenceError::DownloadFailed(format!("init hf api: {e}")))?;
15289 let repo = api.model(repo_id.to_string());
15290 let info = repo
15291 .info()
15292 .await
15293 .map_err(|e| InferenceError::DownloadFailed(format!("{repo_id}: {e}")))?;
15294
15295 let snapshot_path = crate::hf_cache::repo_dir(repo_id)
15296 .join("snapshots")
15297 .join(&info.sha);
15298 let mut downloaded = 0usize;
15299 for sibling in &info.siblings {
15300 let local_path = snapshot_path.join(&sibling.rfilename);
15301 if crate::download::cache_file_usable(&local_path) {
15309 downloaded += 1;
15310 continue;
15311 }
15312 let _ = std::fs::remove_file(&local_path);
15317 repo.download(&sibling.rfilename).await.map_err(|e| {
15318 InferenceError::DownloadFailed(format!("{repo_id}/{}: {e}", sibling.rfilename))
15319 })?;
15320 crate::retire::note_fetched(repo_id, &sibling.rfilename, Some(info.sha.clone()));
15321 downloaded += 1;
15322 }
15323
15324 Ok((snapshot_path, downloaded))
15325}
15326
15327fn temp_work_dir(prefix: &str) -> Result<PathBuf, InferenceError> {
15328 let unique = SystemTime::now()
15329 .duration_since(UNIX_EPOCH)
15330 .map_err(|e| InferenceError::InferenceFailed(format!("clock error: {e}")))?
15331 .as_nanos();
15332 let dir = std::env::temp_dir().join(format!("car-inference-{prefix}-{unique}"));
15333 std::fs::create_dir_all(&dir)?;
15334 Ok(dir)
15335}
15336
15337fn ensure_parent_dir(path: &Path) -> Result<(), InferenceError> {
15338 if let Some(parent) = path.parent() {
15339 std::fs::create_dir_all(parent)?;
15340 }
15341 Ok(())
15342}
15343
15344fn requested_or_temp_output(
15345 output_path: Option<&str>,
15346 format: &str,
15347) -> Result<PathBuf, InferenceError> {
15348 if let Some(path) = output_path {
15349 return Ok(PathBuf::from(path));
15350 }
15351 let dir = temp_work_dir("audio-out")?;
15352 Ok(dir.join(format!("speech.{format}")))
15353}
15354
15355#[allow(dead_code)] fn requested_or_temp_media_output(
15357 output_path: Option<&str>,
15358 format: &str,
15359 stem: &str,
15360) -> Result<PathBuf, InferenceError> {
15361 if let Some(path) = output_path {
15362 return Ok(PathBuf::from(path));
15363 }
15364 let dir = temp_work_dir(&format!("{stem}-out"))?;
15365 Ok(dir.join(format!("{stem}.{format}")))
15366}
15367
15368fn materialize_audio_output(
15369 produced: &Path,
15370 requested: Option<&str>,
15371 format: &str,
15372) -> Result<PathBuf, InferenceError> {
15373 if let Some(path) = requested {
15374 let dest = PathBuf::from(path);
15375 ensure_parent_dir(&dest)?;
15376 std::fs::copy(produced, &dest)?;
15377 Ok(dest)
15378 } else {
15379 let dest = requested_or_temp_output(None, format)?;
15380 ensure_parent_dir(&dest)?;
15381 std::fs::copy(produced, &dest)?;
15382 Ok(dest)
15383 }
15384}
15385
15386#[cfg(target_os = "windows")]
15392fn winrt_synthesize_wav(text: &str, voice: &str, rate: f64) -> Result<Vec<u8>, InferenceError> {
15393 use windows::core::HSTRING;
15394 use windows::Media::SpeechSynthesis::SpeechSynthesizer;
15395 use windows::Storage::Streams::DataReader;
15396
15397 let err = |m: String| InferenceError::InferenceFailed(m);
15398 let synth =
15399 SpeechSynthesizer::new().map_err(|e| err(format!("SpeechSynthesizer::new: {e}")))?;
15400 if let Ok(opts) = synth.Options() {
15401 let _ = opts.SetSpeakingRate(rate.clamp(0.5, 6.0));
15402 }
15403 if !voice.is_empty() {
15404 if let Ok(all) = SpeechSynthesizer::AllVoices() {
15405 let want = voice.to_lowercase();
15406 let count = all.Size().unwrap_or(0);
15407 for i in 0..count {
15408 if let Ok(info) = all.GetAt(i) {
15409 if let Ok(name) = info.DisplayName() {
15410 if name.to_string_lossy().to_lowercase().contains(&want) {
15411 let _ = synth.SetVoice(&info);
15412 break;
15413 }
15414 }
15415 }
15416 }
15417 }
15418 }
15419 let stream = synth
15420 .SynthesizeTextToStreamAsync(&HSTRING::from(text))
15421 .map_err(|e| err(format!("SynthesizeTextToStreamAsync: {e}")))?
15422 .get()
15423 .map_err(|e| err(format!("synthesize await: {e}")))?;
15424 let size = stream
15425 .Size()
15426 .map_err(|e| err(format!("stream size: {e}")))?;
15427 let input = stream
15428 .GetInputStreamAt(0)
15429 .map_err(|e| err(format!("input stream: {e}")))?;
15430 let reader =
15431 DataReader::CreateDataReader(&input).map_err(|e| err(format!("data reader: {e}")))?;
15432 reader
15433 .LoadAsync(size as u32)
15434 .map_err(|e| err(format!("load async: {e}")))?
15435 .get()
15436 .map_err(|e| err(format!("load await: {e}")))?;
15437 let mut buf = vec![0u8; size as usize];
15438 reader
15439 .ReadBytes(&mut buf)
15440 .map_err(|e| err(format!("read bytes: {e}")))?;
15441 Ok(buf)
15442}
15443
15444#[allow(dead_code)] fn read_transcription_result(output_prefix: &Path) -> Result<Option<String>, InferenceError> {
15446 let candidates = [
15447 output_prefix.with_extension("json"),
15448 output_prefix.to_path_buf(),
15449 ];
15450
15451 for path in candidates {
15452 if path.exists() {
15453 let contents = std::fs::read_to_string(path)?;
15454 if let Some(text) = extract_text_from_payload(&contents) {
15455 return Ok(Some(text));
15456 }
15457 }
15458 }
15459
15460 Ok(None)
15461}
15462
15463#[allow(dead_code)] fn extract_text_from_payload(payload: &str) -> Option<String> {
15465 let value: serde_json::Value = serde_json::from_str(payload).ok()?;
15466 if let Some(text) = value.get("text").and_then(|v| v.as_str()) {
15467 return Some(text.to_string());
15468 }
15469 if let Some(transcripts) = value.get("transcripts").and_then(|v| v.as_array()) {
15470 let joined = transcripts
15471 .iter()
15472 .filter_map(|item| item.get("text").and_then(|v| v.as_str()))
15473 .collect::<Vec<_>>()
15474 .join("\n");
15475 if !joined.is_empty() {
15476 return Some(joined);
15477 }
15478 }
15479 if let Some(items) = value.as_array() {
15480 let joined = items
15481 .iter()
15482 .filter_map(|item| {
15483 item.get("text")
15484 .or_else(|| item.get("Content"))
15485 .and_then(|v| v.as_str())
15486 })
15487 .collect::<Vec<_>>()
15488 .join(" ");
15489 if !joined.is_empty() {
15490 return Some(joined);
15491 }
15492 }
15493 None
15494}
15495
15496#[allow(dead_code)] fn find_audio_file(output_dir: &Path) -> Result<Option<PathBuf>, InferenceError> {
15498 let mut audio_files = Vec::new();
15499 collect_audio_files(output_dir, &mut audio_files)?;
15500 audio_files.sort();
15501 Ok(audio_files.into_iter().next())
15502}
15503
15504#[allow(dead_code)] fn collect_audio_files(dir: &Path, audio_files: &mut Vec<PathBuf>) -> Result<(), InferenceError> {
15506 for entry in std::fs::read_dir(dir)? {
15507 let path = entry?.path();
15508 if path.is_dir() {
15509 collect_audio_files(&path, audio_files)?;
15510 } else if matches!(
15511 path.extension().and_then(|ext| ext.to_str()),
15512 Some("wav" | "mp3" | "flac" | "pcm" | "m4a")
15513 ) {
15514 audio_files.push(path);
15515 }
15516 }
15517 Ok(())
15518}
15519
15520fn media_type_for_format(format: &str) -> String {
15521 match format.to_ascii_lowercase().as_str() {
15522 "mp3" => "audio/mpeg".to_string(),
15523 "flac" => "audio/flac".to_string(),
15524 "pcm" => "audio/L16".to_string(),
15525 "m4a" => "audio/mp4".to_string(),
15526 _ => "audio/wav".to_string(),
15527 }
15528}
15529
15530fn kokoro_lang_code(language: Option<&str>) -> &'static str {
15531 match language.unwrap_or("en").to_ascii_lowercase().as_str() {
15532 "en-gb" | "british" | "british english" => "b",
15533 "ja" | "japanese" => "j",
15534 "zh" | "zh-cn" | "mandarin" | "chinese" => "z",
15535 "es" | "spanish" => "e",
15536 "fr" | "french" => "f",
15537 _ => "a",
15538 }
15539}
15540
15541#[allow(dead_code)] fn normalize_lang_code(language: &str) -> String {
15543 match language.to_ascii_lowercase().as_str() {
15544 "english" | "en-us" | "en_us" => "en".to_string(),
15545 "spanish" => "es".to_string(),
15546 "french" => "fr".to_string(),
15547 "japanese" => "ja".to_string(),
15548 "chinese" | "mandarin" => "zh".to_string(),
15549 other => match other {
15550 "en" | "es" | "fr" | "ja" | "zh" => other.to_string(),
15551 _ => "en".to_string(),
15552 },
15553 }
15554}
15555
15556fn elevenlabs_auth(
15557 schema: &ModelSchema,
15558 resolve_credential: fn(&str) -> Option<String>,
15559) -> Result<(String, String), InferenceError> {
15560 match &schema.source {
15561 ModelSource::Proprietary {
15562 endpoint,
15563 auth: schema::ProprietaryAuth::ApiKeyEnv { env_var },
15564 ..
15565 } => {
15566 let key = resolve_credential(env_var).ok_or_else(|| {
15567 InferenceError::InferenceFailed(format!(
15568 "missing API key {env_var}; set the environment variable or \
15569 store it with `car secrets put {env_var}`"
15570 ))
15571 })?;
15572 Ok((endpoint.clone(), key))
15573 }
15574 _ => Err(InferenceError::InferenceFailed(format!(
15575 "model {} is not an ElevenLabs proprietary model",
15576 schema.id
15577 ))),
15578 }
15579}
15580
15581fn elevenlabs_output_format(format: &str) -> &'static str {
15582 match format.to_ascii_lowercase().as_str() {
15583 "mp3" => "mp3_44100_128",
15584 "pcm" => "pcm_16000",
15585 _ => "wav_44100",
15586 }
15587}
15588
15589fn quality_seeds(
15593 state_models_dir: &Path,
15594) -> std::collections::HashMap<String, outcome::QualitySeed> {
15595 let mut seeds = std::collections::HashMap::new();
15596 for path in benchmark_priors_paths(state_models_dir) {
15597 let Ok(priors) = routing_ext::load_benchmark_priors(&path) else {
15598 continue;
15599 };
15600 for (model_id, prior) in priors {
15601 seeds.insert(
15602 model_id,
15603 outcome::QualitySeed {
15604 overall: Some(prior.overall_score),
15605 tasks: prior.task_scores,
15606 },
15607 );
15608 }
15609 }
15610 seeds
15611}
15612
15613fn benchmark_priors_paths(state_models_dir: &Path) -> Vec<PathBuf> {
15618 let mut paths = Vec::new();
15619
15620 let direct = state_models_dir.join("benchmark_priors.json");
15621 if !paths.contains(&direct) {
15622 paths.push(direct);
15623 }
15624
15625 if let Some(parent) = state_models_dir.parent() {
15626 let parent_path = parent.join("benchmark_priors.json");
15627 if !paths.contains(&parent_path) {
15628 paths.push(parent_path);
15629 }
15630 }
15631
15632 if let Some(path) = std::env::var_os("CAR_BENCHMARK_PRIORS_PATH") {
15633 let path = PathBuf::from(path);
15634 if !paths.contains(&path) {
15635 paths.push(path);
15636 }
15637 }
15638
15639 paths
15640}
15641
15642fn load_benchmark_prior_health(
15643 state_models_dir: &Path,
15644 schemas: &[ModelSchema],
15645) -> Vec<ModelBenchmarkPriorHealth> {
15646 let mut priors = std::collections::BTreeMap::new();
15647 for path in benchmark_priors_paths(state_models_dir) {
15648 let Ok(loaded) = routing_ext::load_benchmark_priors(&path) else {
15649 continue;
15650 };
15651 for (model_id, prior) in loaded {
15652 let model_name = schemas
15653 .iter()
15654 .find(|schema| schema.id == model_id)
15655 .map(|schema| schema.name.clone());
15656 priors.insert(
15657 model_id.clone(),
15658 ModelBenchmarkPriorHealth {
15659 model_id,
15660 model_name,
15661 overall_score: prior.overall_score,
15662 overall_latency_ms: prior.overall_latency_ms,
15663 task_scores: prior.task_scores,
15664 task_latency_ms: prior.task_latency_ms,
15665 source_path: path.clone(),
15666 },
15667 );
15668 }
15669 }
15670
15671 priors.into_values().collect()
15672}
15673
15674fn kokoro_runtime_fallback_enabled() -> bool {
15675 std::env::var("CAR_SPEECH_KOKORO_FALLBACK")
15676 .ok()
15677 .map(|value| {
15678 !matches!(
15679 value.trim().to_ascii_lowercase().as_str(),
15680 "0" | "false" | "off"
15681 )
15682 })
15683 .unwrap_or(true)
15684}
15685
15686fn speech_runtime_mlx_audio_spec() -> String {
15687 std::env::var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC")
15688 .ok()
15689 .filter(|value| !value.trim().is_empty())
15690 .unwrap_or_else(|| "mlx-audio==0.4.2".to_string())
15691}
15692
15693fn speech_runtime_spacy_model_spec() -> String {
15694 std::env::var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC")
15695 .ok()
15696 .filter(|value| !value.trim().is_empty())
15697 .unwrap_or_else(|| {
15698 "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl".to_string()
15699 })
15700}
15701
15702#[cfg(test)]
15703pub(crate) fn run_in_isolated_test_process(test_name: &str, sentinel: &str) -> bool {
15704 static CHILD_PROCESS_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
15705
15706 if std::env::var_os(sentinel).is_some() {
15707 return true;
15708 }
15709
15710 let _child_process = CHILD_PROCESS_MUTEX
15711 .lock()
15712 .unwrap_or_else(std::sync::PoisonError::into_inner);
15713 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
15714 command
15715 .arg("--exact")
15716 .arg(test_name)
15717 .arg("--nocapture")
15718 .arg("--test-threads=1")
15719 .env(sentinel, "1");
15720 for name in [
15721 car_home::ENV_VAR,
15722 "CAR_SECRETS_FILE_DIR",
15723 "CAR_TEST_NATIVE_KEYCHAIN",
15724 "CAR_AUTH_LOCK_PATH",
15725 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
15726 car_auth::PARSLEE_API_BASE_KEY,
15727 crate::openrouter::API_KEY_ENV,
15728 "OPENAI_API_KEY",
15729 "ANTHROPIC_API_KEY",
15730 "GOOGLE_API_KEY",
15731 "ELEVENLABS_API_KEY",
15732 "SSL_CERT_FILE",
15733 "SSL_CERT_DIR",
15734 ] {
15735 command.env_remove(name);
15736 }
15737 const CHILD_BUDGET: std::time::Duration = std::time::Duration::from_secs(300);
15744 let mut stdout = tempfile::tempfile().expect("create isolated-test stdout capture");
15745 let mut stderr = tempfile::tempfile().expect("create isolated-test stderr capture");
15746 command
15747 .stdin(std::process::Stdio::null())
15748 .stdout(stdout.try_clone().expect("clone stdout capture"))
15749 .stderr(stderr.try_clone().expect("clone stderr capture"));
15750 #[cfg(unix)]
15751 {
15752 use std::os::unix::process::CommandExt;
15753 command.process_group(0);
15754 }
15755
15756 let mut child = command.spawn().expect("spawn isolated inference test");
15757 let deadline = std::time::Instant::now() + CHILD_BUDGET;
15758 let mut timed_out = false;
15759 let status = loop {
15760 match child.try_wait().expect("poll isolated inference test") {
15761 Some(status) => break status,
15762 None if std::time::Instant::now() < deadline => {
15763 std::thread::sleep(std::time::Duration::from_millis(50));
15764 }
15765 None => {
15766 timed_out = true;
15767 #[cfg(unix)]
15768 unsafe {
15769 libc::kill(-(child.id() as i32), libc::SIGKILL);
15770 }
15771 let _ = child.kill();
15772 break child
15773 .wait()
15774 .expect("reap timed-out isolated inference test");
15775 }
15776 }
15777 };
15778
15779 use std::io::{Read, Seek};
15780 stdout.rewind().expect("rewind isolated-test stdout");
15781 stderr.rewind().expect("rewind isolated-test stderr");
15782 let mut stdout_bytes = Vec::new();
15783 let mut stderr_bytes = Vec::new();
15784 stdout
15785 .read_to_end(&mut stdout_bytes)
15786 .expect("read isolated-test stdout");
15787 stderr
15788 .read_to_end(&mut stderr_bytes)
15789 .expect("read isolated-test stderr");
15790 assert!(
15791 !timed_out,
15792 "isolated {test_name} exceeded {}s and was killed\nstdout:\n{}\nstderr:\n{}",
15793 CHILD_BUDGET.as_secs(),
15794 String::from_utf8_lossy(&stdout_bytes),
15795 String::from_utf8_lossy(&stderr_bytes),
15796 );
15797 assert!(
15798 status.success(),
15799 "isolated {test_name} failed\nstdout:\n{}\nstderr:\n{}",
15800 String::from_utf8_lossy(&stdout_bytes),
15801 String::from_utf8_lossy(&stderr_bytes),
15802 );
15803 false
15804}
15805
15806fn pinned_by(agents: &[String]) -> String {
15808 let quoted: Vec<String> = agents.iter().map(|a| format!("\"{a}\"")).collect();
15809 match quoted.as_slice() {
15810 [one] => format!("pinned by the agent {one}"),
15811 _ => format!("pinned by the agents {}", quoted.join(", ")),
15812 }
15813}
15814
15815#[cfg(test)]
15816mod tests {
15817
15818 #[test]
15828 fn every_successful_generation_records_its_span() {
15829 let source = include_str!("lib.rs");
15830 let start = source
15831 .find("async fn generate_tracked_inner")
15832 .expect("the tracked-inference entry point");
15833 let open = source[start..].find('{').expect("function body") + start;
15834 let mut depth = 0usize;
15835 let mut end = open;
15836 for (offset, ch) in source[open..].char_indices() {
15837 match ch {
15838 '{' => depth += 1,
15839 '}' => {
15840 depth -= 1;
15841 if depth == 0 {
15842 end = open + offset;
15843 break;
15844 }
15845 }
15846 _ => {}
15847 }
15848 }
15849 let body = &source[start..end];
15850 let returns = body.matches("return Ok(InferenceResult {").count();
15851 let records = body.matches("record_generation_span(").count();
15852 assert!(
15853 returns >= 3,
15854 "control: the function still has several success returns, got {returns}"
15855 );
15856 assert_eq!(
15857 returns, records,
15858 "every successful return must record its generation span: \
15859 {returns} returns, {records} recorder calls"
15860 );
15861 }
15862 use super::*;
15863
15864 #[test]
15865 fn classify_defaults_to_jev_only_while_it_is_available() {
15866 let catalog: Vec<ModelSchema> =
15867 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
15868 let mut jev = catalog
15869 .iter()
15870 .find(|s| s.id == "parslee/jev")
15871 .unwrap()
15872 .clone();
15873 let local = catalog
15874 .iter()
15875 .find(|s| s.id == "mlx/qwen3-0.6b:6bit")
15876 .unwrap()
15877 .clone();
15878 jev.available = false;
15879 assert_eq!(
15880 default_system_one_classifier([&jev, &local].into_iter()),
15881 None,
15882 "signed out: keep the local default"
15883 );
15884 jev.available = true;
15885 assert_eq!(
15886 default_system_one_classifier([&local, &jev].into_iter()).as_deref(),
15887 Some("parslee/jev")
15888 );
15889 let chat = catalog
15891 .iter()
15892 .find(|s| s.id == "parslee/fast")
15893 .unwrap()
15894 .clone();
15895 assert_eq!(default_system_one_classifier([&chat].into_iter()), None);
15896 }
15897
15898 #[tokio::test]
15899 async fn generate_pinned_to_a_system_one_row_is_refused_streaming_included() {
15900 let dir = tempfile::tempdir().unwrap();
15901 let engine = InferenceEngine::new(test_config(dir.path().to_path_buf()));
15902 let req = || GenerateRequest {
15903 prompt: "hello".into(),
15904 model: Some("parslee/jev".into()),
15905 ..Default::default()
15906 };
15907 let refused = |e: &InferenceError| {
15908 matches!(
15909 e,
15910 InferenceError::UnsupportedMode {
15911 backend: "parslee-system-one",
15912 ..
15913 }
15914 )
15915 };
15916 match engine.generate_tracked(req()).await {
15917 Err(e) => assert!(refused(&e), "{e}"),
15918 Ok(_) => panic!("generate on a System One row must be refused"),
15919 }
15920 match engine.generate_tracked_stream(req()).await {
15921 Err(e) => assert!(refused(&e), "{e}"),
15922 Ok(_) => panic!("streaming generate on a System One row must be refused"),
15923 }
15924 }
15925 use std::ffi::OsString;
15926 use tempfile::TempDir;
15927
15928 #[test]
15935 fn local_models_keep_the_callers_default_output_budget() {
15936 use crate::tasks::generate::DEFAULT_MAX_TOKENS;
15937
15938 let catalog = crate::registry::builtin_catalog();
15939 let local = catalog
15940 .iter()
15941 .find(|s| s.id == "mlx/qwen3-8b:4bit")
15942 .expect("mlx/qwen3-8b:4bit is a builtin catalog entry");
15943
15944 assert!(local.is_local(), "mlx/* is decoded in-process");
15945 assert_eq!(
15946 local.effective_max_output(),
15947 32_768,
15948 "the budget that made the reported turn ~24 minutes long"
15949 );
15950 assert_eq!(
15951 resolved_max_tokens(DEFAULT_MAX_TOKENS, local),
15952 DEFAULT_MAX_TOKENS,
15953 "a local model must keep the budget the caller actually asked for"
15954 );
15955
15956 let mut remote = catalog
15959 .iter()
15960 .find(|s| !s.is_local())
15961 .expect("the builtin catalog ships remote models")
15962 .clone();
15963 remote.context_length = local.context_length;
15964 remote.max_output_tokens = local.max_output_tokens;
15965 assert!(!remote.is_local());
15966 assert_eq!(
15967 resolved_max_tokens(DEFAULT_MAX_TOKENS, &remote),
15968 32_768,
15969 "remote models still get their advertised output budget"
15970 );
15971
15972 assert_eq!(resolved_max_tokens(512, local), 512);
15974 assert_eq!(resolved_max_tokens(512, &remote), 512);
15975
15976 let codex = catalog
15977 .iter()
15978 .find(|s| s.id == "openai/gpt-5.6-sol:high")
15979 .expect("subscription-backed Codex row is builtin");
15980 assert_eq!(
15981 resolved_max_tokens(DEFAULT_MAX_TOKENS, codex),
15982 DEFAULT_MAX_TOKENS,
15983 "an approximate instruction must not widen the default to 128K tokens"
15984 );
15985
15986 if let Some(vllm) = catalog.iter().find(|s| s.is_vllm_mlx()) {
15992 assert!(vllm.is_local(), "vLLM-MLX runs on this machine");
15993 assert!(
15994 !vllm.decodes_in_process(),
15995 "but CAR does not decode it token by token"
15996 );
15997 assert_eq!(
15998 resolved_max_tokens(DEFAULT_MAX_TOKENS, vllm),
15999 vllm.effective_max_output(),
16000 );
16001 }
16002 }
16003
16004 #[test]
16007 fn a_ceiling_stop_fails_instead_of_retrying() {
16008 use EmptyPassAction::*;
16009 const CEILING: Option<&str> = Some(LOCAL_DECODE_TIMEOUT_STOP_REASON);
16010
16011 assert_eq!(
16014 classify_empty_pass(true, CEILING, "", true),
16015 FailDecodeCeiling
16016 );
16017 assert_eq!(
16018 classify_empty_pass(false, CEILING, "", true),
16019 FailDecodeCeiling
16020 );
16021 assert_eq!(
16023 classify_empty_pass(true, CEILING, " \n ", true),
16024 FailDecodeCeiling
16025 );
16026
16027 assert_eq!(
16030 classify_empty_pass(true, Some("length"), "", true),
16031 RetryWithoutThinking
16032 );
16033 assert_eq!(
16034 classify_empty_pass(true, None, "", true),
16035 RetryWithoutThinking
16036 );
16037
16038 assert_eq!(classify_empty_pass(false, Some("stop"), "", true), Accept);
16041
16042 assert_eq!(classify_empty_pass(true, CEILING, "partial", true), Accept);
16045 assert_eq!(classify_empty_pass(true, CEILING, "", false), Accept);
16046
16047 assert_ne!(LOCAL_DECODE_TIMEOUT_STOP_REASON, "timeout");
16051 assert_eq!(
16052 classify_empty_pass(true, Some("timeout"), "", true),
16053 RetryWithoutThinking
16054 );
16055 }
16056
16057 #[test]
16061 fn decode_deadline_and_heartbeat_predicates() {
16062 use std::time::Duration;
16063 let limit = Duration::from_secs(300);
16064
16065 assert!(!deadline_exceeded(Duration::from_secs(86_400), None));
16067
16068 assert!(!deadline_exceeded(Duration::from_secs(299), Some(limit)));
16069 assert!(deadline_exceeded(limit, Some(limit)));
16072 assert!(deadline_exceeded(Duration::from_secs(301), Some(limit)));
16073
16074 let every = Duration::from_secs(10);
16075 assert!(!heartbeat_due(
16077 Duration::from_secs(9),
16078 Duration::ZERO,
16079 every
16080 ));
16081 assert!(heartbeat_due(
16082 Duration::from_secs(10),
16083 Duration::ZERO,
16084 every
16085 ));
16086 assert!(!heartbeat_due(
16088 Duration::from_secs(11),
16089 Duration::from_secs(10),
16090 every
16091 ));
16092 assert!(heartbeat_due(
16093 Duration::from_secs(20),
16094 Duration::from_secs(10),
16095 every
16096 ));
16097 assert!(!heartbeat_due(
16099 Duration::from_secs(5),
16100 Duration::from_secs(10),
16101 every
16102 ));
16103 }
16104
16105 #[test]
16106 fn decode_timeout_parses_with_a_safe_fallback() {
16107 let default = std::time::Duration::from_secs(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS);
16108 assert_eq!(parse_decode_timeout(None), Some(default));
16109 assert_eq!(
16110 parse_decode_timeout(Some(" 45 ")),
16111 Some(std::time::Duration::from_secs(45))
16112 );
16113 assert_eq!(
16114 parse_decode_timeout(Some("0")),
16115 None,
16116 "0 disables the ceiling"
16117 );
16118 assert_eq!(
16119 parse_decode_timeout(Some("banana")),
16120 Some(default),
16121 "garbage must fall back to the default, never silently disable it"
16122 );
16123 }
16124
16125 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16130 #[test]
16131 fn local_decode_stops_at_the_wall_clock_ceiling() {
16132 struct NeverStops;
16135
16136 impl crate::backend::local::TextDecoder for NeverStops {
16137 fn encode(&self, _text: &str) -> Result<Vec<u32>, InferenceError> {
16138 Ok(vec![1, 2, 3])
16139 }
16140 fn decode(&self, tokens: &[u32]) -> Result<String, InferenceError> {
16141 Ok("x".repeat(tokens.len()))
16142 }
16143 fn forward(
16144 &mut self,
16145 _tokens: &[u32],
16146 _pos: usize,
16147 ) -> Result<Vec<f32>, InferenceError> {
16148 std::thread::sleep(std::time::Duration::from_millis(5));
16149 let mut logits = vec![0.0f32; 16];
16150 logits[7] = 10.0;
16151 Ok(logits)
16152 }
16153 fn eos_ids(&self) -> Vec<u32> {
16154 vec![0]
16155 }
16156 fn context_length(&self) -> usize {
16157 4096
16158 }
16159 fn clear_kv_cache(&mut self) {}
16160 }
16161
16162 let params = GenerateParams {
16165 max_tokens: 100_000,
16166 temperature: 0.0,
16167 ..Default::default()
16168 };
16169 let ceiling = std::time::Duration::from_millis(300);
16170
16171 let mut backend = NeverStops;
16172 let started = std::time::Instant::now();
16173 let generated = match InferenceEngine::drive_generation_with_timeout(
16174 &mut backend,
16175 "anything",
16176 ¶ms,
16177 Some(ceiling),
16178 ) {
16179 Ok(generated) => generated,
16180 Err(e) => panic!(
16182 "a decode cut short by the ceiling returns its partial text, not an error: {}",
16183 e.into_inner()
16184 ),
16185 };
16186 let elapsed = started.elapsed();
16187
16188 assert_eq!(
16189 generated.stop_reason.as_deref(),
16190 Some(LOCAL_DECODE_TIMEOUT_STOP_REASON),
16191 "the caller must be able to tell a deadline stop from a clean finish"
16192 );
16193 assert!(
16194 InferenceResult {
16195 text: generated.text.clone(),
16196 bounding_boxes: Vec::new(),
16197 tool_calls: vec![],
16198 trace_id: String::new(),
16199 model_used: String::new(),
16200 model_identity: Default::default(),
16201 latency_ms: 0,
16202 time_to_first_token_ms: None,
16203 usage: None,
16204 provider_output_items: Vec::new(),
16205 thinking: Vec::new(),
16206 stop_reason: generated.stop_reason.clone(),
16207 auth_fallback_from: None,
16208 local_last_resort: false,
16209 fallback_from: Vec::new(),
16210 }
16211 .was_truncated(),
16212 "a ceiling stop is a cut-short answer, not a complete one"
16213 );
16214 assert!(
16215 generated.completion_tokens > 0,
16216 "the partial response is kept, not discarded"
16217 );
16218 assert!(
16219 generated.completion_tokens < params.max_tokens,
16220 "the loop stopped on the clock, not by exhausting the budget"
16221 );
16222 assert!(
16223 elapsed < std::time::Duration::from_secs(30),
16224 "bounded in wall clock; took {elapsed:?}"
16225 );
16226 }
16227
16228 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
16229
16230 impl RestoredEnvironment {
16231 fn capture(names: &[&'static str]) -> Self {
16232 Self(
16233 names
16234 .iter()
16235 .map(|name| (*name, std::env::var_os(name)))
16236 .collect(),
16237 )
16238 }
16239 }
16240
16241 impl Drop for RestoredEnvironment {
16242 fn drop(&mut self) {
16243 for (name, value) in &self.0 {
16244 unsafe {
16245 match value {
16246 Some(value) => std::env::set_var(name, value),
16247 None => std::env::remove_var(name),
16248 }
16249 }
16250 }
16251 }
16252 }
16253
16254 struct FixtureLocalOffload {
16255 emit_done: bool,
16256 }
16257
16258 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16263 struct ExactPinCaptureOffload {
16264 dispatched_models: std::sync::Mutex<Vec<String>>,
16265 }
16266
16267 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16268 impl ExactPinCaptureOffload {
16269 fn new() -> Self {
16270 Self {
16271 dispatched_models: std::sync::Mutex::new(Vec::new()),
16272 }
16273 }
16274
16275 fn dispatched_models(&self) -> Vec<String> {
16276 self.dispatched_models.lock().unwrap().clone()
16277 }
16278 }
16279
16280 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16281 #[async_trait::async_trait]
16282 impl crate::offload::LocalGenerationOffload for ExactPinCaptureOffload {
16283 async fn generate(
16284 &self,
16285 request: GenerateRequest,
16286 ) -> Result<InferenceResult, InferenceError> {
16287 let model_id = request.model.expect("resolved worker model");
16288 self.dispatched_models
16289 .lock()
16290 .unwrap()
16291 .push(model_id.clone());
16292 Ok(InferenceResult {
16293 text: "exact pin".into(),
16294 tool_calls: vec![],
16295 bounding_boxes: vec![],
16296 trace_id: "worker-trace".into(),
16297 model_used: model_id,
16298 model_identity: Default::default(),
16299 latency_ms: 0,
16300 time_to_first_token_ms: None,
16301 usage: None,
16302 provider_output_items: vec![],
16303 thinking: vec![],
16304 stop_reason: Some("stop".into()),
16305 auth_fallback_from: None,
16306 local_last_resort: false,
16307 fallback_from: Vec::new(),
16308 })
16309 }
16310
16311 async fn stream(
16312 &self,
16313 request: GenerateRequest,
16314 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16315 let model_id = request.model.expect("resolved worker model");
16316 self.dispatched_models.lock().unwrap().push(model_id);
16317 let (tx, rx) = tokio::sync::mpsc::channel(2);
16318 tx.send(StreamEvent::Done {
16319 text: "exact pin".into(),
16320 tool_calls: vec![],
16321 })
16322 .await
16323 .unwrap();
16324 Ok(rx)
16325 }
16326 }
16327
16328 struct ReleaseRetryOffload {
16329 calls: std::sync::Mutex<Vec<String>>,
16330 model_id: String,
16331 resident: bool,
16332 release_acknowledged: bool,
16333 }
16334
16335 struct SequencedRetryProbe {
16336 calls: Arc<std::sync::atomic::AtomicUsize>,
16337 first_available_mb: u64,
16338 subsequent_available_mb: u64,
16339 }
16340
16341 impl crate::resource_policy::LiveMemoryProbe for SequencedRetryProbe {
16342 fn available_memory_mb(
16343 &self,
16344 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
16345 let call = self
16346 .calls
16347 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16348 Ok(Some(if call == 0 {
16349 self.first_available_mb
16350 } else {
16351 self.subsequent_available_mb
16352 }))
16353 }
16354 }
16355
16356 #[async_trait::async_trait]
16357 impl crate::offload::LocalGenerationOffload for ReleaseRetryOffload {
16358 async fn generate(
16359 &self,
16360 _request: GenerateRequest,
16361 ) -> Result<InferenceResult, InferenceError> {
16362 unreachable!("release-and-retry fixture")
16363 }
16364
16365 async fn stream(
16366 &self,
16367 _request: GenerateRequest,
16368 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16369 unreachable!("release-and-retry fixture")
16370 }
16371
16372 async fn resident_models(&self) -> Vec<String> {
16373 self.resident
16374 .then(|| self.model_id.clone())
16375 .into_iter()
16376 .collect()
16377 }
16378
16379 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
16380 Some(format!("worker:retry-fixture:{model_id}"))
16381 }
16382
16383 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16384 self.calls
16385 .lock()
16386 .unwrap()
16387 .push(format!("release:{model_id}"));
16388 Ok(self.release_acknowledged)
16389 }
16390 }
16391
16392 struct RetiringReleaseOffload {
16393 calls: std::sync::Mutex<Vec<String>>,
16394 model_id: String,
16395 allocation_id: String,
16396 resident: std::sync::atomic::AtomicBool,
16397 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16398 }
16399
16400 #[async_trait::async_trait]
16401 impl crate::offload::LocalGenerationOffload for RetiringReleaseOffload {
16402 async fn generate(
16403 &self,
16404 _request: GenerateRequest,
16405 ) -> Result<InferenceResult, InferenceError> {
16406 unreachable!("retiring-release fixture")
16407 }
16408
16409 async fn stream(
16410 &self,
16411 _request: GenerateRequest,
16412 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16413 unreachable!("retiring-release fixture")
16414 }
16415
16416 async fn resident_models(&self) -> Vec<String> {
16417 self.resident
16418 .load(std::sync::atomic::Ordering::Acquire)
16419 .then(|| self.model_id.clone())
16420 .into_iter()
16421 .collect()
16422 }
16423
16424 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16425 Some(self.allocation_id.clone())
16426 }
16427
16428 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16429 self.calls
16430 .lock()
16431 .unwrap()
16432 .push(format!("release:{model_id}"));
16433 self.coordinator
16434 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
16435 self.coordinator
16436 .finish_teardown_allocation(model_id, &self.allocation_id);
16437 self.resident
16438 .store(false, std::sync::atomic::Ordering::Release);
16439 Ok(true)
16440 }
16441 }
16442
16443 #[tokio::test]
16444 async fn release_and_retry_releases_resident_worker_after_live_memory_refusal() {
16445 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16446 let root = tempfile::tempdir().unwrap();
16447 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16448 let schema = crate::registry::builtin_catalog()
16449 .into_iter()
16450 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16451 .unwrap();
16452 let allocation_id = format!("worker:retry-fixture:{}", schema.id);
16453 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16454 let coordinator = Arc::new(
16455 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16456 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16457 metal_mac_for_fit(32),
16458 Arc::new(SequencedRetryProbe {
16459 calls: probe_calls.clone(),
16460 first_available_mb: 1_000,
16461 subsequent_available_mb: 1_048_576,
16462 }),
16463 ),
16464 );
16465 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
16466 engine.local_admission = coordinator.clone();
16467 let offload = Arc::new(RetiringReleaseOffload {
16468 calls: std::sync::Mutex::new(Vec::new()),
16469 model_id: schema.id.clone(),
16470 allocation_id,
16471 resident: std::sync::atomic::AtomicBool::new(true),
16472 coordinator,
16473 });
16474 crate::offload::set_local_offload(Some(offload.clone()));
16475
16476 let result = engine
16477 .reserve_local_request_with_worker_retry(&schema, 9_000)
16478 .await;
16479 crate::offload::set_local_offload(None);
16480
16481 assert!(
16482 result.is_ok(),
16483 "release followed by a second probe should admit"
16484 );
16485 assert_eq!(
16486 *offload.calls.lock().unwrap(),
16487 vec![format!("release:{}", schema.id)]
16488 );
16489 assert_eq!(
16490 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
16491 2,
16492 "admission must probe exactly once before and once after release"
16493 );
16494 }
16495
16496 #[tokio::test]
16497 async fn release_and_retry_ignores_sibling_allocation_after_exact_worker_retirement() {
16498 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16499 let root = tempfile::tempdir().unwrap();
16500 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16501 let schema = crate::registry::builtin_catalog()
16502 .into_iter()
16503 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16504 .unwrap();
16505 let worker_allocation_id = format!("worker:sibling-test:{}", schema.id);
16506 let sibling_allocation_id = format!("cache:sibling-test:{}", schema.id);
16507 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16508 let coordinator = Arc::new(
16509 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16510 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16511 metal_mac_for_fit(32),
16512 Arc::new(SequencedRetryProbe {
16513 calls: probe_calls.clone(),
16514 first_available_mb: 1_000,
16515 subsequent_available_mb: 1_048_576,
16516 }),
16517 ),
16518 );
16519 coordinator.mark_resident_allocation(&schema.id, &worker_allocation_id, 4_800);
16520 coordinator.mark_resident_allocation(&schema.id, &sibling_allocation_id, 500);
16521 engine.local_admission = coordinator.clone();
16522 let offload = Arc::new(RetiringReleaseOffload {
16523 calls: std::sync::Mutex::new(Vec::new()),
16524 model_id: schema.id.clone(),
16525 allocation_id: worker_allocation_id.clone(),
16526 resident: std::sync::atomic::AtomicBool::new(true),
16527 coordinator: coordinator.clone(),
16528 });
16529 crate::offload::set_local_offload(Some(offload.clone()));
16530
16531 let result = engine
16532 .reserve_local_request_with_worker_retry(&schema, 9_000)
16533 .await;
16534 crate::offload::set_local_offload(None);
16535
16536 assert!(
16537 result.is_ok(),
16538 "an unrelated resident allocation must not block the worker retry"
16539 );
16540 assert_eq!(
16541 coordinator.resident_allocation_ids(&schema.id),
16542 vec![sibling_allocation_id]
16543 );
16544 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 2);
16545 assert_eq!(
16546 *offload.calls.lock().unwrap(),
16547 vec![format!("release:{}", schema.id)]
16548 );
16549 }
16550
16551 struct UnretiredReleaseOffload {
16552 calls: std::sync::Mutex<Vec<String>>,
16553 model_id: String,
16554 allocation_id: String,
16555 resident: std::sync::atomic::AtomicBool,
16556 }
16557
16558 #[async_trait::async_trait]
16559 impl crate::offload::LocalGenerationOffload for UnretiredReleaseOffload {
16560 async fn generate(
16561 &self,
16562 _request: GenerateRequest,
16563 ) -> Result<InferenceResult, InferenceError> {
16564 unreachable!("unretired-release fixture")
16565 }
16566
16567 async fn stream(
16568 &self,
16569 _request: GenerateRequest,
16570 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16571 unreachable!("unretired-release fixture")
16572 }
16573
16574 async fn resident_models(&self) -> Vec<String> {
16575 self.resident
16576 .load(std::sync::atomic::Ordering::Acquire)
16577 .then(|| self.model_id.clone())
16578 .into_iter()
16579 .collect()
16580 }
16581
16582 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16583 Some(self.allocation_id.clone())
16584 }
16585
16586 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16587 self.calls
16588 .lock()
16589 .unwrap()
16590 .push(format!("release:{model_id}"));
16591 self.resident
16592 .store(false, std::sync::atomic::Ordering::Release);
16593 Ok(true)
16594 }
16595 }
16596
16597 #[tokio::test]
16598 async fn release_and_retry_rejects_ack_without_accounting_retirement() {
16599 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16600 let warnings_before =
16601 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
16602 let root = tempfile::tempdir().unwrap();
16603 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16604 let schema = crate::registry::builtin_catalog()
16605 .into_iter()
16606 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16607 .unwrap();
16608 let allocation_id = format!("worker:phantom:{}", schema.id);
16609 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16610 let coordinator = Arc::new(
16611 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16612 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16613 metal_mac_for_fit(32),
16614 Arc::new(SequencedRetryProbe {
16615 calls: probe_calls.clone(),
16616 first_available_mb: 1_000,
16617 subsequent_available_mb: 8_000,
16618 }),
16619 ),
16620 );
16621 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
16622 engine.local_admission = coordinator;
16623 let offload = Arc::new(UnretiredReleaseOffload {
16624 calls: std::sync::Mutex::new(Vec::new()),
16625 model_id: schema.id.clone(),
16626 allocation_id,
16627 resident: std::sync::atomic::AtomicBool::new(true),
16628 });
16629 crate::offload::set_local_offload(Some(offload.clone()));
16630
16631 let result = engine
16632 .reserve_local_request_with_worker_retry(&schema, 9_000)
16633 .await;
16634 crate::offload::set_local_offload(None);
16635
16636 assert!(matches!(
16637 result,
16638 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16639 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16640 ));
16641 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16642 assert_eq!(
16643 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
16644 warnings_before + 1,
16645 "the refused retry must emit the stale-residency warning"
16646 );
16647 assert!(engine.local_admission.is_resident(&schema.id));
16648 assert!(offload.resident_models().await.is_empty());
16649 assert_eq!(
16650 *offload.calls.lock().unwrap(),
16651 vec![format!("release:{}", schema.id)]
16652 );
16653 }
16654
16655 struct PendingTeardownAckOffload {
16656 calls: std::sync::Mutex<Vec<String>>,
16657 model_id: String,
16658 allocation_id: String,
16659 resident: std::sync::atomic::AtomicBool,
16660 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16661 }
16662
16663 #[async_trait::async_trait]
16664 impl crate::offload::LocalGenerationOffload for PendingTeardownAckOffload {
16665 async fn generate(
16666 &self,
16667 _request: GenerateRequest,
16668 ) -> Result<InferenceResult, InferenceError> {
16669 unreachable!("pending-teardown ACK fixture")
16670 }
16671
16672 async fn stream(
16673 &self,
16674 _request: GenerateRequest,
16675 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16676 unreachable!("pending-teardown ACK fixture")
16677 }
16678
16679 async fn resident_models(&self) -> Vec<String> {
16680 self.resident
16681 .load(std::sync::atomic::Ordering::Acquire)
16682 .then(|| self.model_id.clone())
16683 .into_iter()
16684 .collect()
16685 }
16686
16687 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16688 Some(self.allocation_id.clone())
16689 }
16690
16691 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16692 self.calls
16693 .lock()
16694 .unwrap()
16695 .push(format!("release:{model_id}"));
16696 self.coordinator
16697 .mark_teardown_pending_allocation_with_charge(
16698 model_id,
16699 &self.allocation_id,
16700 4_800 * 1024 * 1024,
16701 );
16702 self.resident
16703 .store(false, std::sync::atomic::Ordering::Release);
16704 Ok(true)
16705 }
16706 }
16707
16708 #[tokio::test]
16709 async fn release_and_retry_rejects_ack_with_exact_worker_pending_teardown() {
16710 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16711 let warnings_before =
16712 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
16713 let root = tempfile::tempdir().unwrap();
16714 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16715 let schema = crate::registry::builtin_catalog()
16716 .into_iter()
16717 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16718 .unwrap();
16719 let allocation_id = format!("worker:pending-test:{}", schema.id);
16720 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16721 let coordinator = Arc::new(
16722 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16723 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16724 metal_mac_for_fit(32),
16725 Arc::new(SequencedRetryProbe {
16726 calls: probe_calls.clone(),
16727 first_available_mb: 1_000,
16728 subsequent_available_mb: 8_000,
16729 }),
16730 ),
16731 );
16732 engine.local_admission = coordinator.clone();
16733 let offload = Arc::new(PendingTeardownAckOffload {
16734 calls: std::sync::Mutex::new(Vec::new()),
16735 model_id: schema.id.clone(),
16736 allocation_id: allocation_id.clone(),
16737 resident: std::sync::atomic::AtomicBool::new(true),
16738 coordinator: coordinator.clone(),
16739 });
16740 crate::offload::set_local_offload(Some(offload.clone()));
16741
16742 let result = engine
16743 .reserve_local_request_with_worker_retry(&schema, 9_000)
16744 .await;
16745 crate::offload::set_local_offload(None);
16746
16747 assert!(matches!(
16748 result,
16749 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16750 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16751 ));
16752 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16753 assert_eq!(
16754 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
16755 warnings_before + 1,
16756 "the pending exact allocation must emit the fail-closed warning"
16757 );
16758 assert_eq!(
16759 coordinator.resident_allocation_ids(&schema.id),
16760 vec![allocation_id]
16761 );
16762 assert!(offload.resident_models().await.is_empty());
16763 assert_eq!(
16764 *offload.calls.lock().unwrap(),
16765 vec![format!("release:{}", schema.id)]
16766 );
16767 }
16768
16769 struct MissingAllocationIdOffload {
16770 calls: std::sync::Mutex<Vec<String>>,
16771 model_id: String,
16772 }
16773
16774 #[async_trait::async_trait]
16775 impl crate::offload::LocalGenerationOffload for MissingAllocationIdOffload {
16776 async fn generate(
16777 &self,
16778 _request: GenerateRequest,
16779 ) -> Result<InferenceResult, InferenceError> {
16780 unreachable!("missing allocation-id fixture")
16781 }
16782
16783 async fn stream(
16784 &self,
16785 _request: GenerateRequest,
16786 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16787 unreachable!("missing allocation-id fixture")
16788 }
16789
16790 async fn resident_models(&self) -> Vec<String> {
16791 vec![self.model_id.clone()]
16792 }
16793
16794 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16795 self.calls
16796 .lock()
16797 .unwrap()
16798 .push(format!("release:{model_id}"));
16799 Ok(true)
16800 }
16801 }
16802
16803 #[tokio::test]
16804 async fn release_and_retry_without_scoped_allocation_id_fails_closed() {
16805 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16806 let root = tempfile::tempdir().unwrap();
16807 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16808 let schema = crate::registry::builtin_catalog()
16809 .into_iter()
16810 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16811 .unwrap();
16812 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16813 engine.local_admission = Arc::new(
16814 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16815 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16816 metal_mac_for_fit(32),
16817 Arc::new(SequencedRetryProbe {
16818 calls: probe_calls.clone(),
16819 first_available_mb: 1_000,
16820 subsequent_available_mb: 8_000,
16821 }),
16822 ),
16823 );
16824 let offload = Arc::new(MissingAllocationIdOffload {
16825 calls: std::sync::Mutex::new(Vec::new()),
16826 model_id: schema.id.clone(),
16827 });
16828 crate::offload::set_local_offload(Some(offload.clone()));
16829
16830 let result = engine
16831 .reserve_local_request_with_worker_retry(&schema, 9_000)
16832 .await;
16833 crate::offload::set_local_offload(None);
16834
16835 assert!(matches!(
16836 result,
16837 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16838 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16839 ));
16840 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16841 assert!(offload.calls.lock().unwrap().is_empty());
16842 }
16843
16844 struct ReplacingResidentOffload {
16845 calls: std::sync::Mutex<Vec<String>>,
16846 model_id: String,
16847 allocation_id: String,
16848 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16849 }
16850
16851 #[async_trait::async_trait]
16852 impl crate::offload::LocalGenerationOffload for ReplacingResidentOffload {
16853 async fn generate(
16854 &self,
16855 _request: GenerateRequest,
16856 ) -> Result<InferenceResult, InferenceError> {
16857 unreachable!("replacement-resident fixture")
16858 }
16859
16860 async fn stream(
16861 &self,
16862 _request: GenerateRequest,
16863 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16864 unreachable!("replacement-resident fixture")
16865 }
16866
16867 async fn resident_models(&self) -> Vec<String> {
16868 vec![self.model_id.clone()]
16869 }
16870
16871 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16872 Some(self.allocation_id.clone())
16873 }
16874
16875 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16876 self.calls
16877 .lock()
16878 .unwrap()
16879 .push(format!("release:{model_id}"));
16880 self.coordinator
16884 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
16885 self.coordinator
16886 .finish_teardown_allocation(model_id, &self.allocation_id);
16887 self.coordinator
16888 .mark_resident_allocation(model_id, &self.allocation_id, 4_800);
16889 Ok(true)
16890 }
16891 }
16892
16893 #[tokio::test]
16894 async fn release_and_retry_same_id_replacement_blocks_retry_without_erasing_accounting() {
16895 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16896 let root = tempfile::tempdir().unwrap();
16897 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16898 let schema = crate::registry::builtin_catalog()
16899 .into_iter()
16900 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16901 .unwrap();
16902 let allocation_id = format!("worker:reused-scope:{}", schema.id);
16903 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16904 let coordinator = Arc::new(
16905 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16906 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16907 metal_mac_for_fit(32),
16908 Arc::new(SequencedRetryProbe {
16909 calls: probe_calls.clone(),
16910 first_available_mb: 1_000,
16911 subsequent_available_mb: 8_000,
16912 }),
16913 ),
16914 );
16915 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
16916 engine.local_admission = coordinator.clone();
16917 let offload = Arc::new(ReplacingResidentOffload {
16918 calls: std::sync::Mutex::new(Vec::new()),
16919 model_id: schema.id.clone(),
16920 allocation_id,
16921 coordinator,
16922 });
16923 crate::offload::set_local_offload(Some(offload.clone()));
16924
16925 let result = engine
16926 .reserve_local_request_with_worker_retry(&schema, 9_000)
16927 .await;
16928 crate::offload::set_local_offload(None);
16929
16930 assert!(matches!(
16931 result,
16932 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16933 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16934 ));
16935 assert!(engine.local_admission.is_resident(&schema.id));
16936 assert_eq!(
16937 *offload.calls.lock().unwrap(),
16938 vec![format!("release:{}", schema.id)]
16939 );
16940 assert_eq!(
16941 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
16942 1,
16943 "replacement residency must stop the retry before a second probe"
16944 );
16945 }
16946
16947 #[tokio::test]
16948 async fn release_and_retry_nonresident_worker_returns_original_refusal() {
16949 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16950 let root = tempfile::tempdir().unwrap();
16951 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16952 let schema = crate::registry::builtin_catalog()
16953 .into_iter()
16954 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16955 .unwrap();
16956 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16957 engine.local_admission = Arc::new(
16958 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16959 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16960 metal_mac_for_fit(32),
16961 Arc::new(SequencedRetryProbe {
16962 calls: probe_calls.clone(),
16963 first_available_mb: 1_000,
16964 subsequent_available_mb: 1_048_576,
16965 }),
16966 ),
16967 );
16968 let offload = Arc::new(ReleaseRetryOffload {
16969 calls: std::sync::Mutex::new(Vec::new()),
16970 model_id: schema.id.clone(),
16971 resident: false,
16972 release_acknowledged: true,
16973 });
16974 crate::offload::set_local_offload(Some(offload.clone()));
16975
16976 let result = engine
16977 .reserve_local_request_with_worker_retry(&schema, 9_000)
16978 .await;
16979 crate::offload::set_local_offload(None);
16980
16981 assert!(matches!(
16982 result,
16983 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16984 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16985 ));
16986 assert!(offload.calls.lock().unwrap().is_empty());
16987 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16988 }
16989
16990 #[tokio::test]
16991 async fn release_and_retry_unacknowledged_release_does_not_reprobe() {
16992 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16993 let root = tempfile::tempdir().unwrap();
16994 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16995 let schema = crate::registry::builtin_catalog()
16996 .into_iter()
16997 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16998 .unwrap();
16999 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
17000 engine.local_admission = Arc::new(
17001 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
17002 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
17003 metal_mac_for_fit(32),
17004 Arc::new(SequencedRetryProbe {
17005 calls: probe_calls.clone(),
17006 first_available_mb: 1_000,
17007 subsequent_available_mb: 1_048_576,
17008 }),
17009 ),
17010 );
17011 let offload = Arc::new(ReleaseRetryOffload {
17012 calls: std::sync::Mutex::new(Vec::new()),
17013 model_id: schema.id.clone(),
17014 resident: true,
17015 release_acknowledged: false,
17016 });
17017 crate::offload::set_local_offload(Some(offload.clone()));
17018
17019 let result = engine
17020 .reserve_local_request_with_worker_retry(&schema, 9_000)
17021 .await;
17022 crate::offload::set_local_offload(None);
17023
17024 assert!(matches!(
17025 result,
17026 Err(InferenceError::LocalResourceBlocked { preflight, .. })
17027 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
17028 ));
17029 assert_eq!(
17030 *offload.calls.lock().unwrap(),
17031 vec![format!("release:{}", schema.id)]
17032 );
17033 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
17034 }
17035
17036 #[tokio::test]
17037 async fn release_and_retry_allowed_first_verdict_never_releases() {
17038 let _offload_guard = crate::offload::test_offload_lock().lock().await;
17039 let root = tempfile::tempdir().unwrap();
17040 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
17041 let schema = crate::registry::builtin_catalog()
17042 .into_iter()
17043 .find(|model| model.id == "mlx/qwen3-8b:4bit")
17044 .unwrap();
17045 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
17046 engine.local_admission = Arc::new(
17047 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
17048 crate::resource_policy::ResourcePolicy::local_focused(),
17049 metal_mac_for_fit(64),
17050 Arc::new(SequencedRetryProbe {
17051 calls: probe_calls.clone(),
17052 first_available_mb: 1_048_576,
17053 subsequent_available_mb: 1_048_576,
17054 }),
17055 ),
17056 );
17057 let offload = Arc::new(ReleaseRetryOffload {
17058 calls: std::sync::Mutex::new(Vec::new()),
17059 model_id: schema.id.clone(),
17060 resident: true,
17061 release_acknowledged: true,
17062 });
17063 crate::offload::set_local_offload(Some(offload.clone()));
17064
17065 let result = engine
17066 .reserve_local_request_with_worker_retry(&schema, 9_000)
17067 .await;
17068 crate::offload::set_local_offload(None);
17069
17070 assert!(result.is_ok());
17071 assert!(offload.calls.lock().unwrap().is_empty());
17072 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
17073 }
17074
17075 struct ReconcileOffload {
17076 calls: std::sync::Mutex<Vec<String>>,
17077 release_acknowledged: bool,
17078 }
17079
17080 #[async_trait::async_trait]
17081 impl crate::offload::LocalGenerationOffload for ReconcileOffload {
17082 async fn generate(
17083 &self,
17084 _request: GenerateRequest,
17085 ) -> Result<InferenceResult, InferenceError> {
17086 unreachable!("residency reconciliation fixture")
17087 }
17088
17089 async fn stream(
17090 &self,
17091 _request: GenerateRequest,
17092 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
17093 unreachable!("residency reconciliation fixture")
17094 }
17095
17096 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
17097 self.calls
17098 .lock()
17099 .unwrap()
17100 .push(format!("allocation:{model_id}"));
17101 Some(format!("worker:{model_id}:generation-7"))
17102 }
17103
17104 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
17105 self.calls
17106 .lock()
17107 .unwrap()
17108 .push(format!("release:{model_id}"));
17109 Ok(self.release_acknowledged)
17110 }
17111 }
17112
17113 fn reconciliation_reservation() -> (
17114 Arc<crate::resource_policy::LocalAdmissionCoordinator>,
17115 crate::resource_policy::LocalLoadReservation,
17116 ) {
17117 struct FixedProbe;
17118 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
17119 fn available_memory_mb(
17120 &self,
17121 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
17122 Ok(Some(24_000))
17123 }
17124 }
17125 let coordinator = Arc::new(
17126 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
17127 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
17128 crate::hardware::HardwareInfo {
17129 total_ram_mb: 32 * 1024,
17130 ..crate::hardware::HardwareInfo::detect()
17131 },
17132 Arc::new(FixedProbe),
17133 ),
17134 );
17135 let reservation = coordinator
17136 .reserve_measured_host_allocation(
17137 "expected/model",
17138 "expected/model#request",
17139 1024 * 1024,
17140 0,
17141 )
17142 .unwrap();
17143 (coordinator, reservation)
17144 }
17145
17146 #[tokio::test]
17147 async fn worker_residency_reconciliation_releases_mismatched_ack_before_returning_error() {
17148 let offload = ReconcileOffload {
17149 calls: std::sync::Mutex::new(Vec::new()),
17150 release_acknowledged: true,
17151 };
17152 let (_coordinator, mut reservation) = reconciliation_reservation();
17153 let error = InferenceEngine::reconcile_worker_residency(
17154 &offload,
17155 "expected/model",
17156 &crate::offload::LocalWorkerResidency {
17157 model_id: "wrong/model".into(),
17158 measured_weights_bytes: 2 * 1024 * 1024,
17159 },
17160 backend_cache::BackendRetention::Resident,
17161 &mut reservation,
17162 )
17163 .await
17164 .unwrap_err();
17165 assert!(error.to_string().contains("wrong/model"));
17166 assert_eq!(
17167 *offload.calls.lock().unwrap(),
17168 vec!["release:wrong/model".to_string()]
17169 );
17170 }
17171
17172 #[tokio::test]
17173 async fn worker_residency_reconciliation_publishes_exact_owner_for_matching_ack() {
17174 let offload = ReconcileOffload {
17175 calls: std::sync::Mutex::new(Vec::new()),
17176 release_acknowledged: false,
17177 };
17178 let (coordinator, mut reservation) = reconciliation_reservation();
17179 InferenceEngine::reconcile_worker_residency(
17180 &offload,
17181 "expected/model",
17182 &crate::offload::LocalWorkerResidency {
17183 model_id: "expected/model".into(),
17184 measured_weights_bytes: 2 * 1024 * 1024,
17185 },
17186 backend_cache::BackendRetention::Resident,
17187 &mut reservation,
17188 )
17189 .await
17190 .unwrap();
17191 drop(reservation);
17192 assert_eq!(
17193 *offload.calls.lock().unwrap(),
17194 vec!["allocation:expected/model".to_string()]
17195 );
17196 assert!(coordinator.is_resident("expected/model"));
17197 }
17198
17199 #[async_trait::async_trait]
17200 impl crate::offload::LocalGenerationOffload for FixtureLocalOffload {
17201 async fn generate(
17202 &self,
17203 _request: GenerateRequest,
17204 ) -> Result<InferenceResult, InferenceError> {
17205 unreachable!("streaming fixture")
17206 }
17207
17208 async fn stream(
17209 &self,
17210 _request: GenerateRequest,
17211 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
17212 unreachable!("admission-aware streaming fixture")
17213 }
17214
17215 async fn stream_admitted(
17216 &self,
17217 request: GenerateRequest,
17218 admission: crate::offload::LocalWorkerAdmission,
17219 ) -> Result<crate::offload::LocalOffloadStream, InferenceError> {
17220 let (tx, rx) = tokio::sync::mpsc::channel(4);
17221 let emit_done = self.emit_done;
17222 tokio::spawn(async move {
17223 let _ = tx.send(StreamEvent::TextDelta("local answer".into())).await;
17224 if emit_done {
17225 let _ = tx
17226 .send(StreamEvent::Done {
17227 text: "local answer".into(),
17228 tool_calls: vec![],
17229 })
17230 .await;
17231 }
17232 });
17233 Ok(crate::offload::LocalOffloadStream {
17234 events: rx,
17235 residency: crate::offload::LocalWorkerResidency {
17236 model_id: request.model.unwrap_or_else(|| "fixture/local".into()),
17237 measured_weights_bytes: admission.measured_weights_bytes,
17238 },
17239 retention: backend_cache::BackendRetention::Resident,
17240 })
17241 }
17242 }
17243
17244 fn pin_test_local_admission_memory(engine: &mut InferenceEngine) {
17252 let policy = engine.local_admission.policy();
17253 let mut hardware = HardwareInfo::detect();
17254 hardware.total_ram_mb = hardware.total_ram_mb.max(16 * 1024);
17255 hardware.max_model_mb = hardware.max_model_mb.max(16 * 1024);
17256 engine.local_admission = Arc::new(resource_policy::LocalAdmissionCoordinator::with_probe(
17257 policy,
17258 hardware,
17259 Arc::new(resource_policy::FixedLiveMemoryProbe::known(1_048_576)),
17260 ));
17261 }
17262
17263 fn install_small_local_fixture(engine: &InferenceEngine) -> String {
17264 let schema = engine
17265 .unified_registry()
17266 .find_by_name("Qwen3-0.6B")
17267 .expect("small built-in local model")
17268 .clone();
17269 let model_dir = engine.config.models_dir.join(&schema.name);
17270 std::fs::create_dir_all(&model_dir).unwrap();
17271 std::fs::write(model_dir.join("model.gguf"), b"fixture").unwrap();
17272 std::fs::write(model_dir.join("tokenizer.json"), b"{}").unwrap();
17273 schema.id
17274 }
17275
17276 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
17277 fn install_exact_pin_equivalent_fixture(engine: &InferenceEngine) -> String {
17278 let gguf = engine
17279 .unified_registry()
17280 .get("qwen/qwen3-0.6b:q8_0")
17281 .expect("GGUF fixture row")
17282 .clone();
17283 let mlx = engine
17284 .unified_registry()
17285 .get("mlx/qwen3-0.6b:6bit")
17286 .expect("MLX equivalent fixture row")
17287 .clone();
17288
17289 let gguf_dir = engine.config.models_dir.join(&gguf.name);
17290 std::fs::create_dir_all(&gguf_dir).unwrap();
17291 std::fs::write(gguf_dir.join("model.gguf"), b"fixture").unwrap();
17292 std::fs::write(gguf_dir.join("tokenizer.json"), b"{}").unwrap();
17293
17294 let mlx_dir = engine.config.models_dir.join(&mlx.name);
17295 std::fs::create_dir_all(&mlx_dir).unwrap();
17296 std::fs::write(mlx_dir.join("config.json"), b"{}").unwrap();
17297 std::fs::write(mlx_dir.join("model.safetensors"), b"fixture").unwrap();
17298
17299 gguf.id
17300 }
17301
17302 fn remote_stream_fixture_schema(
17303 id: &str,
17304 endpoint: String,
17305 protocol: schema::ApiProtocol,
17306 api_key_env: &str,
17307 ) -> ModelSchema {
17308 ModelSchema {
17309 id: id.into(),
17310 name: "gemini-test".into(),
17311 provider: "test".into(),
17312 family: "test".into(),
17313 version: "1".into(),
17314 capabilities: vec![ModelCapability::Generate],
17315 context_length: 128_000,
17316 max_output_tokens: Some(8_192),
17317 param_count: String::new(),
17318 quantization: None,
17319 performance: Default::default(),
17320 cost: Default::default(),
17321 source: ModelSource::RemoteApi {
17322 endpoint,
17323 api_key_env: api_key_env.into(),
17324 api_key_envs: vec![],
17325 api_version: None,
17326 protocol,
17327 },
17328 tags: vec!["test".into()],
17329 supported_params: vec![],
17330 public_benchmarks: vec![],
17331 trust_tier: TrustTier::Community,
17332 deprecated: false,
17333 available: true,
17334 weights_ready: true,
17335 }
17336 }
17337
17338 async fn assert_remote_model_identity_contract(
17339 protocol: schema::ApiProtocol,
17340 model_id: &str,
17341 provider_model: &str,
17342 api_key_env: &str,
17343 ) {
17344 use wiremock::matchers::{method, path};
17345 use wiremock::{Mock, MockServer, ResponseTemplate};
17346
17347 let server = MockServer::start().await;
17348 let (endpoint_path, response) = match protocol {
17349 schema::ApiProtocol::OpenAiCompat => (
17350 "/v1/chat/completions",
17351 serde_json::json!({
17352 "choices": [{
17353 "message": {"content": "openai exact"},
17354 "finish_reason": "stop"
17355 }],
17356 "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}
17357 }),
17358 ),
17359 schema::ApiProtocol::Anthropic => (
17360 "/v1/messages",
17361 serde_json::json!({
17362 "content": [{"type": "text", "text": "anthropic exact"}],
17363 "stop_reason": "end_turn",
17364 "usage": {"input_tokens": 2, "output_tokens": 2}
17365 }),
17366 ),
17367 _ => unreachable!("identity regression covers the newsroom's two remote providers"),
17368 };
17369 Mock::given(method("POST"))
17370 .and(path(endpoint_path))
17371 .respond_with(ResponseTemplate::new(200).set_body_json(response))
17372 .mount(&server)
17373 .await;
17374 unsafe { std::env::set_var(api_key_env, "fixture") };
17375
17376 let tmp = TempDir::new().unwrap();
17377 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
17378 let mut schema =
17379 remote_stream_fixture_schema(model_id, server.uri(), protocol, api_key_env);
17380 schema.name = provider_model.into();
17381 engine.register_model(schema);
17382
17383 let mut exact_request = GenerateRequest {
17384 prompt: "return the exact identity".into(),
17385 ..Default::default()
17386 };
17387 pin_exact_model_id(&mut exact_request, model_id.into()).unwrap();
17388 let exact = engine.generate_tracked(exact_request).await.unwrap();
17389 assert_eq!(exact.model_used, model_id);
17390 assert_eq!(
17391 exact.model_identity.requested_model_id.as_deref(),
17392 Some(model_id)
17393 );
17394 assert_eq!(exact.model_identity.resolved_model_id, model_id);
17395 let exact_envelope = serde_json::to_value(&exact).unwrap();
17396 assert_eq!(exact_envelope["model_used"], model_id);
17397 assert_eq!(exact_envelope["requested_model_id"], model_id);
17398 assert_eq!(exact_envelope["resolved_model_id"], model_id);
17399 assert!(exact_envelope["row_digest"].is_string());
17400 assert!(exact_envelope["catalog_revision"].is_string());
17401
17402 let loose = engine
17403 .generate_tracked(GenerateRequest {
17404 prompt: "keep legacy display-name routing".into(),
17405 model: Some(provider_model.into()),
17406 params: GenerateParams {
17407 strict_model: true,
17408 ..Default::default()
17409 },
17410 ..Default::default()
17411 })
17412 .await
17413 .unwrap();
17414 assert_eq!(loose.model_used, provider_model);
17415 assert_eq!(loose.model_identity.requested_model_id, None);
17416 assert_eq!(loose.model_identity.resolved_model_id, model_id);
17417
17418 let requests = server.received_requests().await.unwrap();
17419 assert_eq!(requests.len(), 2);
17420 for request in requests {
17421 let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
17422 assert_eq!(body["model"], provider_model);
17423 }
17424
17425 unsafe { std::env::remove_var(api_key_env) };
17426 }
17427
17428 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
17434 #[tokio::test]
17435 async fn mlx_device_lock_is_singleton_and_serializes() {
17436 use std::sync::atomic::{AtomicUsize, Ordering};
17437 use std::sync::Arc;
17438 assert!(
17440 Arc::ptr_eq(
17441 &InferenceEngine::mlx_device_lock(),
17442 &InferenceEngine::mlx_device_lock()
17443 ),
17444 "device lock must be a process-wide singleton"
17445 );
17446 let inside = Arc::new(AtomicUsize::new(0));
17448 let peak = Arc::new(AtomicUsize::new(0));
17449 let mut handles = Vec::new();
17450 for _ in 0..8 {
17451 let inside = inside.clone();
17452 let peak = peak.clone();
17453 handles.push(tokio::spawn(async move {
17454 let _g = InferenceEngine::mlx_device_lock().lock_owned().await;
17455 let n = inside.fetch_add(1, Ordering::SeqCst) + 1;
17456 peak.fetch_max(n, Ordering::SeqCst);
17457 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
17458 inside.fetch_sub(1, Ordering::SeqCst);
17459 }));
17460 }
17461 for h in handles {
17462 h.await.unwrap();
17463 }
17464 assert_eq!(
17465 peak.load(Ordering::SeqCst),
17466 1,
17467 "at most one MLX device holder at a time"
17468 );
17469 }
17470
17471 #[test]
17476 fn reasoning_output_allowance_adds_the_budget_only_where_it_is_charged() {
17477 fn remote(name: &str, protocol: ApiProtocol, max_output: usize) -> ModelSchema {
17478 let mut schema = chain_gate_fixture_schema(
17479 name,
17480 "test",
17481 ModelSource::RemoteApi {
17482 endpoint: "https://example.invalid".into(),
17483 api_key_env: "TEST_KEY".into(),
17484 api_key_envs: vec![],
17485 api_version: None,
17486 protocol,
17487 },
17488 );
17489 schema.max_output_tokens = Some(max_output);
17490 schema
17491 }
17492 let high = 16_384;
17493 let gpt_4o = remote("gpt-4o", ApiProtocol::OpenAiCompat, 16_384);
17494 assert_eq!(reasoning_output_allowance(Some(&gpt_4o), 2048, high), 2048);
17495 let gpt_4_turbo = remote("gpt-4-turbo", ApiProtocol::OpenAiCompat, 4096);
17496 assert_eq!(
17497 reasoning_output_allowance(Some(&gpt_4_turbo), 2048, 4096),
17498 2048
17499 );
17500 let gemini = remote("gemini-3.1-pro", ApiProtocol::Google, 65_536);
17501 assert_eq!(reasoning_output_allowance(Some(&gemini), 2048, high), 2048);
17502 assert_eq!(reasoning_output_allowance(None, 2048, high), 2048);
17503
17504 let opus = remote("claude-opus-4-1", ApiProtocol::Anthropic, 32_000);
17506 assert_eq!(reasoning_output_allowance(Some(&opus), 2048, high), 18_432);
17507 let sol = remote("gpt-5.6-sol", ApiProtocol::OpenAiResponses, 128_000);
17508 assert_eq!(reasoning_output_allowance(Some(&sol), 2048, high), 18_432);
17509 let small_claude = remote("claude-sonnet-4-5", ApiProtocol::Anthropic, 8192);
17511 assert_eq!(
17512 reasoning_output_allowance(Some(&small_claude), 2048, high),
17513 8192
17514 );
17515 assert_eq!(reasoning_output_allowance(Some(&opus), 2048, 0), 2048);
17516 }
17517
17518 #[test]
17523 fn auto_thinking_budget_gates_code_complex_and_capability() {
17524 assert_eq!(auto_thinking_budget(true, false, true), Some(24_000));
17526 assert_eq!(auto_thinking_budget(false, true, true), Some(8_000));
17528 assert_eq!(auto_thinking_budget(true, true, true), Some(24_000));
17530 assert_eq!(auto_thinking_budget(true, true, false), None);
17532 assert_eq!(auto_thinking_budget(false, false, true), None);
17534 }
17535
17536 #[test]
17537 fn is_explicit_code_intent_keys_on_caller_intent_not_keyword_classifier() {
17538 use crate::intent::{IntentHint, TaskHint};
17539 let code = IntentHint {
17541 task: Some(TaskHint::Code),
17542 ..Default::default()
17543 };
17544 assert!(is_explicit_code_intent(Some(&code)));
17545 assert!(!is_explicit_code_intent(None));
17550 let reasoning = IntentHint {
17552 task: Some(TaskHint::Reasoning),
17553 ..Default::default()
17554 };
17555 assert!(!is_explicit_code_intent(Some(&reasoning)));
17556 let unset = IntentHint {
17558 task: None,
17559 ..Default::default()
17560 };
17561 assert!(!is_explicit_code_intent(Some(&unset)));
17562 }
17563
17564 #[test]
17565 fn strict_model_suppresses_the_local_last_resort_append() {
17566 assert!(should_append_local_last_resort(false, false));
17568 assert!(!should_append_local_last_resort(false, true));
17571 assert!(!should_append_local_last_resort(true, false));
17574 assert!(!should_append_local_last_resort(true, true));
17575 }
17576
17577 #[test]
17578 fn last_resort_fallback_never_returns_an_unrunnable_apple_foundation() {
17579 let tmp = TempDir::new().unwrap();
17594 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
17595 if let Some(name) = engine.first_installed_local_model(false) {
17596 let runnable = engine
17597 .unified_registry()
17598 .find_by_name(&name)
17599 .map(|s| !s.is_foundation_models() || s.available)
17600 .unwrap_or(false);
17601 assert!(
17602 runnable,
17603 "last-resort fallback returned a non-runnable model: {name}"
17604 );
17605 }
17606 }
17607
17608 struct ScopedEnv {
17611 name: &'static str,
17612 previous: Option<std::ffi::OsString>,
17613 }
17614
17615 impl ScopedEnv {
17616 fn set(name: &'static str, value: Option<&Path>) -> Self {
17617 let previous = std::env::var_os(name);
17618 unsafe {
17621 match value {
17622 Some(value) => std::env::set_var(name, value),
17623 None => std::env::remove_var(name),
17624 }
17625 }
17626 Self { name, previous }
17627 }
17628 }
17629
17630 impl Drop for ScopedEnv {
17631 fn drop(&mut self) {
17632 unsafe {
17635 match self.previous.take() {
17636 Some(value) => std::env::set_var(self.name, value),
17637 None => std::env::remove_var(self.name),
17638 }
17639 }
17640 }
17641 }
17642
17643 #[test]
17650 fn last_resort_fallback_requires_a_car_install() {
17651 let _environment = crate::openrouter::test_environment_scope();
17652 let tmp = TempDir::new().unwrap();
17653 let hf_home = tmp.path().join("hf-home");
17654 let _hf_home = ScopedEnv::set("HF_HOME", Some(&hf_home));
17655 let _hub = ScopedEnv::set("HF_HUB_CACHE", None);
17656 let _legacy_hub = ScopedEnv::set("HUGGINGFACE_HUB_CACHE", None);
17657 let models_dir = tmp.path().join("models");
17658 std::fs::create_dir_all(&models_dir).unwrap();
17659 let mut engine = InferenceEngine::new(InferenceConfig {
17660 state_root: tmp.path().join("state"),
17661 models_dir: models_dir.clone(),
17662 ..InferenceConfig::default()
17663 });
17664
17665 let catalog = engine.unified_registry();
17668 let foundation = catalog
17669 .all()
17670 .find(|s| s.is_foundation_models())
17671 .expect("the built-in catalog ships the FoundationModels row")
17672 .clone();
17673 assert!(engine.installed_through_car(&foundation.id));
17674 let template = catalog
17675 .all()
17676 .find(|s| {
17677 matches!(s.source, ModelSource::Mlx { .. })
17678 && s.has_capability(ModelCapability::Generate)
17679 && s.has_capability(ModelCapability::ToolUse)
17680 })
17681 .expect("the built-in catalog ships a tool-capable MLX text row")
17682 .clone();
17683 drop(catalog);
17684
17685 let row = |id: &str, size_mb: u64, tools: bool| {
17686 let mut schema = template.clone();
17687 schema.id = id.to_string();
17688 schema.name = id.replace('/', "-");
17689 schema.source = ModelSource::Mlx {
17690 hf_repo: format!("test-org/{}", schema.name),
17691 hf_weight_file: None,
17692 };
17693 schema.capabilities = if tools {
17694 vec![ModelCapability::Generate, ModelCapability::ToolUse]
17695 } else {
17696 vec![ModelCapability::Generate]
17697 };
17698 schema.cost.size_mb = Some(size_mb);
17699 schema.cost.ram_mb = Some(size_mb);
17700 schema
17701 };
17702 let write_mlx = |dir: &Path| {
17703 std::fs::create_dir_all(dir).unwrap();
17704 std::fs::write(dir.join("config.json"), br#"{"model_type":"qwen3"}"#).unwrap();
17705 std::fs::write(dir.join("model.safetensors"), b"weights").unwrap();
17706 std::fs::write(dir.join("tokenizer.json"), b"{}").unwrap();
17707 };
17708 let cached = row("test/cached-tools", 40, true);
17710 let snapshot = hf_home
17711 .join("hub")
17712 .join(format!("models--test-org--{}", cached.name))
17713 .join("snapshots/fixture");
17714 write_mlx(&snapshot);
17715 std::fs::create_dir_all(snapshot.parent().unwrap().parent().unwrap().join("refs")).unwrap();
17716 std::fs::write(
17717 snapshot
17718 .parent()
17719 .unwrap()
17720 .parent()
17721 .unwrap()
17722 .join("refs/main"),
17723 b"fixture",
17724 )
17725 .unwrap();
17726 let installed_big = row("test/installed-tools-big", 30, true);
17729 let installed_small = row("test/installed-tools-small", 10, true);
17730 let installed_text = row("test/installed-text-only", 35, false);
17731 for schema in [&installed_big, &installed_small, &installed_text] {
17732 write_mlx(&models_dir.join(&schema.name));
17733 }
17734 let mut registry = UnifiedRegistry::new_empty(models_dir.clone());
17735 for schema in [&cached, &installed_big, &installed_small, &installed_text] {
17736 registry.register(schema.clone());
17737 }
17738 *engine.unified_registry_mut() = registry;
17739
17740 assert_eq!(
17741 engine.unified_registry().ready_without_download(&cached.id),
17742 Some(true),
17743 "fixture: the cached snapshot is complete"
17744 );
17745 assert!(
17746 !engine.installed_through_car(&cached.id),
17747 "a shared-cache snapshot must not count as installed"
17748 );
17749 assert!(engine.installed_through_car(&installed_small.id));
17750 assert_eq!(
17751 engine.first_installed_local_model(true).as_deref(),
17752 Some(installed_big.name.as_str()),
17753 "a tools turn falls back to the largest installed tool-capable model"
17754 );
17755 assert_eq!(
17756 engine.first_installed_local_model(false).as_deref(),
17757 Some(installed_big.name.as_str()),
17758 "tool-capable ranks ahead of a larger text-only model"
17759 );
17760 assert_eq!(
17761 engine.cache_only_local_models(true),
17762 vec![cached.id.clone()]
17763 );
17764
17765 let hinted = append_adopt_hint(
17767 InferenceError::Transient {
17768 status: Some(401),
17769 message: "provider rejected the API key".into(),
17770 },
17771 &engine.cache_only_local_models(true),
17772 );
17773 assert!(
17774 matches!(&hinted, InferenceError::Transient { message, .. }
17775 if message.contains("`car models adopt test/cached-tools`")),
17776 "the hint keeps the error's type and names the command: {hinted:?}"
17777 );
17778 }
17779
17780 #[tokio::test]
17785 async fn a_last_resort_stream_leads_with_its_substitution() {
17786 let hops = vec![FallbackFrom {
17787 candidate: "anthropic/claude-opus-5:latest".into(),
17788 reason: FallbackReason::CredentialRejected,
17789 }];
17790 let source = || {
17791 let (tx, rx) = tokio::sync::mpsc::channel(8);
17792 tx.try_send(stream::StreamEvent::TextDelta("OK".into()))
17793 .unwrap();
17794 tx.try_send(stream::StreamEvent::Done {
17795 text: "OK".into(),
17796 tool_calls: vec![],
17797 })
17798 .unwrap();
17799 rx
17800 };
17801
17802 let mut rx = lead_with_last_resort_substitution(source(), "mlx/qwen3-4b:4bit", true, &hops);
17803 match rx.recv().await {
17804 Some(stream::StreamEvent::Substituted(substitution)) => assert_eq!(
17805 substitution,
17806 stream::StreamSubstitution {
17807 model_id: "mlx/qwen3-4b:4bit".into(),
17808 substituted_from: Some("anthropic/claude-opus-5:latest".into()),
17809 substitution_reason: Some(FallbackReason::CredentialRejected),
17810 }
17811 ),
17812 other => panic!("expected the substitution first, got {other:?}"),
17813 }
17814 assert!(matches!(rx.recv().await, Some(stream::StreamEvent::TextDelta(t)) if t == "OK"));
17815 assert!(matches!(
17816 rx.recv().await,
17817 Some(stream::StreamEvent::Done { .. })
17818 ));
17819
17820 let mut ordinary =
17821 lead_with_last_resort_substitution(source(), "mlx/qwen3-4b:4bit", false, &hops);
17822 assert!(
17823 matches!(
17824 ordinary.recv().await,
17825 Some(stream::StreamEvent::TextDelta(_))
17826 ),
17827 "a routed local fallback is not a substitution"
17828 );
17829 }
17830
17831 #[test]
17832 fn empty_tool_catalog_is_no_tools() {
17833 let mut req = GenerateRequest {
17840 prompt: "p".into(),
17841 ..Default::default()
17842 };
17843 assert!(!InferenceEngine::request_has_tools(&req));
17844 req.tools = Some(vec![]);
17845 assert!(!InferenceEngine::request_has_tools(&req));
17846 req.tools = Some(vec![serde_json::json!({
17847 "name": "t", "description": "d", "parameters": {"type": "object"}
17848 })]);
17849 assert!(InferenceEngine::request_has_tools(&req));
17850 }
17851
17852 #[test]
17853 fn top_k_keeps_only_k_highest() {
17854 let mut probs = vec![0.1, 0.4, 0.2, 0.25, 0.05];
17856 InferenceEngine::apply_top_k_top_p(&mut probs, 2, 1.0);
17857 assert!(probs[0] == 0.0 && probs[2] == 0.0 && probs[4] == 0.0);
17859 assert!(probs[1] > 0.0 && probs[3] > 0.0);
17860 let sum: f32 = probs.iter().sum();
17861 assert!((sum - 1.0).abs() < 1e-5, "renormalized to 1.0, got {sum}");
17862 }
17863
17864 #[test]
17865 fn top_k_zero_is_a_noop() {
17866 let mut probs = vec![0.1, 0.4, 0.2, 0.3];
17867 let before = probs.clone();
17868 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 1.0);
17869 assert_eq!(probs, before);
17870 }
17871
17872 #[test]
17873 fn top_p_nucleus_truncates_tail() {
17874 let mut probs = vec![0.6, 0.3, 0.07, 0.03];
17875 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 0.9);
17876 assert!(probs[2] == 0.0 && probs[3] == 0.0);
17878 assert!(probs[0] > 0.0 && probs[1] > 0.0);
17879 }
17880
17881 #[test]
17882 fn truncate_at_stop_excludes_stop_sequence() {
17883 let stops = vec!["<|end|>".to_string(), "STOP".to_string()];
17884 assert_eq!(
17885 tasks::generate::truncate_at_stop("hello world<|end|>extra", &stops),
17886 "hello world"
17887 );
17888 assert_eq!(
17890 tasks::generate::truncate_at_stop("aSTOPb<|end|>c", &stops),
17891 "a"
17892 );
17893 assert_eq!(
17895 tasks::generate::truncate_at_stop("clean output", &stops),
17896 "clean output"
17897 );
17898 assert_eq!(
17900 tasks::generate::truncate_at_stop("text", &["".to_string()]),
17901 "text"
17902 );
17903 }
17904
17905 #[test]
17906 fn no_backend_hint_fires_on_missing_backend_phrases() {
17907 for phrase in [
17909 "no credential for proprietary provider 'parslee'",
17910 "model not found",
17911 "no models available",
17912 "model declares ModelSource::Delegated but no inference runner is registered",
17913 ] {
17914 let hint = no_backend_recovery_hint(phrase)
17915 .unwrap_or_else(|| panic!("expected a hint for {phrase:?}"));
17916 assert!(hint.contains("car models pull"));
17917 assert!(hint.contains("car auth login"));
17920 assert!(hint.contains(phrase));
17922 }
17923 }
17924
17925 #[test]
17926 fn no_backend_hint_opens_with_the_exported_marker() {
17927 let hint = no_backend_recovery_hint(
17931 "model declares ModelSource::Delegated but no inference runner is registered",
17932 )
17933 .expect("a delegated model with no runner is a no-backend case");
17934 assert!(
17935 hint.starts_with(NO_BACKEND_RECOVERY_MARKER),
17936 "hint must open with the exported marker: {hint}"
17937 );
17938 assert!(hint.contains(NO_BACKEND_RECOVERY_MARKER), "{hint}");
17939 }
17940
17941 #[test]
17942 fn route_failure_context_keeps_the_no_backend_marker_behind_the_credential_summary() {
17943 let credential = parslee_signed_out_route_failure();
17947 let underlying = InferenceError::InferenceFailed(
17948 "model declares ModelSource::Delegated but no inference runner is registered".into(),
17949 );
17950 let augmented = apply_route_failure_context(underlying, Some(&credential));
17951 let InferenceError::InferenceFailed(message) = &augmented else {
17952 panic!("a no-backend exhaustion stays InferenceFailed: {augmented:?}");
17953 };
17954 assert!(
17955 message.starts_with(&credential.summary),
17956 "credential context is prepended: {message}"
17957 );
17958 assert!(
17959 message.contains(NO_BACKEND_RECOVERY_MARKER),
17960 "the marker survives the prepend: {message}"
17961 );
17962 assert!(
17963 !message.starts_with(NO_BACKEND_RECOVERY_MARKER),
17964 "the marker is mid-message here, which is why consumers use contains: {message}"
17965 );
17966 }
17967
17968 #[test]
17976 fn credential_failure_is_matchable_as_data() {
17977 let expired = InferenceError::CredentialUnavailable {
17978 provider: "parslee".into(),
17979 model: "parslee/reasoning".into(),
17980 reason: CredentialFailure::Expired {
17981 expires_at: 1_754_257_929,
17982 },
17983 detail: "the Parslee token expired at unix 1754257929 and could not be refreshed"
17984 .into(),
17985 };
17986 let InferenceError::CredentialUnavailable { reason, .. } = &expired else {
17987 panic!("expected CredentialUnavailable");
17988 };
17989 assert_eq!(
17990 *reason,
17991 CredentialFailure::Expired {
17992 expires_at: 1_754_257_929
17993 },
17994 "a consumer must be able to branch on the expiry without parsing prose"
17995 );
17996 assert_ne!(
18000 CredentialFailure::SignedOut,
18001 CredentialFailure::StoreUnreadable
18002 );
18003 assert_ne!(
18004 CredentialFailure::SignedOut,
18005 CredentialFailure::Expired { expires_at: 0 }
18006 );
18007 assert_ne!(
18008 CredentialFailure::StoreUnreadable,
18009 CredentialFailure::RaceRetryable
18010 );
18011 }
18012
18013 #[test]
18023 fn typed_credential_error_still_satisfies_the_substring_classifiers() {
18024 for reason in [
18025 CredentialFailure::Expired { expires_at: 1 },
18026 CredentialFailure::SignedOut,
18027 CredentialFailure::StoreUnreadable,
18028 CredentialFailure::RaceRetryable,
18029 CredentialFailure::EnvVarMissing {
18030 env_var: "OPENAI_API_KEY".into(),
18031 },
18032 ] {
18033 let rendered = InferenceError::CredentialUnavailable {
18034 provider: "parslee".into(),
18035 model: "parslee/reasoning".into(),
18036 reason: reason.clone(),
18037 detail: "detail text".into(),
18038 }
18039 .to_string();
18040 assert!(
18042 rendered
18043 .to_ascii_lowercase()
18044 .contains("no credential for proprietary"),
18045 "classifier substring lost for {reason:?}: {rendered}"
18046 );
18047 assert!(rendered.contains("parslee/reasoning"), "{rendered}");
18049 assert!(rendered.contains("detail text"), "{rendered}");
18051 }
18052 }
18053
18054 #[test]
18055 fn auth_expired_hint_fires_on_auth_rejection_but_not_transient() {
18056 for phrase in [
18058 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
18059 "invalid_grant: The refresh token is invalid or expired",
18060 "token expired",
18061 ] {
18062 let hint = auth_expired_recovery_hint(phrase)
18063 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
18064 assert!(hint.contains("car auth login"));
18065 assert!(hint.contains(phrase));
18066 }
18067 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
18069 assert!(auth_expired_recovery_hint("request timed out").is_none());
18070 }
18071
18072 #[test]
18090 fn auth_expired_hint_never_carries_the_no_backend_marker() {
18091 for phrase in [
18094 "Parslee org lookup failed: HTTP 401",
18095 "Authentication required",
18096 "HTTP 401 Unauthorized",
18097 "invalid_grant: the refresh token is invalid or expired",
18098 "token expired",
18099 ] {
18100 let hint = auth_expired_recovery_hint(phrase)
18101 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
18102 assert!(
18103 !hint.contains(NO_BACKEND_RECOVERY_MARKER),
18104 "the auth-expired hint now carries the no-backend marker, so \
18105 car infer would auto-pull for {phrase:?}: {hint}"
18106 );
18107 assert!(hint.contains("car models pull"), "{hint}");
18111 }
18112 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
18115 }
18116
18117 #[test]
18122 fn auth_rejection_classifier_matches_the_real_expired_token_error() {
18123 assert!(is_auth_rejection_message(
18124 "inference failed: Parslee org lookup failed: HTTP 401 Unauthorized: \
18125 Authentication required"
18126 ));
18127 for transient in [
18130 "API returned 503: service unavailable",
18131 "request timed out",
18132 "connection reset by peer",
18133 ] {
18134 assert!(
18135 !is_auth_rejection_message(transient),
18136 "transient wrongly classified as an auth rejection: {transient:?}"
18137 );
18138 }
18139 }
18140
18141 #[test]
18146 fn a_403_is_access_denied_not_a_sign_in_failure() {
18147 use reqwest::StatusCode;
18148 let org_lookup_403 = format!(
18149 "inference failed: Parslee org lookup failed: HTTP {}: no access to this organization",
18150 StatusCode::FORBIDDEN
18151 );
18152 for denied in [
18153 org_lookup_403.as_str(),
18154 "API returned 403 Forbidden: model not enabled for this key",
18155 "HTTP 403 Forbidden",
18156 "HTTP 403: forbidden",
18157 ] {
18158 assert!(!is_auth_rejection_message(denied), "{denied}");
18159 assert!(!is_auth_failure_message(denied), "{denied}");
18160 assert!(auth_expired_recovery_hint(denied).is_none(), "{denied}");
18161 let mut slot = None;
18162 record_auth_dead_lane(&mut slot, "parslee/advisor", denied);
18163 assert_eq!(slot, None, "{denied}");
18164 }
18165 assert_eq!(
18166 classify_fallback_reason(&InferenceError::InferenceFailed(org_lookup_403.clone())),
18167 FallbackReason::Failed
18168 );
18169
18170 for provider in ["parslee", "openai"] {
18173 let summary = route_credential_failure(
18174 &format!("{provider}/model"),
18175 &InferenceError::ProviderAccount {
18176 provider: provider.into(),
18177 status: 403,
18178 message: "provider rejected the API key — check the configured credential"
18179 .into(),
18180 },
18181 true,
18182 )
18183 .expect("a refused account still names its cause");
18184 assert!(summary.contains("403"), "{summary}");
18185 assert!(!summary.contains("auth login"), "{summary}");
18186 assert!(!is_auth_failure_message(&summary), "{summary}");
18187 }
18188
18189 let org_lookup_401 = format!(
18191 "Parslee org lookup failed: HTTP {}: Authentication required",
18192 StatusCode::UNAUTHORIZED
18193 );
18194 assert!(is_auth_rejection_message(&org_lookup_401));
18195 let summary = route_credential_failure(
18196 "parslee/advisor",
18197 &InferenceError::ProviderAccount {
18198 provider: "parslee".into(),
18199 status: 401,
18200 message: "provider rejected the API key — check the configured credential".into(),
18201 },
18202 true,
18203 )
18204 .unwrap();
18205 assert!(summary.contains("car auth login"), "{summary}");
18206 }
18207
18208 #[test]
18217 fn a_rate_limited_or_failing_parslee_lookup_is_not_a_sign_in_failure() {
18218 use reqwest::StatusCode;
18219 let org_lookup = |status: StatusCode, body: &str| {
18220 format!("inference failed: Parslee org lookup failed: HTTP {status}: {body}")
18221 };
18222 for not_auth in [
18223 org_lookup(StatusCode::TOO_MANY_REQUESTS, "slow down"),
18224 org_lookup(StatusCode::INTERNAL_SERVER_ERROR, "boom"),
18225 org_lookup(StatusCode::BAD_GATEWAY, ""),
18226 org_lookup(StatusCode::SERVICE_UNAVAILABLE, "Authentication required"),
18227 org_lookup(
18230 StatusCode::INTERNAL_SERVER_ERROR,
18231 "upstream identity call returned HTTP 401",
18232 ),
18233 "API returned 400 Bad Request: your last request 401'd upstream and was unauthorized"
18235 .to_string(),
18236 "API returned 503 Service Unavailable: token expired on the upstream pool".to_string(),
18237 ] {
18238 assert!(!is_auth_rejection_message(¬_auth), "{not_auth}");
18239 assert!(!is_auth_failure_message(¬_auth), "{not_auth}");
18240 assert!(
18241 auth_expired_recovery_hint(¬_auth).is_none(),
18242 "{not_auth}"
18243 );
18244 assert_eq!(
18245 route_credential_failure(
18246 "parslee/advisor",
18247 &InferenceError::InferenceFailed(not_auth.clone()),
18248 true
18249 ),
18250 None,
18251 "{not_auth}"
18252 );
18253 }
18254 for auth in [
18257 org_lookup(StatusCode::UNAUTHORIZED, "Authentication required"),
18258 "API returned 401 Unauthorized: bad key".to_string(),
18259 ] {
18260 assert!(is_auth_rejection_message(&auth), "{auth}");
18261 assert!(is_auth_failure_message(&auth), "{auth}");
18262 }
18263 }
18264
18265 #[test]
18273 fn typed_errors_classify_by_their_structure_not_their_prose() {
18274 use FallbackReason as R;
18275 let cases: Vec<(InferenceError, R)> = vec![
18276 (
18279 InferenceError::ProviderAccount {
18280 provider: "openai".into(),
18281 status: 401,
18282 message: "provider rejected the API key — check the configured credential"
18283 .into(),
18284 },
18285 R::CredentialRejected,
18286 ),
18287 (
18289 InferenceError::ProviderAccount {
18290 provider: "openrouter".into(),
18291 status: 402,
18292 message: "OpenRouter account is out of credits".into(),
18293 },
18294 R::QuotaExhausted,
18296 ),
18297 (
18298 InferenceError::CredentialUnavailable {
18299 provider: "parslee".into(),
18300 model: "parslee/reasoning".into(),
18301 reason: CredentialFailure::Expired { expires_at: 0 },
18302 detail: "session expired".into(),
18303 },
18304 R::CredentialRejected,
18305 ),
18306 (
18307 InferenceError::CredentialUnavailable {
18308 provider: "openai".into(),
18309 model: "openai/gpt-5.6".into(),
18310 reason: CredentialFailure::EnvVarMissing {
18311 env_var: "OPENAI_API_KEY".into(),
18312 },
18313 detail: "not set".into(),
18314 },
18315 R::CredentialAbsent,
18316 ),
18317 (
18320 InferenceError::CredentialUnavailable {
18321 provider: "parslee".into(),
18322 model: "parslee/reasoning".into(),
18323 reason: CredentialFailure::StoreUnreadable,
18324 detail: "the bounded Keychain helper timed out".into(),
18325 },
18326 R::Failed,
18327 ),
18328 (
18329 InferenceError::Transient {
18330 status: Some(429),
18331 message: "slow down".into(),
18332 },
18333 R::RateLimited,
18334 ),
18335 (
18340 InferenceError::Transient {
18341 status: None,
18342 message: "connection reset by peer".into(),
18343 },
18344 R::Failed,
18345 ),
18346 (
18347 InferenceError::Transient {
18348 status: Some(503),
18349 message: "upstream down".into(),
18350 },
18351 R::Failed,
18352 ),
18353 ];
18354 for (err, want) in cases {
18355 assert_eq!(classify_fallback_reason(&err), want, "{err}");
18356 }
18357 }
18358
18359 #[test]
18367 fn a_quoted_status_in_a_provider_body_does_not_decide_the_bucket() {
18368 use FallbackReason as R;
18369 for (msg, want) in [
18370 (
18371 "API returned 400 Bad Request: your last request 401'd upstream and was unauthorized",
18372 R::Failed,
18373 ),
18374 ("API returned 400 Bad Request: timeout param invalid", R::Failed),
18375 ("API returned 429 Too Many Requests: slow down", R::RateLimited),
18376 ("API returned 401 Unauthorized: bad key", R::CredentialRejected),
18377 ] {
18378 assert_eq!(
18379 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
18380 want,
18381 "{msg}"
18382 );
18383 }
18384 }
18385
18386 #[test]
18390 fn a_parslee_org_lookup_failure_is_classified_by_its_status() {
18391 use FallbackReason as R;
18392 for (msg, want) in [
18393 (
18394 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
18395 R::CredentialRejected,
18396 ),
18397 (
18398 "Parslee org lookup failed: HTTP 429 Too Many Requests: slow down",
18399 R::RateLimited,
18400 ),
18401 (
18402 "Parslee org lookup failed: HTTP 500 Internal Server Error: boom",
18403 R::Failed,
18404 ),
18405 (
18406 "Parslee org lookup failed: HTTP 500 Internal Server Error: \
18407 upstream identity call returned HTTP 401",
18408 R::Failed,
18409 ),
18410 (
18411 "Parslee org lookup failed: HTTP 502 Bad Gateway: retry after HTTP 429",
18412 R::Failed,
18413 ),
18414 ] {
18415 assert_eq!(
18416 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
18417 want,
18418 "{msg}"
18419 );
18420 }
18421 }
18422
18423 #[test]
18427 fn every_skipped_lane_is_recorded_in_order() {
18428 let mut hops = Vec::new();
18429 for (cand, err) in [
18430 (
18431 "lane-one",
18432 InferenceError::Transient {
18433 status: Some(429),
18434 message: "x".into(),
18435 },
18436 ),
18437 (
18440 "lane-two",
18441 InferenceError::Transient {
18442 status: None,
18443 message: "connection refused".into(),
18444 },
18445 ),
18446 (
18447 "lane-three",
18448 InferenceError::ProviderAccount {
18449 provider: "openai".into(),
18450 status: 401,
18451 message: "rejected".into(),
18452 },
18453 ),
18454 ] {
18455 record_fallback_from(&mut hops, cand, &err);
18456 }
18457 assert_eq!(
18458 hops.iter()
18459 .map(|h| h.candidate.as_str())
18460 .collect::<Vec<_>>(),
18461 ["lane-one", "lane-two", "lane-three"]
18462 );
18463 assert_eq!(
18464 hops.iter().map(|h| h.reason).collect::<Vec<_>>(),
18465 [
18466 FallbackReason::RateLimited,
18467 FallbackReason::Failed,
18468 FallbackReason::CredentialRejected
18469 ]
18470 );
18471 }
18472
18473 #[test]
18481 fn a_refused_api_key_is_journaled_but_does_not_claim_sign_in_fixes_it() {
18482 let refused = InferenceError::ProviderAccount {
18483 provider: "openai".into(),
18484 status: 401,
18485 message: "provider rejected the API key — check the configured credential".into(),
18486 };
18487 let mut hops = Vec::new();
18488 record_fallback_from(&mut hops, "openai/gpt-5.6", &refused);
18489 assert_eq!(hops[0].reason, FallbackReason::CredentialRejected);
18490
18491 let mut auth = None;
18493 record_auth_dead_lane(&mut auth, "openai/gpt-5.6", &refused.to_string());
18494 assert_eq!(auth, None, "car auth login does not fix a bad OpenAI key");
18495 }
18496
18497 #[test]
18498 fn auth_dead_lane_records_first_rejected_candidate_only() {
18499 let mut slot: Option<String> = None;
18501 record_auth_dead_lane(
18502 &mut slot,
18503 "parslee/reasoning",
18504 "API returned 503: unavailable",
18505 );
18506 record_auth_dead_lane(&mut slot, "openai/gpt-5.6", "request timed out");
18507 assert_eq!(slot, None);
18508
18509 record_auth_dead_lane(
18511 &mut slot,
18512 "parslee/reasoning",
18513 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
18514 );
18515 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
18516
18517 record_auth_dead_lane(&mut slot, "anthropic/claude", "HTTP 403: forbidden");
18520 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
18521 }
18522
18523 #[test]
18524 fn configured_provider_with_expired_token_is_named_before_a_local_oom() {
18525 let expired = InferenceError::CredentialUnavailable {
18526 provider: "parslee".into(),
18527 model: "parslee/reasoning".into(),
18528 reason: CredentialFailure::Expired { expires_at: 42 },
18529 detail: "access token expired".into(),
18530 };
18531 let mut credential = None;
18532 record_route_credential_failure(&mut credential, "parslee/reasoning", &expired, false);
18533
18534 let error = apply_route_failure_context(
18535 InferenceError::InferenceFailed(
18536 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
18537 .into(),
18538 ),
18539 credential.as_ref(),
18540 )
18541 .to_string();
18542
18543 let auth_pos = error
18544 .find("Parslee login expired")
18545 .expect("expired login must be named");
18546 let remedy_pos = error
18547 .find("car auth login")
18548 .expect("credential remedy must be named");
18549 let oom_pos = error
18550 .find("9059 MB")
18551 .expect("fallback error must remain as secondary detail");
18552 assert!(auth_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
18553 assert!(is_auth_failure_message(&error), "{error}");
18554 }
18555
18556 #[test]
18557 fn absent_login_is_named_before_a_local_oom() {
18558 let failure = parslee_signed_out_route_failure();
18559 let error = apply_route_failure_context(
18560 InferenceError::InferenceFailed("local fallback needs 9059 MB".into()),
18561 Some(&failure),
18562 )
18563 .to_string();
18564
18565 let absent_pos = error
18566 .find("Parslee login is absent")
18567 .expect("missing login must be named");
18568 let remedy_pos = error
18569 .find("car auth login")
18570 .expect("credential remedy must be named");
18571 let oom_pos = error
18572 .find("9059 MB")
18573 .expect("fallback OOM must remain as secondary detail");
18574 assert!(absent_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
18575 }
18576
18577 #[test]
18578 fn genuine_local_oom_is_not_reclassified_as_auth() {
18579 let oom = InferenceError::InferenceFailed(
18580 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
18581 .into(),
18582 );
18583 let error = apply_route_failure_context(oom, None);
18584 assert!(matches!(error, InferenceError::InferenceFailed(ref message)
18585 if message.starts_with("This model needs about 9059 MB")));
18586 }
18587
18588 #[test]
18589 fn unconfigured_provider_is_not_surfaced_over_the_terminal_failure() {
18590 let mut credential = None;
18591 record_route_credential_failure(
18592 &mut credential,
18593 "openai/gpt-5.6",
18594 &InferenceError::CredentialUnavailable {
18595 provider: "openai".into(),
18596 model: "openai/gpt-5.6".into(),
18597 reason: CredentialFailure::EnvVarMissing {
18598 env_var: "OPENAI_API_KEY".into(),
18599 },
18600 detail: "set OPENAI_API_KEY".into(),
18601 },
18602 false,
18603 );
18604
18605 let terminal = InferenceError::InferenceFailed(
18606 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
18607 .into(),
18608 );
18609 let error = apply_route_failure_context(terminal, credential.as_ref()).to_string();
18610 assert!(
18611 credential.is_none(),
18612 "an unconfigured fallback is ambient noise"
18613 );
18614 assert!(error.contains("9059 MB"), "{error}");
18615 assert!(!error.contains("OPENAI_API_KEY"), "{error}");
18616 assert!(!is_auth_failure_message(&error), "{error}");
18617 }
18618
18619 fn workspace_required() -> InferenceError {
18623 InferenceError::WorkspaceRequired {
18624 provider: "Parslee".into(),
18625 detail: "finish setting up at https://parslee.ai, then try again".into(),
18626 }
18627 }
18628
18629 #[test]
18638 fn workspace_required_reads_as_configuration_not_sign_in() {
18639 let rendered = workspace_required().to_string();
18640 for marker in AUTH_FAILURE_MESSAGE_MARKERS {
18641 assert!(
18642 !rendered
18643 .to_ascii_lowercase()
18644 .contains(&marker.to_ascii_lowercase()),
18645 "`{marker}` must not appear in the no-workspace text: {rendered}"
18646 );
18647 }
18648 assert!(!is_auth_failure_message(&rendered), "{rendered}");
18649 assert!(!is_auth_rejection_message(&rendered), "{rendered}");
18650 assert!(rendered.contains("https://parslee.ai"), "{rendered}");
18651 }
18652
18653 #[test]
18660 fn workspace_required_survives_a_mixed_chain() {
18661 let mut credential = None;
18662 record_route_credential_failure(
18663 &mut credential,
18664 "openai/gpt-5.6",
18665 &InferenceError::CredentialUnavailable {
18666 provider: "openai".into(),
18667 model: "openai/gpt-5.6".into(),
18668 reason: CredentialFailure::SignedOut,
18669 detail: "no account is signed in".into(),
18670 },
18671 true,
18672 );
18673 assert!(
18674 credential.is_some(),
18675 "the fixture must actually record a credential failure"
18676 );
18677
18678 let final_error = apply_route_failure_context(workspace_required(), credential.as_ref());
18679 assert!(
18680 matches!(final_error, InferenceError::WorkspaceRequired { .. }),
18681 "the type must survive the route context: {final_error}"
18682 );
18683 assert!(!is_auth_failure_message(&final_error.to_string()));
18684 assert!(
18685 error_ends_fallback_chain(&final_error),
18686 "no later candidate on that account can succeed"
18687 );
18688 }
18689
18690 #[test]
18697 fn repeated_workspace_required_leaves_model_health_untouched() {
18698 assert!(
18699 !error_counts_against_circuit_breaker(&workspace_required()),
18700 "the breaker must not count a workspace gap"
18701 );
18702
18703 let mut tracker = OutcomeTracker::new();
18704 for _ in 0..5 {
18705 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
18706 record_dispatch_failure(&mut tracker, &trace, &workspace_required());
18707 }
18708 let failed = tracker
18709 .profile("parslee/advisor")
18710 .map(|p| p.fail_count)
18711 .unwrap_or(0);
18712 assert_eq!(
18713 failed, 0,
18714 "five workspace refusals must not bench the model"
18715 );
18716
18717 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
18718 record_dispatch_failure(
18719 &mut tracker,
18720 &trace,
18721 &InferenceError::InferenceFailed("decoder fell over".into()),
18722 );
18723 assert_eq!(
18724 tracker
18725 .profile("parslee/advisor")
18726 .map(|p| p.fail_count)
18727 .unwrap_or(0),
18728 1,
18729 "a genuine failure must still count, or the test above proves nothing"
18730 );
18731 }
18732
18733 #[test]
18734 fn explicitly_requested_missing_credential_uses_the_shared_auth_table() {
18735 let mut credential = None;
18736 record_route_credential_failure(
18737 &mut credential,
18738 "openai/gpt-5.6",
18739 &InferenceError::CredentialUnavailable {
18740 provider: "openai".into(),
18741 model: "openai/gpt-5.6".into(),
18742 reason: CredentialFailure::EnvVarMissing {
18743 env_var: "OPENAI_API_KEY".into(),
18744 },
18745 detail: "set OPENAI_API_KEY".into(),
18746 },
18747 true,
18748 );
18749 let summary = credential
18750 .expect("an explicitly requested provider must surface its missing key")
18751 .summary;
18752 assert!(summary.contains(AUTH_ENV_MISSING_MARKER), "{summary}");
18753 assert!(is_auth_failure_message(&summary), "{summary}");
18754 }
18755
18756 #[test]
18757 fn latest_actionable_credential_failure_wins() {
18758 let mut credential = None;
18759 record_route_credential_failure(
18760 &mut credential,
18761 "parslee/reasoning",
18762 &InferenceError::ProviderAccount {
18763 provider: "parslee".into(),
18764 status: 401,
18765 message: "Unauthorized".into(),
18766 },
18767 false,
18768 );
18769 record_route_credential_failure(
18770 &mut credential,
18771 "anthropic/claude",
18772 &InferenceError::CredentialUnavailable {
18773 provider: "anthropic".into(),
18774 model: "anthropic/claude".into(),
18775 reason: CredentialFailure::Expired { expires_at: 43 },
18776 detail: "configured token expired".into(),
18777 },
18778 false,
18779 );
18780
18781 assert_eq!(
18782 credential
18783 .expect("latest actionable credential cause must be retained")
18784 .summary,
18785 "anthropic login expired for `anthropic/claude` — run `car auth login`"
18786 );
18787 }
18788
18789 #[test]
18790 fn store_unreadable_summary_uses_the_shared_auth_table() {
18791 let summary = route_credential_failure(
18792 "parslee/reasoning",
18793 &InferenceError::CredentialUnavailable {
18794 provider: "parslee".into(),
18795 model: "parslee/reasoning".into(),
18796 reason: CredentialFailure::StoreUnreadable,
18797 detail: "keychain helper timed out".into(),
18798 },
18799 false,
18800 )
18801 .expect("an unreadable configured credential store is actionable");
18802 assert!(summary.contains(AUTH_STORE_UNREADABLE_MARKER), "{summary}");
18803 assert!(is_auth_failure_message(&summary), "{summary}");
18804 }
18805
18806 fn chain_gate_fixture_schema(id: &str, provider: &str, source: ModelSource) -> ModelSchema {
18807 ModelSchema {
18808 id: id.into(),
18809 name: id.into(),
18810 provider: provider.into(),
18811 family: "test".into(),
18812 version: "1".into(),
18813 capabilities: vec![ModelCapability::Generate],
18814 context_length: 32_768,
18815 max_output_tokens: Some(4_096),
18816 param_count: String::new(),
18817 quantization: None,
18818 performance: Default::default(),
18819 cost: Default::default(),
18820 source,
18821 tags: vec![],
18822 supported_params: vec![],
18823 public_benchmarks: vec![],
18824 trust_tier: TrustTier::Community,
18825 deprecated: false,
18826 available: true,
18827 weights_ready: true,
18828 }
18829 }
18830
18831 #[test]
18832 fn signed_out_pre_seed_is_gated_on_a_parslee_route_in_the_chain() {
18833 let local = chain_gate_fixture_schema(
18834 "qwen/qwen3-4b:q4_k_m",
18835 "qwen",
18836 ModelSource::Mlx {
18837 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
18838 hf_weight_file: None,
18839 },
18840 );
18841 let ollama = chain_gate_fixture_schema(
18842 "ollama/llama3",
18843 "ollama",
18844 ModelSource::Ollama {
18845 model_tag: "llama3".into(),
18846 host: "http://localhost:11434".into(),
18847 },
18848 );
18849 let parslee = chain_gate_fixture_schema(
18850 "parslee/reasoning",
18851 "parslee",
18852 ModelSource::Proprietary {
18853 provider: "parslee".into(),
18854 endpoint: "https://api.parslee.ai".into(),
18855 auth: ProprietaryAuth::OAuth2Pkce {
18856 authority: "https://login.example".into(),
18857 client_id: "client".into(),
18858 scopes: vec![],
18859 },
18860 protocol: Default::default(),
18861 },
18862 );
18863 let cloud = chain_gate_fixture_schema(
18864 "openai/gpt-5.6",
18865 "openai",
18866 ModelSource::RemoteApi {
18867 endpoint: "https://api.openai.com".into(),
18868 api_key_env: "OPENAI_API_KEY".into(),
18869 api_key_envs: vec![],
18870 api_version: None,
18871 protocol: schema::ApiProtocol::OpenAiCompat,
18872 },
18873 );
18874 let schemas: std::collections::HashMap<&str, &ModelSchema> = [
18875 ("qwen/qwen3-4b:q4_k_m", &local),
18876 ("ollama/llama3", &ollama),
18877 ("parslee/reasoning", &parslee),
18878 ("openai/gpt-5.6", &cloud),
18879 ]
18880 .into_iter()
18881 .collect();
18882 let resolve = |m: &str| schemas.get(m).copied();
18883
18884 let local_only = vec![
18886 "qwen/qwen3-4b:q4_k_m".to_string(),
18887 "ollama/llama3".to_string(),
18888 ];
18889 assert!(!chain_includes_parslee_route(resolve, &local_only));
18890
18891 let with_parslee = vec![
18893 "parslee/reasoning".to_string(),
18894 "qwen/qwen3-4b:q4_k_m".to_string(),
18895 ];
18896 assert!(chain_includes_parslee_route(resolve, &with_parslee));
18897 let with_cloud = vec![
18898 "openai/gpt-5.6".to_string(),
18899 "qwen/qwen3-4b:q4_k_m".to_string(),
18900 ];
18901 assert!(!chain_includes_parslee_route(resolve, &with_cloud));
18905 let mut credential = chain_includes_parslee_route(resolve, &with_cloud)
18906 .then(parslee_signed_out_route_failure);
18907 let outage = InferenceError::Transient {
18908 status: Some(500),
18909 message: "OpenAI HTTP 500 Internal Server Error".into(),
18910 };
18911 record_route_credential_failure(&mut credential, &with_cloud[0], &outage, false);
18912 let terminal = apply_route_failure_context(outage, credential.as_ref());
18913 assert!(matches!(terminal, InferenceError::Transient { .. }));
18914 assert!(!is_auth_failure_message(&terminal.to_string()));
18915 let oom = InferenceError::InferenceFailed("local model out of memory".into());
18916 record_route_credential_failure(&mut credential, &with_cloud[1], &oom, false);
18917 let terminal = apply_route_failure_context(oom, credential.as_ref()).to_string();
18918 assert!(!terminal.contains("Parslee"), "{terminal}");
18919 assert!(!is_auth_failure_message(&terminal), "{terminal}");
18920
18921 record_route_credential_failure(
18923 &mut credential,
18924 &with_cloud[0],
18925 &InferenceError::ProviderAccount {
18926 provider: "openai".into(),
18927 status: 401,
18928 message: "invalid API key".into(),
18929 },
18930 false,
18931 );
18932 let terminal = apply_route_failure_context(
18933 InferenceError::InferenceFailed("local model out of memory".into()),
18934 credential.as_ref(),
18935 )
18936 .to_string();
18937 assert!(terminal.contains("openai"), "{terminal}");
18938 assert!(!terminal.contains("Parslee"), "{terminal}");
18939 assert!(is_auth_failure_message(&terminal), "{terminal}");
18940
18941 let unknown = vec!["missing/model".to_string()];
18943 assert!(!chain_includes_parslee_route(resolve, &unknown));
18944 }
18945
18946 #[test]
18950 fn local_only_terminal_failures_are_not_relabeled_as_auth() {
18951 let cases = [
18952 InferenceError::InferenceFailed(
18954 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
18955 .into(),
18956 ),
18957 InferenceError::Transient {
18959 status: Some(500),
18960 message: "HTTP 500 Internal Server Error".into(),
18961 },
18962 InferenceError::InferenceFailed("llama runner process has terminated".into()),
18964 ];
18965 for case in cases {
18966 let error = apply_route_failure_context(case, None).to_string();
18967 assert!(!error.contains("Parslee login is absent"), "{error}");
18968 assert!(!is_auth_failure_message(&error), "{error}");
18969 }
18970
18971 let error = apply_route_failure_context(
18975 InferenceError::ModelNotFound("qwen/qwen3-4b:q4_k_m".into()),
18976 None,
18977 )
18978 .to_string();
18979 assert!(!error.contains("Parslee login is absent"), "{error}");
18980 assert!(error.contains("car models pull"), "{error}");
18981 }
18982
18983 #[test]
18986 fn credential_context_preserves_the_typed_terminal_variant() {
18987 let credential = parslee_signed_out_route_failure();
18988
18989 let transient = apply_route_failure_context(
18990 InferenceError::Transient {
18991 status: Some(500),
18992 message: "HTTP 500 Internal Server Error".into(),
18993 },
18994 Some(&credential),
18995 );
18996 match &transient {
18997 InferenceError::Transient { status, message } => {
18998 assert_eq!(*status, Some(500));
18999 assert!(message.starts_with("Parslee login is absent"), "{message}");
19000 assert!(message.contains("HTTP 500"), "{message}");
19001 }
19002 other => panic!("Transient must stay Transient, got {other:?}"),
19003 }
19004
19005 let account = apply_route_failure_context(
19006 InferenceError::ProviderAccount {
19007 provider: "openai".into(),
19008 status: 402,
19009 message: "insufficient credits".into(),
19010 },
19011 Some(&credential),
19012 );
19013 assert!(
19014 matches!(
19015 &account,
19016 InferenceError::ProviderAccount {
19017 provider,
19018 status: 402,
19019 ..
19020 } if provider == "openai"
19021 ),
19022 "ProviderAccount must stay ProviderAccount, got {account:?}"
19023 );
19024
19025 let unavailable = apply_route_failure_context(
19026 InferenceError::CredentialUnavailable {
19027 provider: "parslee".into(),
19028 model: "parslee/reasoning".into(),
19029 reason: CredentialFailure::Expired { expires_at: 42 },
19030 detail: "access token expired".into(),
19031 },
19032 Some(&credential),
19033 );
19034 match &unavailable {
19035 InferenceError::CredentialUnavailable { reason, detail, .. } => {
19036 assert_eq!(*reason, CredentialFailure::Expired { expires_at: 42 });
19037 assert!(detail.contains("Parslee login is absent"), "{detail}");
19038 }
19039 other => panic!("CredentialUnavailable must keep its reason data, got {other:?}"),
19040 }
19041 assert!(is_auth_failure_message(&unavailable.to_string()));
19042 }
19043
19044 #[test]
19050 fn non_parslee_rejection_summaries_match_the_shared_classifier() {
19051 let rejected = route_credential_failure(
19052 "openai/gpt-5.6",
19053 &InferenceError::ProviderAccount {
19054 provider: "openai".into(),
19055 status: 401,
19056 message: "key revoked".into(),
19057 },
19058 false,
19059 )
19060 .expect("a 401 from a configured provider is actionable");
19061 assert!(rejected.contains("credential was rejected"), "{rejected}");
19062 assert!(is_auth_failure_message(&rejected), "{rejected}");
19063
19064 let denied = route_credential_failure(
19065 "openai/gpt-5.6",
19066 &InferenceError::ProviderAccount {
19067 provider: "openai".into(),
19068 status: 403,
19069 message: "model not enabled for this key".into(),
19070 },
19071 false,
19072 )
19073 .expect("a 403 from a configured provider is still actionable");
19074 assert!(denied.contains("denied access"), "{denied}");
19075 assert!(!is_auth_failure_message(&denied), "{denied}");
19076
19077 let generic = route_credential_failure(
19078 "openai/gpt-5.6",
19079 &InferenceError::InferenceFailed("upstream said: token expired".into()),
19080 false,
19081 )
19082 .expect("an auth-rejection message from a non-Parslee route is actionable");
19083 assert!(generic.contains("repair its provider login"), "{generic}");
19084 assert!(is_auth_failure_message(&generic), "{generic}");
19085 }
19086
19087 #[test]
19092 fn the_recovery_hints_lead_with_parslee_sign_in() {
19093 for hint in [
19094 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
19095 .expect("a missing credential is a no-backend case"),
19096 auth_expired_recovery_hint(
19097 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
19098 )
19099 .expect("a rejected session is an auth case"),
19100 ] {
19101 let login = hint
19102 .find("car auth login")
19103 .expect("sign-in must be offered");
19104 let pull = hint
19105 .find("car models pull")
19106 .expect("the local path must still be offered");
19107 assert!(login < pull, "sign-in must be read first:\n{hint}");
19108 }
19109 assert!(
19112 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
19113 .expect("hint")
19114 .starts_with(NO_BACKEND_RECOVERY_MARKER),
19115 "the marker opening is load-bearing"
19116 );
19117 }
19118
19119 #[test]
19122 fn fresh_install_exhaustion_keeps_the_models_pull_guidance() {
19123 let credential = parslee_signed_out_route_failure();
19124 let error = apply_route_failure_context(
19125 InferenceError::InferenceFailed("no models available for generate".into()),
19126 Some(&credential),
19127 )
19128 .to_string();
19129 assert!(error.contains("Parslee login is absent"), "{error}");
19130 assert!(
19131 error.contains(&format!("car models pull {LOCAL_TOOL_MODEL_PULL_ID}")),
19132 "{error}"
19133 );
19134 assert!(error.contains("car auth login"), "{error}");
19135 }
19136
19137 #[test]
19140 fn auth_fallback_from_round_trips_and_defaults_to_none() {
19141 let mut result: InferenceResult = serde_json::from_value(serde_json::json!({
19142 "text": "hi",
19143 "tool_calls": [],
19144 "trace_id": "t",
19145 "model_used": "openai/gpt-5.6",
19146 "latency_ms": 1,
19147 }))
19148 .expect("a payload without the field still deserializes");
19149 assert_eq!(result.auth_fallback_from, None);
19150 let json = serde_json::to_value(&result).unwrap();
19151 assert!(json.get("auth_fallback_from").is_none());
19152
19153 result.auth_fallback_from = Some("parslee/reasoning".to_string());
19154 let json = serde_json::to_value(&result).unwrap();
19155 assert_eq!(json["auth_fallback_from"], "parslee/reasoning");
19156 let back: InferenceResult = serde_json::from_value(json).unwrap();
19157 assert_eq!(
19158 back.auth_fallback_from.as_deref(),
19159 Some("parslee/reasoning")
19160 );
19161 }
19162
19163 #[test]
19168 fn local_last_resort_turn_carries_model_id_and_flag() {
19169 let ordinary: InferenceResult = serde_json::from_value(serde_json::json!({
19170 "text": "hi",
19171 "tool_calls": [],
19172 "trace_id": "ordinary",
19173 "model_used": "anthropic/claude-haiku-4-5:latest",
19174 "latency_ms": 1,
19175 }))
19176 .expect("older payloads default the marker");
19177 assert!(!ordinary.local_last_resort);
19178
19179 let fallback: InferenceResult = serde_json::from_value(serde_json::json!({
19180 "text": "offline answer",
19181 "tool_calls": [],
19182 "trace_id": "fallback",
19183 "model_used": "Qwen3 4B MLX",
19184 "resolved_model_id": "mlx/qwen3-4b:4bit",
19185 "latency_ms": 1,
19186 "local_last_resort": true,
19187 }))
19188 .expect("fallback attribution payload");
19189 assert_eq!(fallback.served_model_id(), "mlx/qwen3-4b:4bit");
19190 assert!(fallback.local_last_resort);
19191
19192 assert!(!is_local_last_resort(None, "mlx/qwen3-4b:4bit"));
19193 assert!(report_local_last_resort_served(
19194 Some("mlx/qwen3-4b:4bit"),
19195 "mlx/qwen3-4b:4bit",
19196 "mlx/qwen3-4b:4bit"
19197 ));
19198 assert!(!is_local_last_resort(
19199 Some("mlx/qwen3-4b:4bit"),
19200 "anthropic/claude-haiku-4-5:latest"
19201 ));
19202 }
19203
19204 #[test]
19205 fn no_backend_hint_passes_through_transient_errors() {
19206 for phrase in [
19209 "API returned 401 Unauthorized",
19210 "API returned 429 Too Many Requests",
19211 "API returned 500 Internal Server Error",
19212 "connection refused",
19213 "request timed out",
19214 "parse response: unexpected end of input",
19215 ] {
19216 assert!(
19217 no_backend_recovery_hint(phrase).is_none(),
19218 "transient error wrongly classified as no-backend: {phrase:?}"
19219 );
19220 }
19221 }
19222
19223 static ENV_MUTEX: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
19226
19227 #[cfg(unix)]
19228 #[tokio::test]
19229 async fn exact_codex_subscription_row_dispatches_without_an_openai_key() {
19230 use std::os::unix::fs::PermissionsExt;
19231
19232 let _environment = ENV_MUTEX.lock().await;
19233 let _restore = RestoredEnvironment::capture(&["CAR_CODEX_BIN", "OPENAI_API_KEY"]);
19234 let tmp = TempDir::new().unwrap();
19235 let fixture = tmp.path().join("codex-fixture.sh");
19236 std::fs::write(
19237 &fixture,
19238 r#"#!/bin/sh
19239if [ -n "${OPENAI_API_KEY-}" ]; then
19240 echo 'OPENAI_API_KEY leaked' >&2
19241 exit 91
19242fi
19243cat >/dev/null
19244printf '%s\n' '{"type":"turn.started"}'
19245printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"fixture newsroom answer"}}'
19246printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":17,"output_tokens":5}}'
19247"#,
19248 )
19249 .unwrap();
19250 std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o700)).unwrap();
19251 unsafe {
19252 std::env::set_var("CAR_CODEX_BIN", &fixture);
19253 std::env::set_var("OPENAI_API_KEY", "must-not-reach-codex");
19254 }
19255
19256 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19257 let mut request = GenerateRequest {
19258 prompt: "write a brief".into(),
19259 params: GenerateParams {
19260 max_tokens: 1056,
19261 strict_model: true,
19262 ..Default::default()
19263 },
19264 ..Default::default()
19265 };
19266 pin_exact_model_id(&mut request, "openai/gpt-5.6-sol:high".into()).unwrap();
19267 let result = engine.generate_tracked(request).await.unwrap();
19268
19269 assert_eq!(result.model_used, "openai/gpt-5.6-sol:high");
19270 assert_eq!(result.text, "fixture newsroom answer");
19271 assert_eq!(result.usage.as_ref().unwrap().total_tokens, 22);
19272 assert_eq!(result.stop_reason, None, "Codex reports no finish reason");
19273 assert!(result.tool_calls.is_empty());
19274 }
19275
19276 #[test]
19284 fn routed_requests_with_media_codex_cli_leaves_out_skip_its_rows() {
19285 let tmp = TempDir::new().unwrap();
19286 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19287 let codex = "openai/gpt-5.6-sol:high";
19288 let inline = ContentBlock::ImageBase64 {
19289 data: "iVBORw0KGgo=".into(),
19290 media_type: "image/png".into(),
19291 };
19292 let link = ContentBlock::ImageUrl {
19293 url: "https://example.com/cat.png".into(),
19294 detail: "auto".into(),
19295 };
19296 let video = ContentBlock::VideoBase64 {
19297 data: "AAAA".into(),
19298 media_type: "video/mp4".into(),
19299 fps: None,
19300 max_frames: None,
19301 };
19302 let audio = ContentBlock::AudioBase64 {
19303 data: "AAAA".into(),
19304 media_type: "audio/wav".into(),
19305 sample_rate: None,
19306 };
19307 let with_images = |blocks: Vec<ContentBlock>| GenerateRequest {
19308 prompt: "What is this?".into(),
19309 images: Some(blocks),
19310 ..Default::default()
19311 };
19312 let in_history = |block: ContentBlock| GenerateRequest {
19313 prompt: "And this?".into(),
19314 messages: Some(vec![Message::UserMultimodal {
19315 content: vec![
19316 ContentBlock::Text {
19317 text: "look".into(),
19318 },
19319 block,
19320 ],
19321 }]),
19322 ..Default::default()
19323 };
19324 let routed = |req: &GenerateRequest, streaming: bool| {
19325 let mut registry = (*engine.unified_registry()).clone();
19326 assert!(
19327 registry.get(codex).is_some(),
19328 "the catalog lists the Codex CLI row"
19329 );
19330 let withheld = InferenceEngine::withhold_codex_cli_rows(req, streaming, &mut registry);
19331 (registry.get(codex).is_some(), withheld, registry)
19332 };
19333
19334 for req in [
19335 with_images(vec![link.clone()]),
19336 with_images(vec![inline.clone(), video.clone()]),
19337 with_images(vec![audio.clone()]),
19338 in_history(link.clone()),
19339 in_history(video.clone()),
19340 in_history(audio.clone()),
19341 ] {
19342 for streaming in [false, true] {
19343 let (kept, withheld, registry) = routed(&req, streaming);
19344 assert!(!kept && withheld, "{:?} streaming={streaming}", req.images);
19345 for name in [codex, "gpt-5.6-sol:high", "GPT-5.6-SOL:HIGH"] {
19348 assert!(
19349 !InferenceEngine::override_still_listed(true, ®istry, name),
19350 "{name}"
19351 );
19352 }
19353 assert!(InferenceEngine::override_still_listed(
19354 true,
19355 ®istry,
19356 "openai/gpt-5.6-sol:latest"
19357 ));
19358 }
19359 }
19360 for req in [with_images(vec![inline.clone()]), in_history(inline)] {
19363 let (kept, withheld, _) = routed(&req, false);
19364 assert!(kept && !withheld);
19365 let (kept, withheld, _) = routed(&req, true);
19366 assert!(!kept && withheld);
19367 }
19368 for streaming in [false, true] {
19370 let (kept, withheld, registry) = routed(&GenerateRequest::default(), streaming);
19371 assert!(kept && !withheld);
19372 assert!(InferenceEngine::override_still_listed(
19373 false,
19374 ®istry,
19375 "no/such-model"
19376 ));
19377 }
19378 }
19379
19380 #[cfg(unix)]
19381 #[tokio::test]
19382 async fn exact_codex_subscription_row_attaches_inline_chat_images() {
19383 use base64::Engine as _;
19384 use std::os::unix::fs::PermissionsExt;
19385
19386 let _environment = ENV_MUTEX.lock().await;
19387 let _restore = RestoredEnvironment::capture(&["CAR_CODEX_BIN", "OPENAI_API_KEY"]);
19388 let tmp = TempDir::new().unwrap();
19389 let log = tmp.path().join("images.txt");
19390 let fixture = tmp.path().join("codex-fixture.sh");
19391 std::fs::write(
19392 &fixture,
19393 format!(
19394 r#"#!/bin/sh
19395cat >/dev/null
19396while [ $# -gt 0 ]; do
19397 if [ "$1" = --image ]; then cmp -s "$2" '{png}' && echo "$2" >> '{log}'; shift; fi
19398 shift
19399done
19400printf '%s\n' '{{"type":"turn.started"}}'
19401printf '%s\n' '{{"type":"item.completed","item":{{"type":"agent_message","text":"{{\"text\":\"Red\",\"tool_calls\":[]}}"}}}}'
19402printf '%s\n' '{{"type":"turn.completed","usage":{{"input_tokens":17,"output_tokens":5}}}}'
19403"#,
19404 png = tmp.path().join("expected.png").display(),
19405 log = log.display()
19406 ),
19407 )
19408 .unwrap();
19409 let png = b"\x89PNG\r\n\x1a\nchat-image";
19410 std::fs::write(tmp.path().join("expected.png"), png).unwrap();
19411 std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o700)).unwrap();
19412 unsafe {
19413 std::env::set_var("CAR_CODEX_BIN", &fixture);
19414 }
19415
19416 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19417 let mut request = GenerateRequest {
19418 messages: Some(vec![Message::User {
19419 content: "What colour is this?".into(),
19420 }]),
19421 images: Some(vec![ContentBlock::ImageBase64 {
19422 data: base64::engine::general_purpose::STANDARD.encode(png),
19423 media_type: "image/png".into(),
19424 }]),
19425 params: GenerateParams {
19426 strict_model: true,
19427 ..Default::default()
19428 },
19429 ..Default::default()
19430 };
19431 pin_exact_model_id(&mut request, "openai/gpt-5.6-sol:high".into()).unwrap();
19432 let result = engine.generate_tracked(request).await.unwrap();
19433
19434 assert_eq!(result.model_used, "openai/gpt-5.6-sol:high");
19435 assert_eq!(result.text, "Red");
19436 let attached = std::fs::read_to_string(&log).unwrap();
19437 assert_eq!(
19438 attached.lines().count(),
19439 1,
19440 "one --image carrying the chat image"
19441 );
19442 }
19443
19444 #[tokio::test]
19445 async fn isolated_test_child_removes_parent_auth_lock_override() {
19446 const CHILD_SENTINEL: &str = "CAR_AUTH_LOCK_SANITIZER_CHILD";
19447 if std::env::var_os(CHILD_SENTINEL).is_some() {
19448 assert!(
19449 std::env::var_os("CAR_AUTH_LOCK_PATH").is_none(),
19450 "an isolated credential test must not inherit the invoking process's auth lock"
19451 );
19452 return;
19453 }
19454
19455 let _environment = ENV_MUTEX.lock().await;
19456 let _restore = RestoredEnvironment::capture(&["CAR_AUTH_LOCK_PATH"]);
19457 unsafe {
19458 std::env::set_var(
19459 "CAR_AUTH_LOCK_PATH",
19460 "/sentinel/must-not-reach-isolated-child.lock",
19461 );
19462 }
19463 assert!(!crate::run_in_isolated_test_process(
19464 "tests::isolated_test_child_removes_parent_auth_lock_override",
19465 CHILD_SENTINEL,
19466 ));
19467 }
19468
19469 fn test_config(models_dir: PathBuf) -> InferenceConfig {
19470 let state_root = models_dir
19474 .parent()
19475 .map(Path::to_path_buf)
19476 .unwrap_or_else(|| models_dir.clone());
19477 InferenceConfig {
19478 models_dir,
19479 state_root,
19480 device: None,
19481 generation_model: "Qwen3-0.6B".into(),
19482 preferred_generation_model: None,
19483 embedding_model: "Qwen3-Embedding-0.6B".into(),
19484 preferred_embedding_model: None,
19485 classification_model: "Qwen3-0.6B".into(),
19486 preferred_classification_model: None,
19487 }
19488 }
19489
19490 fn metal_mac_for_fit(ram_gb: u64) -> HardwareInfo {
19491 HardwareInfo {
19492 os: "macos".into(),
19493 arch: "aarch64".into(),
19494 cpu_cores: 8,
19495 total_ram_mb: ram_gb * 1024,
19496 gpu_backend: crate::hardware::GpuBackend::Metal,
19497 gpu_memory_mb: None,
19498 gpu_devices: vec![],
19499 recommended_model: String::new(),
19500 recommended_context: 8_192,
19501 max_model_mb: 0,
19502 }
19503 }
19504
19505 #[test]
19511 fn unified_fit_reads_the_installed_checkpoints_kv_geometry() {
19512 let policy = resource_policy::ResourcePolicy::everyday();
19513 let eight_b_at_24 = |install: bool| {
19514 let root = tempfile::tempdir().unwrap();
19515 let models_dir = root.path().join("weights");
19516 let engine = InferenceEngine::new(test_config(models_dir.clone()));
19517 let schema = engine
19518 .list_schemas()
19519 .into_iter()
19520 .find(|schema| schema.id == "mlx/qwen3-8b:4bit")
19521 .expect("builtin 8B MLX row");
19522 if install {
19523 let dir = models_dir.join(&schema.name);
19524 std::fs::create_dir_all(&dir).unwrap();
19525 std::fs::write(
19526 dir.join("config.json"),
19527 r#"{"num_hidden_layers":36,"num_key_value_heads":32,"head_dim":128,"hidden_size":4096,"num_attention_heads":32,"torch_dtype":"bfloat16"}"#,
19528 )
19529 .unwrap();
19530 }
19531 engine
19532 .list_models_unified_for(&metal_mac_for_fit(24), &policy)
19533 .into_iter()
19534 .find(|row| row.id == schema.id)
19535 .expect("8B row listed")
19536 };
19537
19538 let estimated = eight_b_at_24(false);
19539 let installed = eight_b_at_24(true);
19540 assert_eq!(estimated.fit, ModelFitStatus::Fits, "{estimated:?}");
19541 assert_eq!(installed.fit, ModelFitStatus::TooBig, "{installed:?}");
19542 assert!(installed.estimated_peak_mb > estimated.estimated_peak_mb);
19543 }
19544
19545 #[test]
19551 fn unified_fit_follows_a_removed_or_replaced_checkpoint() {
19552 let policy = resource_policy::ResourcePolicy::everyday();
19553 let root = tempfile::tempdir().unwrap();
19554 let models_dir = root.path().join("weights");
19555 let engine = InferenceEngine::new(test_config(models_dir.clone()));
19556 let schema = engine
19557 .list_schemas()
19558 .into_iter()
19559 .find(|schema| schema.id == "mlx/qwen3-8b:4bit")
19560 .expect("builtin 8B MLX row");
19561 let dir = models_dir.join(&schema.name);
19562 let config = dir.join("config.json");
19563 let listed = || {
19564 engine
19565 .list_models_unified_for(&metal_mac_for_fit(24), &policy)
19566 .into_iter()
19567 .find(|row| row.id == schema.id)
19568 .expect("8B row listed")
19569 };
19570 let write = |kv_heads: u32| {
19571 std::fs::write(
19572 &config,
19573 format!(
19574 r#"{{"num_hidden_layers":36,"num_key_value_heads":{kv_heads},"head_dim":128,"hidden_size":4096,"num_attention_heads":32,"torch_dtype":"bfloat16"}}"#
19575 ),
19576 )
19577 .unwrap();
19578 };
19579
19580 let estimated = listed();
19581 std::fs::create_dir_all(&dir).unwrap();
19582 write(32);
19583 let installed = listed();
19584 std::fs::remove_dir_all(&dir).unwrap();
19585 let removed = listed();
19586 std::fs::create_dir_all(&dir).unwrap();
19587 write(32);
19588 assert_eq!(listed().fit, ModelFitStatus::TooBig, "reinstalled");
19589 let modified = std::fs::metadata(&config).unwrap().modified().unwrap();
19590 let length = std::fs::metadata(&config).unwrap().len();
19591 write(16);
19592 assert_eq!(std::fs::metadata(&config).unwrap().len(), length);
19593 std::fs::File::options()
19594 .write(true)
19595 .open(&config)
19596 .unwrap()
19597 .set_modified(modified + std::time::Duration::from_secs(10))
19598 .unwrap();
19599 let replaced = listed();
19600
19601 assert_eq!(installed.fit, ModelFitStatus::TooBig, "{installed:?}");
19602 assert_eq!(removed.fit, ModelFitStatus::Fits, "{removed:?}");
19603 assert_eq!(removed.estimated_peak_mb, estimated.estimated_peak_mb);
19604 assert_eq!(replaced.fit, ModelFitStatus::Fits, "{replaced:?}");
19605 assert!(
19606 replaced.estimated_peak_mb > estimated.estimated_peak_mb
19607 && replaced.estimated_peak_mb < installed.estimated_peak_mb,
19608 "16 KV heads sit between the 8B fallback and 32 heads: {replaced:?}"
19609 );
19610 }
19611
19612 #[test]
19619 fn unified_rows_carry_fit_per_machine_and_keep_deprecated_rows() {
19620 if !crate::run_in_isolated_test_process(
19621 "tests::unified_rows_carry_fit_per_machine_and_keep_deprecated_rows",
19622 "CAR_UNIFIED_ROWS_FIT_TEST_CHILD",
19623 ) {
19624 return;
19625 }
19626
19627 let _credential_scope = crate::openrouter::test_credential_scope();
19634 crate::openrouter::set_test_credential(Some("unified-rows-fit-test-key"));
19635 let secret_activity_before = car_secrets::secret_store_activity();
19636 let root = tempfile::tempdir().unwrap();
19637 let config = test_config(root.path().join("weights"));
19638 let mut engine = InferenceEngine::new(config);
19639 let mut retired = engine
19640 .list_schemas()
19641 .into_iter()
19642 .find(|schema| schema.id == "mlx/qwen3-4b:4bit")
19643 .expect("builtin 4B MLX row");
19644 retired.id = "test/retired-4b:4bit".into();
19645 retired.name = "Retired 4B".into();
19646 retired.deprecated = true;
19647 engine.register_model(retired);
19648 let policy = resource_policy::ResourcePolicy::everyday();
19649
19650 let at_8 = engine.list_models_unified_for(&metal_mac_for_fit(8), &policy);
19651 let at_32 = engine.list_models_unified_for(&metal_mac_for_fit(32), &policy);
19652 let ids = |rows: &[ModelInfo]| rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
19653 assert_eq!(
19654 ids(&at_8),
19655 ids(&at_32),
19656 "the machine never removes or reorders a row"
19657 );
19658 let row = |rows: &[ModelInfo], id: &str| {
19659 rows.iter()
19660 .find(|row| row.id == id)
19661 .unwrap_or_else(|| panic!("{id} missing"))
19662 .clone()
19663 };
19664
19665 let eight_b_small = row(&at_8, "mlx/qwen3-8b:4bit");
19666 let eight_b_large = row(&at_32, "mlx/qwen3-8b:4bit");
19667 assert_eq!(eight_b_small.fit, ModelFitStatus::TooBig);
19668 assert_eq!(eight_b_large.fit, ModelFitStatus::Fits);
19669 assert!(eight_b_small.platform_compatible && eight_b_large.platform_compatible);
19670 assert_eq!(
19671 eight_b_small.estimated_peak_mb, eight_b_large.estimated_peak_mb,
19672 "the estimate is the model's; only the budget differs"
19673 );
19674 assert!(eight_b_small.estimated_peak_mb.is_some_and(|mb| mb > 4_800));
19675 assert_eq!(eight_b_small.family.as_deref(), Some("qwen3"));
19676 assert!(eight_b_small.version.is_some());
19677
19678 let retired = row(&at_8, "test/retired-4b:4bit");
19679 assert!(retired.deprecated, "deprecated rows stay listed, flagged");
19680 assert!(!eight_b_small.deprecated);
19681
19682 let mut saw_remote = false;
19683 for remote in at_8.iter().filter(|row| !row.is_local) {
19684 saw_remote = true;
19685 assert_eq!(remote.fit, ModelFitStatus::Fits, "{}", remote.id);
19686 assert!(remote.platform_compatible, "{}", remote.id);
19687 assert_eq!(remote.estimated_peak_mb, None, "{}", remote.id);
19688 assert_eq!(
19689 remote.family, None,
19690 "{}: no upstream identifier here",
19691 remote.id
19692 );
19693 assert_eq!(remote.version, None, "{}", remote.id);
19694 }
19695 assert!(saw_remote);
19696 assert_eq!(
19697 car_secrets::secret_store_activity(),
19698 secret_activity_before,
19699 "catalog snapshots must use the injected credential without consulting host secrets"
19700 );
19701
19702 for (small, large) in at_8.iter().zip(&at_32) {
19705 let strip = |row: &ModelInfo| {
19706 let mut value = serde_json::to_value(row).unwrap();
19707 let object = value.as_object_mut().unwrap();
19708 for key in ["fit", "estimated_peak_mb", "platform_compatible"] {
19709 object.remove(key);
19710 }
19711 value
19712 };
19713 assert_eq!(strip(small), strip(large), "{}", small.id);
19714 }
19715 }
19716
19717 #[tokio::test]
19718 async fn explicit_model_blocked_by_zero_budget_does_not_substitute_or_download() {
19719 let root = tempfile::tempdir().unwrap();
19720 let models_dir = root.path().join("weights");
19721 let config = test_config(models_dir.clone());
19722 let repository =
19723 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19724 crate::resource_policy::ResourcePolicyRepository::save(
19725 &repository,
19726 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19727 )
19728 .unwrap();
19729 let engine = InferenceEngine::new(config);
19730
19731 let error = engine
19732 .generate_tracked(GenerateRequest {
19733 prompt: "hello".into(),
19734 model: Some("mlx/qwen3-4b:4bit".into()),
19735 ..Default::default()
19736 })
19737 .await
19738 .unwrap_err();
19739
19740 assert!(matches!(
19741 error,
19742 InferenceError::LocalResourceBlocked {
19743 preflight: crate::resource_policy::LocalLoadPreflight {
19744 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
19745 ..
19746 },
19747 ..
19748 }
19749 ));
19750 assert!(
19751 !models_dir.exists() || std::fs::read_dir(models_dir).unwrap().next().is_none(),
19752 "admission must happen before download/load"
19753 );
19754 }
19755
19756 #[tokio::test]
19757 async fn external_vllm_mlx_bypasses_local_admission_while_managed_artifact_does_not() {
19758 let schema = crate::vllm_mlx::to_model_schema(
19759 &crate::vllm_mlx::DiscoveredModel {
19760 id: "mlx-community/Qwen3-4B-4bit".into(),
19761 owned_by: None,
19762 },
19763 "http://localhost:8000",
19764 );
19765 assert!(!InferenceEngine::requires_local_admission(&schema));
19766
19767 let root = tempfile::tempdir().unwrap();
19768 let config = test_config(root.path().join("weights"));
19769 let repository =
19770 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19771 crate::resource_policy::ResourcePolicyRepository::save(
19772 &repository,
19773 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19774 )
19775 .unwrap();
19776 let engine = InferenceEngine::new(config);
19777 let (external, reservation) = engine
19778 .vllm_live_schema(schema.clone(), None, 0)
19779 .await
19780 .unwrap();
19781 assert!(matches!(
19782 external.source,
19783 ModelSource::VllmMlx { ref endpoint, .. } if endpoint == "http://localhost:8000"
19784 ));
19785 assert!(reservation.is_none());
19786
19787 let mut managed = schema;
19788 managed.source = ModelSource::ManagedVllmMlx {
19789 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
19790 hf_weight_file: None,
19791 };
19792 assert!(InferenceEngine::requires_local_admission(&managed));
19793 let error = engine
19794 .reserve_local_request(&managed, 0)
19795 .expect_err("zero-GB policy must reject before vllm runtime/download/spawn");
19796 assert!(matches!(
19797 error,
19798 InferenceError::LocalResourceBlocked {
19799 preflight: crate::resource_policy::LocalLoadPreflight {
19800 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
19801 ..
19802 },
19803 ..
19804 }
19805 ));
19806 }
19807
19808 #[cfg(unix)]
19809 async fn assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(
19810 streaming: bool,
19811 cancel_first: bool,
19812 ) {
19813 struct FixedProbe;
19814 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
19815 fn available_memory_mb(
19816 &self,
19817 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
19818 Ok(Some(24_000))
19819 }
19820 }
19821
19822 let Some(python) = crate::vllm_runtime::test_python_interpreter() else {
19823 panic!("a real Python interpreter is required for the managed-vllm dispatch fixture");
19824 };
19825 let root = tempfile::tempdir().unwrap();
19826 let script = root.path().join("fake-vllm-mlx");
19827 let spawned = root.path().join("spawned");
19828 let release = root.path().join("release");
19829 std::fs::write(
19830 &script,
19831 format!(
19832 "#!{}\n\
19833 import http.server, os, sys, time\n\
19834 import socket as _s; _s.getfqdn = lambda n='': n\n\
19835 marker = {:?}\n\
19836 release = {:?}\n\
19837 open(marker, 'w').close()\n\
19838 while not os.path.exists(release): time.sleep(0.01)\n\
19839 port = int(sys.argv[sys.argv.index('--port') + 1])\n\
19840 class H(http.server.BaseHTTPRequestHandler):\n\
19841 \x20 def do_GET(self):\n\
19842 \x20 self.send_response(200); self.end_headers(); self.wfile.write(b'ok')\n\
19843 \x20 def do_POST(self):\n\
19844 \x20 length = int(self.headers.get('content-length', '0'))\n\
19845 \x20 request = self.rfile.read(length).replace(b' ', b'')\n\
19846 \x20 if b'\"stream\":true' in request:\n\
19847 \x20 body = b'data: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"round13-stream\"}},\"finish_reason\":null}}]}}\\n\\ndata: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"stop\"}}]}}\\n\\ndata: [DONE]\\n\\n'\n\
19848 \x20 content_type = 'text/event-stream'\n\
19849 \x20 else:\n\
19850 \x20 body = b'{{\"id\":\"fixture\",\"object\":\"chat.completion\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"message\":{{\"role\":\"assistant\",\"content\":\"round13-ok\"}},\"finish_reason\":\"stop\"}}],\"usage\":{{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}}}'\n\
19851 \x20 content_type = 'application/json'\n\
19852 \x20 self.send_response(200); self.send_header('content-type', content_type); self.send_header('content-length', str(len(body))); self.end_headers(); self.wfile.write(body)\n\
19853 \x20 def log_message(self, *args): pass\n\
19854 http.server.HTTPServer(('127.0.0.1', port), H).serve_forever()\n",
19855 python.display(),
19856 spawned.display().to_string(),
19857 release.display().to_string(),
19858 ),
19859 )
19860 .unwrap();
19861 use std::os::unix::fs::PermissionsExt;
19862 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
19863
19864 let models_dir = root.path().join("models");
19865 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
19866 let mut schema = crate::registry::builtin_catalog()
19867 .into_iter()
19868 .find(ModelSchema::is_car_managed_vllm_mlx)
19869 .expect("managed vllm fixture schema");
19870 schema.id = "vllm-mlx/round12-singleflight".into();
19871 schema.name = "round12-singleflight".into();
19872 schema.cost.size_mb = Some(1);
19873 schema.cost.ram_mb = Some(1);
19874 schema.source = ModelSource::ManagedVllmMlx {
19875 hf_repo: "fixture/round12-singleflight".into(),
19876 hf_weight_file: None,
19877 };
19878 let model_dir = models_dir.join(&schema.name);
19879 std::fs::create_dir_all(&model_dir).unwrap();
19880 std::fs::write(model_dir.join("config.json"), b"{}").unwrap();
19881 std::fs::write(model_dir.join("model.safetensors"), b"fixture").unwrap();
19882 engine
19883 .unified_registry_mut()
19884 .register_project_model(schema.clone());
19885
19886 let coordinator = Arc::new(
19887 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
19888 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
19893 crate::hardware::HardwareInfo::detect(),
19894 Arc::new(FixedProbe),
19895 ),
19896 );
19897 engine.local_admission = coordinator.clone();
19898 engine.vllm_pool = Arc::new(crate::vllm_pool::VllmServerPool::with_test_runtime(
19899 std::time::Duration::from_secs(300),
19900 coordinator,
19901 script,
19902 ));
19903 let engine = Arc::new(engine);
19904 let model_id = schema.id.clone();
19905 let mut request = GenerateRequest {
19906 prompt: "hello from round 13".into(),
19907 model: Some(model_id.clone()),
19908 ..Default::default()
19909 };
19910 request.params.strict_model = true;
19915 let (estimated_input, _, _) = engine.estimated_tokens(&request, Some(&model_id));
19916 let expected_footprint = estimated_input.saturating_add(request.params.max_tokens);
19917 let expected_estimate = engine
19918 .local_model_preflight(&model_id, expected_footprint)
19919 .unwrap()
19920 .estimate;
19921 let expected_request_overhead_mb = expected_estimate
19922 .runtime_overhead_mb
19923 .saturating_add(expected_estimate.context_overhead_mb)
19924 .saturating_add(expected_estimate.transient_margin_mb);
19925
19926 async fn execute(
19927 engine: Arc<InferenceEngine>,
19928 request: GenerateRequest,
19929 streaming: bool,
19930 direct_vllm: bool,
19931 ) -> Result<String, InferenceError> {
19932 if direct_vllm {
19933 let model_id = request.model.as_deref().expect("explicit fixture model");
19934 let schema = engine
19935 .unified_registry()
19936 .get(model_id)
19937 .cloned()
19938 .expect("fixture schema");
19939 let (schema, _) = engine.vllm_live_schema(schema, None, 0).await?;
19940 return match schema.source {
19941 ModelSource::VllmMlx { endpoint, .. } => Ok(endpoint),
19942 source => Err(InferenceError::InferenceFailed(format!(
19943 "fixture did not resolve to a vllm endpoint: {source:?}"
19944 ))),
19945 };
19946 }
19947 if !streaming {
19948 return engine
19949 .generate_tracked(request)
19950 .await
19951 .map(|result| result.text);
19952 }
19953 let mut tracked = engine.generate_tracked_stream(request).await?;
19954 let mut accumulator = crate::stream::StreamAccumulator::default();
19955 while let Some(event) = tracked.events.recv().await {
19956 let done = matches!(event, crate::stream::StreamEvent::Done { .. });
19957 accumulator.push(&event);
19958 if done {
19959 break;
19960 }
19961 }
19962 Ok(accumulator.finish().0)
19963 }
19964
19965 let first_engine = engine.clone();
19966 let first_request = request.clone();
19967 let mut first = tokio::spawn(async move {
19968 execute(first_engine, first_request, streaming, cancel_first).await
19969 });
19970 tokio::time::timeout(std::time::Duration::from_secs(5), async {
19971 while !spawned.exists() {
19972 if first.is_finished() {
19973 let result = (&mut first).await;
19974 panic!("first dispatch ended before spawn: {result:?}");
19975 }
19976 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
19977 }
19978 })
19979 .await
19980 .expect("first request must spawn before publication");
19981 if !cancel_first {
19982 let during_startup = engine.local_model_preflight(&model_id, 0).unwrap();
19983 assert_eq!(
19984 during_startup.active_reservations_mb, expected_request_overhead_mb,
19985 "the cold-to-pending transfer must retain all request context/KV overhead"
19986 );
19987 assert_eq!(
19988 during_startup.resident_model_mb, expected_estimate.weights_mb,
19989 "the pending allocation must replace the exact cold-weight charge"
19990 );
19991 assert_eq!(
19992 during_startup
19993 .active_reservations_mb
19994 .saturating_add(during_startup.resident_model_mb),
19995 expected_estimate.estimated_peak_mb,
19996 "active request overhead plus pending weights must preserve the full admitted footprint"
19997 );
19998 }
19999
20000 let second_engine = engine.clone();
20001 let mut second =
20002 tokio::spawn(
20003 async move { execute(second_engine, request, streaming, cancel_first).await },
20004 );
20005 assert!(
20006 tokio::time::timeout(std::time::Duration::from_millis(100), &mut second)
20007 .await
20008 .is_err(),
20009 "request 2 must wait for request 1 to publish instead of failing on startup state"
20010 );
20011
20012 let first_result = if cancel_first {
20013 std::fs::remove_file(&spawned).unwrap();
20014 first.abort();
20015 let _ = first.await;
20016 tokio::time::timeout(std::time::Duration::from_secs(5), async {
20017 while !spawned.exists() {
20018 if second.is_finished() {
20019 let result = (&mut second).await;
20020 panic!("waiter ended before replacement spawn: {result:?}");
20021 }
20022 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
20023 }
20024 })
20025 .await
20026 .expect("waiter must continue after cancelled owner teardown");
20027 None
20028 } else {
20029 std::fs::write(&release, b"release").unwrap();
20030 Some(
20031 tokio::time::timeout(std::time::Duration::from_secs(5), first)
20032 .await
20033 .expect("first dispatch completes")
20034 .unwrap()
20035 .unwrap(),
20036 )
20037 };
20038 std::fs::write(&release, b"release").unwrap();
20039 let second_result = tokio::time::timeout(std::time::Duration::from_secs(5), second)
20040 .await
20041 .expect("second dispatch completes")
20042 .unwrap()
20043 .unwrap();
20044 let expected = if cancel_first {
20045 None
20046 } else if streaming {
20047 Some("round13-stream")
20048 } else {
20049 Some("round13-ok")
20050 };
20051 if let (Some(first_result), Some(expected)) = (first_result, expected) {
20052 assert_eq!(first_result, expected);
20053 }
20054 if let Some(expected) = expected {
20055 assert_eq!(second_result, expected);
20056 } else {
20057 assert!(second_result.starts_with("http://127.0.0.1:"));
20058 }
20059 assert_eq!(engine.vllm_pool.len().await, 1);
20060 assert!(engine
20061 .vllm_pool
20062 .release_model_if_present(&model_id)
20063 .await
20064 .unwrap());
20065 }
20066
20067 #[cfg(unix)]
20068 #[tokio::test]
20069 async fn concurrent_managed_vllm_generate_waits_before_outer_reservation() {
20070 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, false).await;
20071 }
20072
20073 #[cfg(unix)]
20074 #[tokio::test]
20075 async fn concurrent_managed_vllm_stream_waits_before_outer_reservation() {
20076 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(true, false).await;
20077 }
20078
20079 #[cfg(unix)]
20080 #[tokio::test]
20081 async fn managed_vllm_waiter_continues_after_startup_owner_cancellation() {
20082 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, true).await;
20083 }
20084
20085 #[tokio::test]
20086 async fn same_state_root_engines_share_runtime_components() {
20087 const CHILD_ENV: &str = "CAR_TWO_ENGINE_RUNTIME_TEST_CHILD";
20088 if std::env::var_os(CHILD_ENV).is_some() {
20089 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
20090 return;
20091 }
20092 let root = tempfile::tempdir().unwrap();
20093 let first = InferenceEngine::new(test_config(root.path().join("weights")));
20094 let second = InferenceEngine::new(test_config(root.path().join("weights")));
20095
20096 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
20097 assert!(Arc::ptr_eq(&first.vllm_pool, &second.vllm_pool));
20098 assert!(Arc::ptr_eq(
20099 &first.resource_policy_generation,
20100 &second.resource_policy_generation
20101 ));
20102 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
20103 {
20104 assert!(Arc::ptr_eq(&first.text_backends, &second.text_backends));
20105 assert!(Arc::ptr_eq(&first.local_backends, &second.local_backends));
20106 }
20107 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
20108 {
20109 assert!(Arc::ptr_eq(&first.backend, &second.backend));
20110 assert!(Arc::ptr_eq(
20111 &first.embedding_backend,
20112 &second.embedding_backend
20113 ));
20114 }
20115
20116 let loads = Arc::new(std::sync::atomic::AtomicU64::new(0));
20117 let first_cache = first._runtime_scope.load_probe.clone();
20118 let second_cache = second._runtime_scope.load_probe.clone();
20119 let mut threads = Vec::new();
20120 for cache in [first_cache, second_cache] {
20121 let loads = loads.clone();
20122 threads.push(std::thread::spawn(move || {
20123 cache
20124 .get_or_load::<()>("same/model", 1, || {
20125 loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
20126 std::thread::sleep(std::time::Duration::from_millis(20));
20127 Ok(())
20128 })
20129 .unwrap()
20130 }));
20131 }
20132 let handles = threads
20133 .into_iter()
20134 .map(|thread| thread.join().unwrap())
20135 .collect::<Vec<_>>();
20136 assert_eq!(loads.load(std::sync::atomic::Ordering::SeqCst), 1);
20137 assert!(Arc::ptr_eq(&handles[0], &handles[1]));
20138
20139 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
20140 .arg("--exact")
20141 .arg("tests::same_state_root_engines_share_runtime_components")
20142 .env(CHILD_ENV, "1")
20143 .stdin(std::process::Stdio::null())
20144 .stdout(std::process::Stdio::null())
20145 .stderr(std::process::Stdio::null())
20146 .kill_on_drop(true)
20147 .spawn()
20148 .unwrap();
20149 first
20150 .vllm_pool
20151 .insert_test_process("vllm-mlx/two-engine", child)
20152 .await;
20153 first.local_admission.mark_resident_allocation(
20154 "vllm-mlx/two-engine",
20155 &resource_policy::vllm_process_allocation_id("vllm-mlx/two-engine"),
20156 1,
20157 );
20158 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
20159 drop(first);
20160 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
20161 assert!(second.local_admission.is_resident("vllm-mlx/two-engine"));
20162 assert!(second.vllm_pool.evict_model("vllm-mlx/two-engine").await);
20163 assert!(!second.local_admission.is_resident("vllm-mlx/two-engine"));
20164 }
20165
20166 #[tokio::test]
20167 async fn last_engine_drop_reaps_vllm_before_new_runtime_admission() {
20168 const CHILD_ENV: &str = "CAR_LAST_ENGINE_VLLM_DROP_TEST_CHILD";
20169 if std::env::var_os(CHILD_ENV).is_some() {
20170 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
20171 return;
20172 }
20173 let root = tempfile::tempdir().unwrap();
20174 let config = test_config(root.path().join("weights"));
20175 let engine = InferenceEngine::new(config.clone());
20176 let coordinator = engine.local_admission.clone();
20177 let schema = engine
20178 .unified_registry()
20179 .all()
20180 .find(|schema| schema.is_vllm_mlx())
20181 .cloned()
20182 .expect("supervised vllm schema");
20183 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
20184 .arg("--exact")
20185 .arg("tests::last_engine_drop_reaps_vllm_before_new_runtime_admission")
20186 .env(CHILD_ENV, "1")
20187 .stdin(std::process::Stdio::null())
20188 .stdout(std::process::Stdio::null())
20189 .stderr(std::process::Stdio::null())
20190 .kill_on_drop(true)
20191 .spawn()
20192 .unwrap();
20193 engine
20194 .vllm_pool
20195 .insert_test_process(&schema.id, child)
20196 .await;
20197 coordinator.mark_resident_allocation(
20198 &schema.id,
20199 &resource_policy::vllm_process_allocation_id(&schema.id),
20200 1,
20201 );
20202
20203 drop(engine);
20204
20205 tokio::time::timeout(std::time::Duration::from_secs(2), async {
20206 while coordinator.teardown_pending(&schema.id) {
20207 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
20208 }
20209 })
20210 .await
20211 .expect("last runtime drop must eventually confirm supervised child exit");
20212 assert!(!coordinator.is_resident(&schema.id));
20213 let next = InferenceEngine::new(config);
20214 assert!(Arc::ptr_eq(&coordinator, &next.local_admission));
20215 let preflight = next.local_model_preflight(&schema.id, 0).unwrap();
20216 assert_eq!(preflight.resident_model_mb, 0);
20217 assert!(preflight.estimated_incremental_mb >= preflight.estimate.weights_mb);
20218 }
20219
20220 #[cfg(unix)]
20221 #[test]
20222 fn symlinked_state_roots_share_exact_coordinator_and_runtime() {
20223 use std::os::unix::fs::symlink;
20224
20225 let fixture = tempfile::tempdir().unwrap();
20226 let real = fixture.path().join("real-state");
20227 std::fs::create_dir(&real).unwrap();
20228 let alias = fixture.path().join("state-alias");
20229 symlink(&real, &alias).unwrap();
20230
20231 let first = InferenceEngine::new(test_config(real.join("weights")));
20232 let second = InferenceEngine::new(test_config(alias.join("weights")));
20233 assert!(Arc::ptr_eq(&first.local_admission, &second.local_admission));
20234 assert!(Arc::ptr_eq(&first._runtime_scope, &second._runtime_scope));
20235 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
20236 }
20237
20238 #[tokio::test]
20239 async fn stream_reservation_lives_until_returned_receiver_is_released() {
20240 let root = tempfile::tempdir().unwrap();
20241 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
20242 let schema = crate::registry::builtin_catalog()
20243 .into_iter()
20244 .find(|schema| schema.is_local() && !schema.is_vllm_mlx())
20245 .expect("local model schema");
20246 engine.local_admission.mark_resident(&schema.id, 1);
20247 let reservation = engine.reserve_local_request(&schema, 64).unwrap();
20248 assert_eq!(engine.local_admission.active_request_count(&schema.id), 1);
20249 let (source_tx, source_rx) = tokio::sync::mpsc::channel(1);
20250 let returned =
20251 InferenceEngine::hold_optional_reservation_for_stream(source_rx, Some(reservation));
20252
20253 drop(returned);
20254 source_tx
20255 .send(stream::StreamEvent::TextDelta("release".into()))
20256 .await
20257 .unwrap();
20258 for _ in 0..20 {
20259 if engine.local_admission.active_request_count(&schema.id) == 0 {
20260 break;
20261 }
20262 tokio::task::yield_now().await;
20263 }
20264 assert_eq!(engine.local_admission.active_request_count(&schema.id), 0);
20265
20266 let compact_source = include_str!("lib.rs")
20267 .split_whitespace()
20268 .collect::<String>();
20269 let remote_handoff = [
20270 "Self::hold_optional_reservation_for_stream",
20271 "(receiver,",
20272 "candidate_reservation.take(),",
20273 ");",
20274 ]
20275 .concat();
20276 assert!(compact_source.contains(&remote_handoff));
20277 }
20278
20279 #[tokio::test]
20280 async fn residual_voice_allocation_blocks_removal_with_typed_error() {
20281 let root = tempfile::tempdir().unwrap();
20282 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
20283 engine
20284 .local_admission
20285 .mark_resident("voice/removal-fixture", 1);
20286
20287 let error = match engine
20288 .prepare_local_model_removal("voice/removal-fixture")
20289 .await
20290 {
20291 Err(error) => error,
20292 Ok(_) => panic!("voice allocation must be released by its owner first"),
20293 };
20294 assert!(matches!(
20295 error,
20296 crate::resource_policy::ModelMaintenanceError::ResidualResidency {
20297 model_id,
20298 allocation_ids,
20299 } if model_id == "voice/removal-fixture"
20300 && allocation_ids == vec!["voice/removal-fixture"]
20301 ));
20302 }
20303
20304 #[tokio::test]
20305 async fn catalog_voice_id_blocks_removal_of_live_provider_alias() {
20306 let root = tempfile::tempdir().unwrap();
20307 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
20308 engine.local_admission.register_model_aliases(
20309 "voice/parakeet-tdt-0.6b",
20310 ["mlx/parakeet-tdt-0.6b-v3:default"],
20311 );
20312 let mut reservation = engine
20313 .local_admission
20314 .reserve_measured_host_allocation(
20315 "voice/parakeet-tdt-0.6b",
20316 "voice/parakeet-tdt-0.6b#voice-allocation-0",
20317 1024 * 1024,
20318 0,
20319 )
20320 .unwrap();
20321 reservation.publish_resident_weights(1024 * 1024);
20322 drop(reservation);
20323
20324 let result = engine
20325 .prepare_local_model_removal("mlx/parakeet-tdt-0.6b-v3:default")
20326 .await;
20327 assert!(matches!(
20328 result,
20329 Err(crate::resource_policy::ModelMaintenanceError::ResidualResidency { .. })
20330 ));
20331 }
20332
20333 struct RefusingRemovalOffload;
20334
20335 #[async_trait::async_trait]
20336 impl crate::offload::LocalGenerationOffload for RefusingRemovalOffload {
20337 async fn generate(
20338 &self,
20339 _request: GenerateRequest,
20340 ) -> Result<InferenceResult, InferenceError> {
20341 unreachable!("removal test does not generate")
20342 }
20343
20344 async fn stream(
20345 &self,
20346 _request: GenerateRequest,
20347 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
20348 unreachable!("removal test does not stream")
20349 }
20350
20351 async fn resident_models(&self) -> Vec<String> {
20352 vec!["worker/refuses-release".into()]
20353 }
20354
20355 async fn release_model(&self, _model_id: &str) -> Result<bool, InferenceError> {
20356 Ok(false)
20357 }
20358 }
20359
20360 #[tokio::test]
20361 async fn worker_release_boolean_is_required_for_model_removal() {
20362 let _offload_guard = crate::offload::test_offload_lock().lock().await;
20363 crate::offload::set_local_offload(Some(Arc::new(RefusingRemovalOffload)));
20364 let root = tempfile::tempdir().unwrap();
20365 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
20366 let result = engine
20367 .prepare_local_model_removal("worker/refuses-release")
20368 .await;
20369 crate::offload::set_local_offload(None);
20370
20371 assert!(matches!(
20372 result,
20373 Err(crate::resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
20374 model_id
20375 )) if model_id == "worker/refuses-release"
20376 ));
20377 }
20378
20379 #[test]
20380 fn transient_mlx_vlm_allocation_never_becomes_resident() {
20381 struct FixedProbe;
20382 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
20383 fn available_memory_mb(
20384 &self,
20385 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
20386 Ok(Some(24_000))
20387 }
20388 }
20389
20390 let root = tempfile::tempdir().unwrap();
20391 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
20392 engine.local_admission = Arc::new(
20393 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
20394 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
20395 crate::hardware::HardwareInfo {
20396 total_ram_mb: 32 * 1024,
20397 ..crate::hardware::HardwareInfo::detect()
20398 },
20399 Arc::new(FixedProbe),
20400 ),
20401 );
20402 let mut schema = crate::registry::builtin_catalog()
20403 .into_iter()
20404 .find(|schema| schema.tags.iter().any(|tag| tag == "mlx-vlm-cli"))
20405 .expect("one-shot mlx-vlm schema");
20406 schema.param_count = "1M".into();
20407 schema.quantization = Some(Quantization::parse("Q4"));
20408 let mut reservation = engine.reserve_local_request(&schema, 64).unwrap();
20409 InferenceEngine::reconcile_transient_local_allocation(&mut reservation, 1024 * 1024)
20410 .unwrap();
20411 drop(reservation);
20412 assert!(!engine.local_admission.is_resident(&schema.id));
20413 }
20414
20415 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
20416 #[test]
20417 fn non_apple_backend_map_keeps_two_local_model_identities_exact() {
20418 let mut backends = std::collections::HashMap::<String, u32>::new();
20419 backends.insert("local/model-a".into(), 1);
20420 backends.insert("local/model-b".into(), 2);
20421 assert_eq!(backends.get("local/model-a"), Some(&1));
20422 assert_eq!(backends.get("local/model-b"), Some(&2));
20423 assert_eq!(backends.remove("local/model-a"), Some(1));
20424 assert!(!backends.contains_key("local/model-a"));
20425 assert_eq!(backends.get("local/model-b"), Some(&2));
20426 }
20427
20428 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
20429 #[tokio::test]
20430 async fn non_apple_remote_classification_at_zero_gb_routes_without_local_allocation() {
20431 use wiremock::matchers::{method, path};
20432 use wiremock::{Mock, MockServer, ResponseTemplate};
20433
20434 let _credential_scope = crate::openrouter::test_credential_scope();
20435 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
20436 let server = MockServer::start().await;
20437 Mock::given(method("POST"))
20438 .and(path("/v1/chat/completions"))
20439 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
20440 "choices": [{
20441 "message": {"role": "assistant", "content": "positive"},
20442 "finish_reason": "stop"
20443 }],
20444 "usage": {"prompt_tokens": 16, "completion_tokens": 1}
20445 })))
20446 .mount(&server)
20447 .await;
20448
20449 let fixture = tempfile::tempdir().unwrap();
20450 let config = test_config(fixture.path().join("weights"));
20451 let repository =
20452 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
20453 crate::resource_policy::ResourcePolicyRepository::save(
20454 &repository,
20455 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
20456 )
20457 .unwrap();
20458 let mut engine = InferenceEngine::new(config);
20459 let mut remote = crate::openrouter::curated_schemas()
20460 .into_iter()
20461 .next()
20462 .expect("OpenRouter schema");
20463 remote.id = "openrouter/test/classifier".into();
20464 remote.name = remote.id.clone();
20465 if let ModelSource::RemoteApi { endpoint, .. } = &mut remote.source {
20466 *endpoint = server.uri();
20467 }
20468 let remote_id = remote.id.clone();
20469 engine.unified_registry_mut().register_project_model(remote);
20470
20471 let result = engine
20472 .classify(ClassifyRequest {
20473 work_context: None,
20474 text: "a good outcome".into(),
20475 labels: vec!["positive".into(), "negative".into()],
20476 model: Some(remote_id),
20477 })
20478 .await
20479 .unwrap();
20480 assert_eq!(
20481 result.first().map(|item| item.label.as_str()),
20482 Some("positive")
20483 );
20484 assert_eq!(engine.local_admission.resident_model_mb(), 0);
20485 assert!(
20486 !engine.config.models_dir.exists()
20487 || std::fs::read_dir(&engine.config.models_dir)
20488 .unwrap()
20489 .next()
20490 .is_none(),
20491 "remote classification must not download local weights"
20492 );
20493 }
20494
20495 #[test]
20496 fn adaptive_speech_skips_blocked_local_but_explicit_speech_fails() {
20497 let fixture = TempDir::new().unwrap();
20498 let config = InferenceConfig {
20499 models_dir: fixture.path().join("weights"),
20500 state_root: fixture.path().join("state"),
20501 ..Default::default()
20502 };
20503 let repository =
20504 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
20505 crate::resource_policy::ResourcePolicyRepository::save(
20506 &repository,
20507 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
20508 )
20509 .unwrap();
20510 let engine = InferenceEngine::new(config);
20511 let local = engine
20512 .unified_registry()
20513 .all()
20514 .find(|schema| {
20515 schema.is_local() && schema.has_capability(ModelCapability::SpeechToText)
20516 })
20517 .cloned()
20518 .expect("built-in local STT model");
20519
20520 assert!(matches!(
20521 engine.admit_speech_candidate(&local, false),
20522 SpeechCandidateAdmission::SkipBlocked(_)
20523 ));
20524 assert!(matches!(
20525 engine.admit_speech_candidate(&local, true),
20526 SpeechCandidateAdmission::FailBlocked(InferenceError::LocalResourceBlocked { .. })
20527 ));
20528
20529 let mut os_owned = local;
20530 os_owned.id = "windows/speech-synthesis:test".into();
20531 os_owned.source = ModelSource::WindowsSpeech {};
20532 assert!(matches!(
20533 engine.admit_speech_candidate(&os_owned, true),
20534 SpeechCandidateAdmission::Proceed(None)
20535 ));
20536 }
20537
20538 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
20539 async fn cancelled_native_blocking_work_keeps_its_exact_charge_until_completion() {
20540 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
20541 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
20542 crate::hardware::HardwareInfo {
20543 os: "test".into(),
20544 arch: "test".into(),
20545 cpu_cores: 8,
20546 total_ram_mb: 32 * 1024,
20547 gpu_backend: crate::hardware::GpuBackend::Cpu,
20548 gpu_memory_mb: None,
20549 gpu_devices: Vec::new(),
20550 recommended_model: "fixture".into(),
20551 recommended_context: 4096,
20552 max_model_mb: 32 * 1024,
20553 },
20554 ));
20555 let reservation = coordinator
20556 .reserve_measured_host("detached-native-a", 512 * 1024 * 1024, 0)
20557 .unwrap();
20558 let lease = reservation.detached_lease();
20559 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
20560 let release =
20561 std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
20562 let release_worker = release.clone();
20563 let operation = tokio::spawn(async move {
20564 let _request_reservation = reservation;
20565 run_admitted_blocking(lease, move || {
20566 let _ = started_tx.send(());
20567 let (lock, ready) = &*release_worker;
20568 let mut released = lock
20569 .lock()
20570 .unwrap_or_else(std::sync::PoisonError::into_inner);
20571 while !*released {
20572 released = ready
20573 .wait(released)
20574 .unwrap_or_else(std::sync::PoisonError::into_inner);
20575 }
20576 })
20577 .await
20578 });
20579 started_rx.await.unwrap();
20580 operation.abort();
20581 let _ = operation.await;
20582
20583 let blocked = coordinator.reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0);
20584 assert!(
20585 blocked.is_err(),
20586 "cancelling the await must not advertise memory still owned by spawn_blocking"
20587 );
20588
20589 let (lock, ready) = &*release;
20590 *lock
20591 .lock()
20592 .unwrap_or_else(std::sync::PoisonError::into_inner) = true;
20593 ready.notify_one();
20594 tokio::time::timeout(std::time::Duration::from_secs(2), async {
20595 loop {
20596 if coordinator
20597 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
20598 .is_ok()
20599 {
20600 break;
20601 }
20602 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
20603 }
20604 })
20605 .await
20606 .expect("the detached charge must clear after native work really exits");
20607 }
20608
20609 #[test]
20610 fn nonstream_native_text_generation_uses_cancellable_blocking_boundary() {
20611 let source = include_str!("lib.rs");
20612 let mlx = source
20613 .split("async fn generate_mlx(")
20614 .nth(1)
20615 .and_then(|tail| tail.split("async fn generate_local(").next())
20616 .expect("generate_mlx source");
20617 let local = source
20618 .split("async fn generate_local(")
20619 .nth(1)
20620 .and_then(|tail| tail.split("fn apply_top_k_top_p").next())
20621 .expect("generate_local source");
20622
20623 assert!(
20624 mlx.contains("run_admitted_blocking"),
20625 "native MLX decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
20626 );
20627 assert!(
20628 local.contains("run_admitted_blocking"),
20629 "polymorphic local decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
20630 );
20631 }
20632
20633 #[cfg(unix)]
20634 #[tokio::test]
20635 async fn cancelled_speech_subprocess_is_killed_and_reaped_before_charge_clears() {
20636 use std::os::unix::fs::PermissionsExt;
20637
20638 let fixture = TempDir::new().unwrap();
20639 let runtime = SpeechRuntime::new(fixture.path().join("speech-runtime"));
20640 std::fs::create_dir_all(runtime.stt_program.parent().unwrap()).unwrap();
20641 std::fs::write(
20642 &runtime.stt_program,
20643 b"#!/bin/sh\necho started > \"$1\"\nsleep 1\necho continued > \"$2\"\n",
20644 )
20645 .unwrap();
20646 let mut permissions = std::fs::metadata(&runtime.stt_program)
20647 .unwrap()
20648 .permissions();
20649 permissions.set_mode(0o755);
20650 std::fs::set_permissions(&runtime.stt_program, permissions).unwrap();
20651
20652 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
20653 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
20654 crate::hardware::HardwareInfo {
20655 total_ram_mb: 32 * 1024,
20656 ..crate::hardware::HardwareInfo::detect()
20657 },
20658 ));
20659 let reservation = coordinator
20660 .reserve_measured_host("mlx-audio-a", 512 * 1024 * 1024, 0)
20661 .unwrap();
20662 let lease = reservation.detached_lease();
20663 let started = fixture.path().join("started");
20664 let continued = fixture.path().join("continued");
20665 let args = vec![
20666 started.display().to_string(),
20667 continued.display().to_string(),
20668 ];
20669 let command = tokio::spawn(async move {
20670 let _request_reservation = reservation;
20671 run_mlx_audio_command(&runtime, "stt.generate", &args, lease).await
20672 });
20673 tokio::time::timeout(std::time::Duration::from_secs(2), async {
20674 while !started.exists() {
20675 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
20676 }
20677 })
20678 .await
20679 .expect("speech fixture child must start");
20680 command.abort();
20681 let _ = command.await;
20682
20683 tokio::time::timeout(std::time::Duration::from_secs(2), async {
20684 loop {
20685 if coordinator
20686 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
20687 .is_ok()
20688 {
20689 break;
20690 }
20691 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
20692 }
20693 })
20694 .await
20695 .expect("charge clears only after the cancelled child is reaped");
20696 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
20697 assert!(
20698 !continued.exists(),
20699 "the cancelled local speech process must not keep allocating in the background"
20700 );
20701 }
20702
20703 #[test]
20704 fn local_model_eviction_surface_includes_every_in_process_cache() {
20705 let source = include_str!("lib.rs");
20706 for cache in [
20707 "self.text_backends.evict_if_idle(model_id)",
20708 "self.local_backends.evict_if_idle(model_id)",
20709 "self.flux_cache.evict_if_idle(model_id)",
20710 "self.ltx_cache.evict_if_idle(model_id)",
20711 "self.kokoro_cache.evict_if_idle(model_id)",
20712 "self.local_backends.evict_idle()",
20713 ] {
20714 assert!(source.contains(cache), "missing eviction seam: {cache}");
20715 }
20716 }
20717
20718 #[tokio::test]
20724 async fn catalog_refresh_uses_authority_hints_without_secret_reads() {
20725 if !crate::run_in_isolated_test_process(
20726 "tests::catalog_refresh_uses_authority_hints_without_secret_reads",
20727 "CAR_CATALOG_ZERO_SECRET_READ_CHILD",
20728 ) {
20729 return;
20730 }
20731 let _environment = ENV_MUTEX.lock().await;
20732 let fixture = TempDir::new().unwrap();
20733 let credential_envs = [
20734 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20735 car_auth::PARSLEE_API_BASE_KEY,
20736 crate::openrouter::API_KEY_ENV,
20737 "OPENAI_API_KEY",
20738 "ANTHROPIC_API_KEY",
20739 "GOOGLE_API_KEY",
20740 "ELEVENLABS_API_KEY",
20741 ];
20742 let mut restored_names = vec![car_home::ENV_VAR, "CAR_SECRETS_FILE_DIR"];
20743 restored_names.extend(credential_envs);
20744 let _restore = RestoredEnvironment::capture(&restored_names);
20745 unsafe {
20746 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
20747 std::env::set_var(
20748 "CAR_SECRETS_FILE_DIR",
20749 fixture.path().join("isolated-secrets"),
20750 );
20751 for name in credential_envs {
20752 std::env::remove_var(name);
20753 }
20754 }
20755
20756 let before = car_secrets::secret_store_activity();
20757 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
20758
20759 let listed = engine.list_models_unified();
20761 let schemas = engine.list_schemas();
20762 for provider in ["parslee", "openai", "anthropic", "google", "elevenlabs"] {
20763 assert!(
20764 schemas.iter().any(|schema| schema.provider == provider),
20765 "real builtin catalog lost the credential-bearing {provider} rows"
20766 );
20767 }
20768 assert!(
20769 schemas.iter().any(|schema| matches!(
20770 schema.source,
20771 ModelSource::RemoteApi {
20772 protocol: ApiProtocol::OpenRouter,
20773 ..
20774 }
20775 )),
20776 "real builtin catalog lost the reviewed OpenRouter rows"
20777 );
20778 assert_eq!(listed.len(), schemas.len());
20779
20780 let schema_refs: Vec<&ModelSchema> = schemas.iter().collect();
20782 let _setup_plan = crate::recommend(
20783 &schema_refs,
20784 &HardwareInfo::detect(),
20785 UseCase::default(),
20786 QualityTier::default(),
20787 Privacy::OnDevice,
20788 &CheckpointGeometry::default(),
20789 );
20790
20791 let _speech = engine.speech_health();
20793 let _concierge = engine.concierge_status(false).await;
20794 let _health = engine.model_health().await;
20795
20796 let after = car_secrets::secret_store_activity();
20797 assert_eq!(
20798 after, before,
20799 "passive builtin catalog surfaces performed secret-store operations"
20800 );
20801 }
20802
20803 #[tokio::test]
20804 async fn list_models_unified_and_model_health_are_zero_secret_store_probes() {
20805 if !crate::run_in_isolated_test_process(
20806 "tests::list_models_unified_and_model_health_are_zero_secret_store_probes",
20807 "CAR_DIRECT_MODEL_SURFACES_ZERO_SECRET_CHILD",
20808 ) {
20809 return;
20810 }
20811 let _environment = ENV_MUTEX.lock().await;
20812 let fixture = TempDir::new().unwrap();
20813 let _restore = RestoredEnvironment::capture(&[
20814 car_home::ENV_VAR,
20815 "CAR_SECRETS_FILE_DIR",
20816 crate::openrouter::API_KEY_ENV,
20817 ]);
20818 unsafe {
20819 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
20820 std::env::set_var(
20821 "CAR_SECRETS_FILE_DIR",
20822 fixture.path().join("isolated-secrets"),
20823 );
20824 std::env::remove_var(crate::openrouter::API_KEY_ENV);
20825 }
20826 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
20827 let before = car_secrets::secret_store_activity();
20828
20829 let rows = engine.list_models_unified();
20830 let health = engine.model_health().await;
20831
20832 assert!(!rows.is_empty(), "the unified catalog fixture must exist");
20833 assert!(health.total_models > 0, "the health fixture must exist");
20834 let after = car_secrets::secret_store_activity();
20835 assert_eq!(after.status_attempts, before.status_attempts);
20836 assert_eq!(after.get_attempts, before.get_attempts);
20837
20838 assert!(
20848 rows.iter().all(|row| row.credential_required.is_none()),
20849 "a passive catalog read must not populate `credential_required` — its \
20850 availability verdict cannot see the secret store, so any credential it \
20851 named might already be present: {:?}",
20852 rows.iter()
20853 .filter(|row| row.credential_required.is_some())
20854 .map(|row| (&row.id, &row.credential_required))
20855 .collect::<Vec<_>>()
20856 );
20857 }
20858
20859 #[test]
20869 fn car_home_moves_the_catalog_and_discovery_caches_but_never_the_weights() {
20870 let _environment = crate::openrouter::test_environment_scope();
20871 let prior = std::env::var_os(car_home::ENV_VAR);
20872
20873 unsafe { std::env::remove_var(car_home::ENV_VAR) };
20874 let shared = InferenceConfig::default();
20875 let default_catalog = crate::catalog::cache_path(&shared.state_root);
20876 let default_discovery = crate::discovery::cache_path(&shared.state_models_dir());
20877
20878 let alt = Path::new("/tmp/car-home-inference-cache-test");
20879 unsafe { std::env::set_var(car_home::ENV_VAR, alt) };
20880 let relocated = InferenceConfig::default();
20881 let catalog = crate::catalog::cache_path(&relocated.state_root);
20882 let discovery = crate::discovery::cache_path(&relocated.state_models_dir());
20883
20884 match prior {
20885 Some(value) => unsafe { std::env::set_var(car_home::ENV_VAR, value) },
20886 None => unsafe { std::env::remove_var(car_home::ENV_VAR) },
20887 }
20888
20889 assert_eq!(catalog, alt.join(crate::catalog::CATALOG_CACHE_FILE));
20890 assert_eq!(
20891 discovery,
20892 alt.join("models")
20893 .join(crate::discovery::DISCOVERED_MODELS_FILE)
20894 );
20895 assert_ne!(
20896 catalog, default_catalog,
20897 "the catalog cache must not resolve back into the shared root",
20898 );
20899 assert_ne!(
20900 discovery, default_discovery,
20901 "the discovery cache must not resolve back into the shared root",
20902 );
20903
20904 assert_eq!(
20905 relocated.models_dir, shared.models_dir,
20906 "the weights cache is machine-global and must not follow CAR_HOME",
20907 );
20908 assert!(
20909 !relocated.models_dir.starts_with(alt),
20910 "the weights cache must not be dragged under the override",
20911 );
20912 }
20913
20914 #[derive(Clone, Copy)]
20915 enum CacheRoutingSurface {
20916 Generate,
20917 Stream,
20918 }
20919
20920 async fn invoke_cache_routed_openrouter(
20926 surface: CacheRoutingSurface,
20927 cache_read_estimate: usize,
20928 cache_write_estimate: usize,
20929 ) -> String {
20930 use wiremock::matchers::{method, path};
20931 use wiremock::{Mock, MockServer, ResponseTemplate};
20932
20933 let _credential_scope = crate::openrouter::test_credential_scope();
20934 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
20935 let server = MockServer::start().await;
20936 let response = match surface {
20937 CacheRoutingSurface::Generate => ResponseTemplate::new(200).set_body_json(
20938 serde_json::json!({
20939 "choices": [{
20940 "message": {"role": "assistant", "content": "cache-route-ok"},
20941 "finish_reason": "stop"
20942 }],
20943 "usage": {"prompt_tokens": 40_000, "completion_tokens": 8}
20944 }),
20945 ),
20946 CacheRoutingSurface::Stream => ResponseTemplate::new(200).set_body_raw(
20947 concat!(
20948 "data: {\"choices\":[{\"delta\":{\"content\":\"cache-route-ok\"},\"finish_reason\":\"stop\"}]}\n\n",
20949 "data: [DONE]\n\n"
20950 ),
20951 "text/event-stream",
20952 ),
20953 };
20954 Mock::given(method("POST"))
20955 .and(path("/v1/chat/completions"))
20956 .respond_with(response)
20957 .mount(&server)
20958 .await;
20959
20960 let tmp = TempDir::new().unwrap();
20961 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20962 let mut uncached_cheap = crate::openrouter::curated_schemas()
20963 .into_iter()
20964 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
20965 .unwrap();
20966 uncached_cheap.id = "openrouter/test/uncached-cheap".into();
20967 uncached_cheap.name = uncached_cheap.id.clone();
20968 uncached_cheap.cost = CostModel {
20969 input_per_mtok: Some(1.0),
20970 output_per_mtok: Some(1.0),
20971 cache_read_input_per_mtok: Some(100.0),
20972 cache_write_input_per_mtok: Some(100.0),
20973 ..Default::default()
20974 };
20975 if let ModelSource::RemoteApi { endpoint, .. } = &mut uncached_cheap.source {
20976 *endpoint = server.uri();
20977 }
20978
20979 let mut cached_cheap = uncached_cheap.clone();
20980 cached_cheap.id = "openrouter/test/cached-cheap".into();
20981 cached_cheap.name = cached_cheap.id.clone();
20982 cached_cheap.cost = CostModel {
20983 input_per_mtok: Some(80.0),
20984 output_per_mtok: Some(1.0),
20985 cache_read_input_per_mtok: Some(0.001),
20986 cache_write_input_per_mtok: Some(0.001),
20987 ..Default::default()
20988 };
20989
20990 let uncached_id = uncached_cheap.id.clone();
20991 let cached_id = cached_cheap.id.clone();
20992 engine
20993 .unified_registry_mut()
20994 .register_project_model(uncached_cheap);
20995 engine
20996 .unified_registry_mut()
20997 .register_project_model(cached_cheap);
20998 let exclude_models = engine
20999 .list_schemas()
21000 .into_iter()
21001 .map(|schema| schema.id)
21002 .filter(|id| id != &uncached_id && id != &cached_id)
21003 .collect();
21004
21005 let mut params = GenerateParams {
21009 max_tokens: 8,
21010 ..Default::default()
21011 };
21012 assert_eq!(params.estimated_cache_read_input_tokens, 0);
21013 assert_eq!(params.estimated_cache_write_input_tokens, 0);
21014 params.estimated_cache_read_input_tokens = cache_read_estimate;
21015 params.estimated_cache_write_input_tokens = cache_write_estimate;
21016 let req = GenerateRequest {
21017 prompt: "x".repeat(160_000),
21018 params,
21019 cache_control: true,
21020 intent: Some(IntentHint {
21021 prefer_quality: true,
21022 exclude_models,
21023 ..Default::default()
21024 }),
21025 ..Default::default()
21026 };
21027
21028 match surface {
21029 CacheRoutingSurface::Generate => {
21030 engine
21031 .generate_tracked(req)
21032 .await
21033 .expect("mocked OpenRouter generation should succeed")
21034 .model_used
21035 }
21036 CacheRoutingSurface::Stream => {
21037 let mut handle = engine
21038 .generate_tracked_stream(req)
21039 .await
21040 .expect("mocked OpenRouter stream should start");
21041 let selected = handle.model_used.clone();
21042 while handle.events.recv().await.is_some() {}
21043 selected
21044 }
21045 }
21046 }
21047
21048 #[tokio::test(flavor = "current_thread")]
21049 async fn tracked_generate_uses_explicit_cache_estimate_and_defaults_to_zero() {
21050 let without_estimate =
21051 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 0, 0).await;
21052 let with_estimate =
21053 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 40_000, 0).await;
21054 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
21055 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
21056 }
21057
21058 #[tokio::test(flavor = "current_thread")]
21059 async fn tracked_stream_uses_explicit_cache_estimate_and_defaults_to_zero() {
21060 let without_estimate =
21061 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 0).await;
21062 let with_estimate =
21063 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 40_000).await;
21064 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
21065 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
21066 }
21067
21068 #[tokio::test(flavor = "current_thread")]
21069 async fn authenticated_openrouter_registry_stays_static_and_rejects_unknown_ids() {
21070 let _credential_scope = crate::openrouter::test_credential_scope();
21071 crate::openrouter::set_test_credential(Some("static-key"));
21072 let tmp = TempDir::new().unwrap();
21073 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
21074 let personal: Vec<_> = engine
21075 .list_schemas()
21076 .into_iter()
21077 .filter(|schema| schema.id.starts_with("openrouter/"))
21078 .collect();
21079 assert_eq!(personal.len(), crate::openrouter::curated_model_count());
21080 assert!(personal.iter().all(ModelSchema::available_now));
21081 assert!(personal
21082 .iter()
21083 .all(|schema| schema.trust_tier == TrustTier::Curated));
21084 assert!(personal
21085 .iter()
21086 .all(|schema| !schema.tags.iter().any(|tag| tag == "dynamic")));
21087
21088 for unknown in [
21089 "openrouter/vendor/brand-new-model",
21090 "openrouter/openai/gpt-5.4-typo",
21091 ] {
21092 assert!(engine
21093 .list_schemas()
21094 .iter()
21095 .all(|schema| schema.id != unknown));
21096 assert_eq!(engine.model_context_window(unknown), 0);
21097 let error = engine
21098 .generate_tracked(GenerateRequest {
21099 prompt: "must fail before transport".into(),
21100 model: Some(unknown.into()),
21101 params: GenerateParams {
21102 strict_model: true,
21103 ..Default::default()
21104 },
21105 ..Default::default()
21106 })
21107 .await
21108 .expect_err("unregistered personal OpenRouter ids must not reach inference");
21109 assert!(
21110 matches!(&error, InferenceError::ModelNotFound(id) if id == unknown),
21111 "{unknown}: {error}"
21112 );
21113 let stream_error = match engine
21114 .generate_tracked_stream(GenerateRequest {
21115 prompt: "must fail before stream transport".into(),
21116 model: Some(unknown.into()),
21117 ..Default::default()
21118 })
21119 .await
21120 {
21121 Ok(_) => panic!("unregistered ids must also fail before streaming"),
21122 Err(error) => error,
21123 };
21124 assert!(
21125 matches!(&stream_error, InferenceError::ModelNotFound(id) if id == unknown),
21126 "{unknown}: {stream_error}"
21127 );
21128 }
21129 }
21130
21131 #[tokio::test(flavor = "current_thread")]
21132 async fn static_openrouter_rows_participate_in_adaptive_routing_only_with_a_key() {
21133 let (_credential_scope, _provider_env) =
21134 crate::openrouter::test_credential_and_environment_scope_async().await;
21135 let tmp = TempDir::new().unwrap();
21136 crate::openrouter::set_test_credential(Some("static-key"));
21137 unsafe {
21138 std::env::set_var("CAR_STATIC_ROUTING_PEER_KEY", "peer-key");
21139 }
21140 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
21141 let mut peer = remote_stream_fixture_schema(
21142 "test/openai-routing-peer",
21143 "http://127.0.0.1:9".into(),
21144 schema::ApiProtocol::OpenAiCompat,
21145 "CAR_STATIC_ROUTING_PEER_KEY",
21146 );
21147 peer.provider = "openai".into();
21148 peer.trust_tier = TrustTier::Curated;
21149 engine.unified_registry_mut().register_project_model(peer);
21150 let reviewed: std::collections::HashSet<_> = engine
21151 .list_schemas()
21152 .into_iter()
21153 .filter(|schema| schema.id.starts_with("openrouter/"))
21154 .map(|schema| schema.id)
21155 .collect();
21156 assert_eq!(reviewed.len(), crate::openrouter::curated_model_count());
21157
21158 let with_key = engine
21159 .route_adaptive_with_intent(
21160 "Answer this simple question cheaply.",
21161 Some(IntentHint::default()),
21162 )
21163 .await;
21164 let openrouter_candidates: Vec<_> = std::iter::once(with_key.model_id.as_str())
21165 .chain(
21166 with_key
21167 .candidates
21168 .iter()
21169 .map(|candidate| candidate.model_id.as_str()),
21170 )
21171 .chain(with_key.fallbacks.iter().map(String::as_str))
21172 .filter(|id| id.starts_with("openrouter/"))
21173 .collect();
21174 assert!(
21175 !openrouter_candidates.is_empty(),
21176 "keyed adaptive decision must include a reviewed OpenRouter row: {with_key:?}"
21177 );
21178 assert!(openrouter_candidates
21179 .iter()
21180 .all(|id| reviewed.contains(*id)));
21181 assert!(
21182 std::iter::once(with_key.model_id.as_str())
21183 .chain(with_key.fallbacks.iter().map(String::as_str),)
21184 .any(|id| !id.starts_with("openrouter/")),
21185 "fallback chain must retain cross-provider alternatives: {with_key:?}"
21186 );
21187
21188 crate::openrouter::set_test_credential(None);
21189 let without_key = engine
21190 .route_adaptive_with_intent(
21191 "Answer this simple question cheaply.",
21192 Some(IntentHint::default()),
21193 )
21194 .await;
21195 assert!(!std::iter::once(without_key.model_id.as_str())
21196 .chain(
21197 without_key
21198 .candidates
21199 .iter()
21200 .map(|candidate| candidate.model_id.as_str()),
21201 )
21202 .chain(without_key.fallbacks.iter().map(String::as_str))
21203 .any(|id| id.starts_with("openrouter/")));
21204 unsafe {
21205 std::env::remove_var("CAR_STATIC_ROUTING_PEER_KEY");
21206 }
21207 }
21208
21209 #[tokio::test(flavor = "current_thread")]
21210 async fn v2_parslee_auth_drives_managed_registration_routing_lane_and_logout() {
21211 if !crate::run_in_isolated_test_process(
21212 "tests::v2_parslee_auth_drives_managed_registration_routing_lane_and_logout",
21213 "CAR_V2_ROUTING_AUTH_CHILD",
21214 ) {
21215 return;
21216 }
21217 let tmp = TempDir::new().unwrap();
21218 let (_credential_scope, _provider_env) =
21219 crate::openrouter::test_credential_and_environment_scope_async().await;
21220 let _restore = RestoredEnvironment::capture(&[
21221 "CAR_SECRETS_FILE_DIR",
21222 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
21223 ]);
21224 let _home = crate::openrouter::StateRootScope::new();
21231 crate::openrouter::clear_gateway_unconfigured();
21232 crate::parslee_credential::clear_credential_rejected();
21233 let secrets_dir = tmp.path().join("secrets");
21234 unsafe {
21235 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
21236 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
21237 }
21238 crate::openrouter::set_test_credential(Some("personal-openrouter-key"));
21239
21240 let store = car_secrets::SecretStore::new();
21241 let state_ref =
21242 car_secrets::SecretRef::with_default_service(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
21243 store
21244 .publish(
21245 &state_ref,
21246 &serde_json::json!({
21247 "schema": 2,
21248 "revision": 7,
21249 "generation": 3,
21250 "active": {
21251 "account_id": "account-v2",
21252 "access_token": "v2-access",
21253 "refresh_token": "v2-refresh",
21254 "expires_at": 9_999_999_999_u64,
21255 "api_base": "https://api.parslee.ai"
21256 },
21257 "accounts": [{
21258 "account_id": "account-v2",
21259 "access_token": "v2-access",
21260 "refresh_token": "v2-refresh",
21261 "expires_at": 9_999_999_999_u64,
21262 "api_base": "https://api.parslee.ai"
21263 }],
21264 "tombstone": false
21265 })
21266 .to_string(),
21267 )
21268 .unwrap();
21269 assert_eq!(car_auth::access_token().as_deref(), Some("v2-access"));
21270 assert!(
21271 !store
21272 .status(&car_secrets::SecretRef::with_default_service(
21273 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
21274 ))
21275 .unwrap()
21276 .exists,
21277 "the regression must exercise V2 state without the import-only legacy slot"
21278 );
21279
21280 let managed_ids = [
21281 "parslee/openrouter/frontier-general",
21282 "parslee/openrouter/balanced-general",
21283 ];
21284 let personal_id = "openrouter/deepseek/deepseek-v3.2";
21285 let rogue_id = "community/custom-oauth";
21286 let schemas = crate::openrouter::curated_schemas();
21287 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("registry-models"));
21288 for id in managed_ids.into_iter().chain(std::iter::once(personal_id)) {
21289 registry.register_project_model(
21290 schemas
21291 .iter()
21292 .find(|schema| schema.id == id)
21293 .unwrap_or_else(|| panic!("missing curated schema {id}"))
21294 .clone(),
21295 );
21296 }
21297 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
21298 for id in managed_ids {
21299 assert!(
21300 registry.get(id).unwrap().available_now(),
21301 "{id} must be available immediately when registered from V2 auth"
21302 );
21303 }
21304 assert!(registry.get(personal_id).unwrap().available_now());
21305 let mut rogue = schemas
21306 .iter()
21307 .find(|schema| schema.id == managed_ids[0])
21308 .unwrap()
21309 .clone();
21310 rogue.id = rogue_id.into();
21311 rogue.provider = "community".into();
21312 if let ModelSource::Proprietary {
21313 provider, endpoint, ..
21314 } = &mut rogue.source
21315 {
21316 *provider = "community".into();
21317 *endpoint = "https://untrusted.example".into();
21318 } else {
21319 panic!("managed fixture must remain proprietary");
21320 }
21321 registry.register(rogue);
21322 assert!(
21323 !registry.get(rogue_id).unwrap().available_now(),
21324 "a non-Parslee OAuth schema must not inherit Parslee V2 availability"
21325 );
21326
21327 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
21328 for id in managed_ids {
21329 assert!(
21330 registry.get(id).unwrap().available_now(),
21331 "{id} must stay available in a refreshed V2-auth snapshot"
21332 );
21333 }
21334 assert!(
21335 !registry.get(rogue_id).unwrap().available_now(),
21336 "refresh must keep non-Parslee OAuth schemas unavailable"
21337 );
21338
21339 let router = AdaptiveRouter::new(
21340 crate::hardware::HardwareInfo::detect(),
21341 RoutingConfig {
21342 prefer_local: false,
21343 prior_strength: 1_000_000.0,
21344 quality_first_cold_start: false,
21345 ..RoutingConfig::default()
21346 },
21347 );
21348 let tracker = OutcomeTracker::new();
21349 let intent = IntentHint {
21350 task: Some(crate::intent::TaskHint::Chat),
21351 exclude_models: vec![personal_id.into()],
21352 ..Default::default()
21353 };
21354 let decision = router.route_with(crate::adaptive_router::RouteRequest {
21355 intent: Some(&intent),
21356 ..crate::adaptive_router::RouteRequest::new(
21357 "Explain this architecture.",
21358 ®istry,
21359 &tracker,
21360 )
21361 });
21362 assert!(
21363 managed_ids.contains(&decision.model_id.as_str()),
21364 "managed V2-auth alias must be selectable: {decision:?}"
21365 );
21366 assert!(
21367 decision
21368 .candidates
21369 .iter()
21370 .any(|candidate| managed_ids.contains(&candidate.model_id.as_str())),
21371 "managed V2-auth alias must appear in adaptive candidates: {decision:?}"
21372 );
21373 assert!(
21374 decision
21375 .fallbacks
21376 .iter()
21377 .any(|id| managed_ids.contains(&id.as_str())),
21378 "managed V2-auth alias must appear in adaptive fallbacks: {decision:?}"
21379 );
21380 assert!(
21381 !std::iter::once(decision.model_id.as_str())
21382 .chain(
21383 decision
21384 .candidates
21385 .iter()
21386 .map(|candidate| candidate.model_id.as_str())
21387 )
21388 .chain(decision.fallbacks.iter().map(String::as_str))
21389 .any(|id| id == rogue_id),
21390 "a non-Parslee OAuth schema must never enter adaptive selection, candidates, or fallbacks: {decision:?}"
21391 );
21392
21393 let engine = InferenceEngine::new(test_config(tmp.path().join("engine-models")));
21394 let managed_lane_id = managed_ids[0];
21395 engine.lane_defaults_cache.write().unwrap().set(
21396 None,
21397 crate::intent::UseCase::Assistant,
21398 managed_lane_id.into(),
21399 1,
21400 );
21401 let request = GenerateRequest {
21402 prompt: "lane default".into(),
21403 intent: Some(IntentHint {
21404 task: Some(crate::intent::TaskHint::Chat),
21405 ..Default::default()
21406 }),
21407 ..Default::default()
21408 };
21409 assert_eq!(
21410 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
21411 Some(managed_lane_id.to_string()),
21412 "a managed lane default must resolve from authoritative V2 auth"
21413 );
21414
21415 car_auth::logout()
21416 .await
21417 .expect("production logout must publish a signed-out tombstone");
21418 let persisted_logout: serde_json::Value =
21419 serde_json::from_str(&store.get(&state_ref).unwrap()).unwrap();
21420 assert_eq!(persisted_logout["tombstone"], true);
21421 assert_eq!(persisted_logout["accounts"], serde_json::json!([]));
21422 assert!(persisted_logout.get("active").is_none());
21423 assert_eq!(car_auth::access_token(), None);
21424
21425 registry.refresh_routing_availability(None, true);
21426 for id in managed_ids {
21427 assert!(
21428 !registry.get(id).unwrap().available_now(),
21429 "{id} must disappear from routing after the signed-out tombstone"
21430 );
21431 }
21432 assert!(
21433 registry.get(personal_id).unwrap().available_now(),
21434 "personal rows must continue to follow their independent personal-key seam"
21435 );
21436
21437 let signed_out = engine.routing_registry_snapshot().await;
21438 assert!(
21439 signed_out
21440 .list()
21441 .into_iter()
21442 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
21443 .all(|schema| !schema.available_now()),
21444 "the next engine snapshot must exclude every managed alias after logout"
21445 );
21446 assert!(
21447 signed_out.get(personal_id).unwrap().available_now(),
21448 "logout must not disable a still-keyed personal OpenRouter row"
21449 );
21450 assert_eq!(
21451 engine.lane_pin_for(&request, &signed_out),
21452 None,
21453 "a signed-out managed lane default must stop resolving"
21454 );
21455
21456 crate::openrouter::set_test_credential(None);
21457 registry.refresh_availability();
21458 assert!(
21459 !registry.get(personal_id).unwrap().available_now(),
21460 "personal row availability must still turn off with the personal-key seam"
21461 );
21462 }
21463
21464 #[tokio::test(flavor = "current_thread")]
21465 async fn legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates() {
21466 if !crate::run_in_isolated_test_process(
21467 "tests::legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates",
21468 "CAR_LEGACY_V2_MIGRATION_CHILD",
21469 ) {
21470 return;
21471 }
21472 let tmp = TempDir::new().unwrap();
21473 let _provider_env = crate::openrouter::test_environment_scope_async().await;
21474 let _restore = RestoredEnvironment::capture(&[
21475 "CAR_SECRETS_FILE_DIR",
21476 car_home::ENV_VAR,
21477 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
21478 ]);
21479 let _home = crate::openrouter::StateRootScope::new();
21494 crate::openrouter::clear_gateway_unconfigured();
21495 crate::parslee_credential::clear_credential_rejected();
21496 unsafe {
21497 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
21498 std::env::set_var(car_home::ENV_VAR, tmp.path().join("car-home"));
21499 }
21500
21501 let secret = |key| car_secrets::SecretRef::with_default_service(key);
21502 let seed_legacy = |store: &car_secrets::SecretStore| {
21503 store
21504 .put(
21505 &secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY),
21506 "legacy-access",
21507 )
21508 .unwrap();
21509 store
21510 .put(
21511 &secret(car_secrets::PARSLEE_ACTIVE_ACCOUNT_ID_KEY),
21512 "legacy-account",
21513 )
21514 .unwrap();
21515 store.put(
21516 &secret(car_secrets::PARSLEE_ACCOUNTS_KEY),
21517 r#"{"active":"legacy-account","accounts":[{"id":"legacy-account","email":"legacy@example.test"}]}"#,
21518 )
21519 .unwrap();
21520 };
21521 let state_ref = secret(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
21522 let managed = crate::openrouter::curated_schemas()
21523 .into_iter()
21524 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
21525 .unwrap();
21526
21527 unsafe {
21528 std::env::set_var("CAR_SECRETS_FILE_DIR", tmp.path().join("tombstone-secrets"));
21529 }
21530 let tombstoned_store = car_secrets::SecretStore::new();
21531 seed_legacy(&tombstoned_store);
21532 tombstoned_store
21533 .publish(
21534 &state_ref,
21535 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
21536 )
21537 .unwrap();
21538 let mut tombstoned_registry =
21539 UnifiedRegistry::new_empty(tmp.path().join("tombstoned-models"));
21540 tombstoned_registry.register_project_model(managed.clone());
21541 assert!(
21542 !tombstoned_registry
21543 .get(&managed.id)
21544 .unwrap()
21545 .available_now(),
21546 "passive catalog registration must remain disabled without a configured hint"
21547 );
21548 let tombstoned = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
21549 .await
21550 .unwrap();
21551 assert!(tombstoned.is_none(), "the V2 tombstone is authoritative");
21552 tombstoned_registry.refresh_routing_availability(None, true);
21553 assert!(!tombstoned_registry
21554 .get(&managed.id)
21555 .unwrap()
21556 .available_now());
21557
21558 unsafe {
21559 std::env::set_var(
21560 "CAR_SECRETS_FILE_DIR",
21561 tmp.path().join("legacy-only-secrets"),
21562 );
21563 }
21564 let legacy_store = car_secrets::SecretStore::new();
21565 seed_legacy(&legacy_store);
21566 assert!(!legacy_store.status(&state_ref).unwrap().exists);
21567 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("legacy-models"));
21568 registry.register_project_model(managed.clone());
21569 assert!(
21570 !registry.get(&managed.id).unwrap().available_now(),
21571 "passive registration must not inspect attributable legacy slots"
21572 );
21573 registry.refresh_availability();
21574 assert!(
21575 !registry.get(&managed.id).unwrap().available_now(),
21576 "passive refresh must remain secret-store free until request-time migration"
21577 );
21578 let resolved = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
21579 .await
21580 .unwrap()
21581 .expect("request-time auth reconciliation must migrate attributable legacy state");
21582 registry.refresh_routing_availability(Some(&resolved.api_base), false);
21583 assert!(registry.get(&managed.id).unwrap().available_now());
21584 assert!(
21585 legacy_store.status(&state_ref).unwrap().exists,
21586 "request-time auth reconciliation must publish the migrated V2 record"
21587 );
21588 assert!(
21589 !legacy_store
21590 .status(&secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY))
21591 .unwrap()
21592 .exists,
21593 "successful V2 migration must clean the legacy access slot"
21594 );
21595 }
21596
21597 #[tokio::test]
21598 async fn reviewed_openrouter_lane_default_tracks_live_credential_availability() {
21599 let _credential_scope = crate::openrouter::test_credential_scope();
21600 crate::openrouter::set_test_credential(Some("static-key"));
21601 let tmp = TempDir::new().unwrap();
21602 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
21603 let reviewed_id = "openrouter/deepseek/deepseek-v3.2";
21604 engine.lane_defaults_cache.write().unwrap().set(
21605 None,
21606 crate::intent::UseCase::Assistant,
21607 reviewed_id.into(),
21608 1,
21609 );
21610 let request = GenerateRequest {
21611 prompt: "lane default".into(),
21612 intent: Some(IntentHint {
21613 task: Some(crate::intent::TaskHint::Chat),
21614 ..Default::default()
21615 }),
21616 ..Default::default()
21617 };
21618 assert_eq!(
21619 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
21620 Some(reviewed_id.to_string()),
21621 "a reviewed keyed row must be eligible as a lane default"
21622 );
21623 crate::openrouter::set_test_credential(None);
21624 assert_eq!(
21625 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
21626 None,
21627 "the same static lane default must stop being eligible immediately after key removal"
21628 );
21629 }
21630
21631 #[tokio::test(flavor = "current_thread")]
21632 async fn personal_openrouter_baseline_remains_visible_but_disabled_without_a_credential() {
21633 let _credential_scope = crate::openrouter::test_credential_scope();
21634 crate::openrouter::set_test_credential(None);
21635 let tmp = TempDir::new().unwrap();
21636 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
21637
21638 let personal: Vec<_> = engine
21639 .list_schemas()
21640 .into_iter()
21641 .filter(|schema| schema.id.starts_with("openrouter/"))
21642 .collect();
21643 assert_eq!(
21644 personal.len(),
21645 crate::openrouter::curated_model_count(),
21646 "the vetted personal rows stay discoverable as a disabled baseline"
21647 );
21648 assert!(
21649 personal.iter().all(|schema| !schema.available_now()),
21650 "no-key baseline rows must never become routing candidates"
21651 );
21652
21653 let error = engine
21654 .generate_tracked(GenerateRequest {
21655 prompt: "hello".into(),
21656 model: Some("openrouter/openai/gpt-5.4".into()),
21657 params: GenerateParams {
21658 strict_model: true,
21659 ..Default::default()
21660 },
21661 ..Default::default()
21662 })
21663 .await
21664 .expect_err("explicitly selecting a disabled baseline row must be actionable");
21665 let message = error.to_string();
21666 assert!(
21667 message.contains("car keys set openrouter") && message.contains("CarHost"),
21668 "disabled personal row must explain how to connect OpenRouter: {message}"
21669 );
21670 }
21671
21672 #[tokio::test]
21673 async fn reviewed_openrouter_metadata_stays_static_across_credential_changes() {
21674 let _credential_scope = crate::openrouter::test_credential_scope();
21675 crate::openrouter::set_test_credential(Some("static-key"));
21676
21677 let tmp = TempDir::new().unwrap();
21678 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
21679 let live = engine.routing_registry_snapshot().await;
21680 let vetted = live.get("openrouter/openai/gpt-5.4").unwrap();
21681 assert!(vetted.available_now());
21682 assert_eq!(vetted.context_length, 1_050_000);
21683 assert_eq!(vetted.max_output_tokens, Some(128_000));
21684 assert_eq!(vetted.cost.input_per_mtok, Some(2.5));
21685 assert_eq!(vetted.cost.output_per_mtok, Some(15.0));
21686 assert!(
21687 !vetted.cost.pricing_tiers.is_empty(),
21688 "reviewed high-context pricing tiers must remain in the static row"
21689 );
21690 assert_eq!(vetted.trust_tier, TrustTier::Curated);
21691 assert!(vetted.tags.iter().any(|tag| tag == "frontier"));
21692 assert!(vetted.has_capability(ModelCapability::Code));
21693 assert!(vetted
21694 .supported_params
21695 .contains(&schema::GenerateParam::ExtendedThinking));
21696
21697 assert!(live.get("openrouter/vendor/unreviewed-model").is_none());
21698
21699 crate::openrouter::set_test_credential(None);
21700 let reverted = engine.routing_registry_snapshot().await;
21701 assert!(reverted.get("openrouter/vendor/unreviewed-model").is_none());
21702 let baseline = reverted.get("openrouter/openai/gpt-5.4").unwrap();
21703 assert!(
21704 !baseline.available_now(),
21705 "credential removal must disable the reviewed row without removing it"
21706 );
21707 }
21708
21709 #[test]
21728 fn model_pulled_at_runtime_is_ready_without_a_daemon_restart() {
21729 let tmp = TempDir::new().unwrap();
21730 let models_dir = tmp.path().join("models");
21731 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
21732 engine.register_model(ModelSchema {
21733 id: "mlx/pulled-later".into(),
21734 name: "PulledLater".into(),
21735 provider: "local".into(),
21736 family: "qwen3".into(),
21737 version: "test".into(),
21738 capabilities: vec![ModelCapability::Generate, ModelCapability::Code],
21739 context_length: 4096,
21740 max_output_tokens: None,
21741 param_count: String::new(),
21742 quantization: None,
21743 performance: schema::PerformanceEnvelope::default(),
21744 cost: schema::CostModel::default(),
21745 source: ModelSource::Mlx {
21746 hf_repo: "example/pulled-later".into(),
21747 hf_weight_file: None,
21748 },
21749 tags: vec![],
21750 supported_params: vec![],
21751 public_benchmarks: vec![],
21752 trust_tier: TrustTier::Curated,
21753 deprecated: false,
21754 available: true,
21755 weights_ready: false,
21756 });
21757
21758 let engine = &engine;
21760 let is_ready = || {
21761 engine
21762 .list_schemas()
21763 .into_iter()
21764 .find(|s| s.id == "mlx/pulled-later")
21765 .expect("registered model must be listed")
21766 .weights_ready
21767 };
21768
21769 assert!(!is_ready(), "precondition: no weights on disk yet");
21770
21771 let dir = models_dir.join("PulledLater");
21774 std::fs::create_dir_all(&dir).unwrap();
21775 std::fs::write(dir.join("config.json"), "{}").unwrap();
21776 std::fs::write(dir.join("model.safetensors"), b"weights").unwrap();
21777
21778 assert!(
21779 is_ready(),
21780 "a model pulled at runtime must be ready without restarting the daemon"
21781 );
21782
21783 std::fs::remove_file(dir.join("model.safetensors")).unwrap();
21786 assert!(
21787 !is_ready(),
21788 "readiness must track the disk in both directions, not latch"
21789 );
21790 }
21791
21792 #[tokio::test(flavor = "current_thread")]
21793 async fn missing_openrouter_key_is_typed_without_changing_its_string_contract() {
21794 let (_credential_scope, _provider_environment) =
21798 crate::openrouter::test_credential_and_environment_scope_async().await;
21799 let home = crate::openrouter::StateRootScope::new();
21800 let _restore = RestoredEnvironment::capture(&["CAR_SECRETS_FILE_DIR"]);
21801 unsafe {
21802 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
21803 }
21804
21805 crate::openrouter::set_test_credential(None);
21809
21810 let models_dir = home.path().join("models");
21811 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
21812 let schema = crate::openrouter::curated_schemas()
21813 .into_iter()
21814 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
21815 .expect("personal OpenRouter fixture");
21816 let model_id = schema.id.clone();
21817 engine.unified_registry_mut().register_project_model(schema);
21818
21819 let error = engine
21820 .generate_tracked(GenerateRequest {
21821 prompt: "must stop before provider dispatch".into(),
21822 model: Some(model_id.clone()),
21823 params: GenerateParams {
21824 strict_model: true,
21825 ..Default::default()
21826 },
21827 ..Default::default()
21828 })
21829 .await
21830 .expect_err("an OpenRouter route without a key must fail before dispatch");
21831
21832 let tracker = engine.outcome_tracker();
21833 assert!(
21834 tracker.read().await.profile(&model_id).is_none(),
21835 "a missing OpenRouter key must not create model health"
21836 );
21837 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
21838 receipts.extend(tracker.write().await.drain_ledger());
21839 let receipts = receipts
21840 .iter()
21841 .filter(|receipt| receipt.model_id == model_id)
21842 .collect::<Vec<_>>();
21843 assert_eq!(receipts.len(), 1);
21844 assert_eq!(receipts[0].success, None);
21845 assert_eq!(receipts[0].quality, None);
21846 assert!(
21847 engine
21848 .adaptive_router
21849 .circuit_breakers
21850 .lock()
21851 .unwrap()
21852 .state(&model_id)
21853 .is_none(),
21854 "a missing OpenRouter key must not create breaker state"
21855 );
21856
21857 let message = "OpenRouter requires a key — run `car keys set openrouter` or connect your OpenRouter account in CarHost";
21858 assert_eq!(
21859 error.to_string(),
21860 format!("inference failed: {message}"),
21861 "guard: the pre-existing Display contract must stay byte-identical"
21862 );
21863 assert!(
21864 !is_auth_failure_message(&error.to_string()),
21865 "guard: an OpenRouter key is not repaired by Parslee sign-in"
21866 );
21867 assert_eq!(
21868 classify_fallback_reason(&error),
21869 classify_fallback_reason(&InferenceError::InferenceFailed(message.into())),
21870 "retyping must preserve prior fallback classification"
21871 );
21872 match error {
21873 InferenceError::ProviderKeyMissing {
21874 provider,
21875 model,
21876 env_vars,
21877 message: actual_message,
21878 } => {
21879 assert_eq!(provider, "openrouter");
21880 assert_eq!(model, model_id);
21881 assert_eq!(env_vars, vec![crate::openrouter::API_KEY_ENV]);
21882 assert_eq!(actual_message, message);
21883 }
21884 other => panic!("expected ProviderKeyMissing, got {other:?}"),
21885 }
21886 }
21887
21888 #[tokio::test(flavor = "current_thread")]
21889 async fn missing_generic_remote_key_is_typed_without_changing_its_string_contract() {
21890 const KEY_ENV: &str = "CAR_TEST_A47_GENERIC_PROVIDER_KEY_DO_NOT_SET";
21891 const EXTRA_KEY_ENV_1: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_1_DO_NOT_SET";
21892 const EXTRA_KEY_ENV_2: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_2_DO_NOT_SET";
21893 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
21894 let home = crate::openrouter::StateRootScope::new();
21895 let _restore = RestoredEnvironment::capture(&[
21896 "CAR_SECRETS_FILE_DIR",
21897 KEY_ENV,
21898 EXTRA_KEY_ENV_1,
21899 EXTRA_KEY_ENV_2,
21900 ]);
21901 unsafe {
21902 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
21903 std::env::remove_var(KEY_ENV);
21904 std::env::remove_var(EXTRA_KEY_ENV_1);
21905 std::env::remove_var(EXTRA_KEY_ENV_2);
21906 }
21907
21908 let endpoint = "http://127.0.0.1:9/a47-missing-key";
21909 let model_id = "test/a47-generic-missing-key";
21910 let models_dir = home.path().join("models");
21911 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
21912 let mut schema = remote_stream_fixture_schema(
21913 model_id,
21914 endpoint.into(),
21915 schema::ApiProtocol::OpenAiCompat,
21916 KEY_ENV,
21917 );
21918 let ModelSource::RemoteApi { api_key_envs, .. } = &mut schema.source else {
21919 panic!("generic remote fixture must use RemoteApi");
21920 };
21921 *api_key_envs = vec![EXTRA_KEY_ENV_1.into(), EXTRA_KEY_ENV_2.into()];
21922 engine.unified_registry_mut().register_project_model(schema);
21923
21924 let error = engine
21925 .generate_tracked(GenerateRequest {
21926 prompt: "must stop before provider dispatch".into(),
21927 model: Some(model_id.into()),
21928 params: GenerateParams {
21929 strict_model: true,
21930 ..Default::default()
21931 },
21932 ..Default::default()
21933 })
21934 .await
21935 .expect_err("a generic remote route without a key must fail before dispatch");
21936
21937 let tracker = engine.outcome_tracker();
21938 assert!(
21939 tracker.read().await.profile(model_id).is_none(),
21940 "a missing generic provider key must not create model health"
21941 );
21942 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
21943 receipts.extend(tracker.write().await.drain_ledger());
21944 let receipts = receipts
21945 .iter()
21946 .filter(|receipt| receipt.model_id == model_id)
21947 .collect::<Vec<_>>();
21948 assert_eq!(receipts.len(), 1);
21949 assert_eq!(receipts[0].success, None);
21950 assert_eq!(receipts[0].quality, None);
21951 assert!(
21952 engine
21953 .adaptive_router
21954 .circuit_breakers
21955 .lock()
21956 .unwrap()
21957 .state(model_id)
21958 .is_none(),
21959 "a missing generic provider key must not create breaker state"
21960 );
21961
21962 let message = format!(
21963 "no API keys available for endpoint {endpoint} (checked env vars: [\"{KEY_ENV}\", \"{EXTRA_KEY_ENV_1}\", \"{EXTRA_KEY_ENV_2}\"])"
21964 );
21965 assert_eq!(
21966 error.to_string(),
21967 format!("inference failed: {message}"),
21968 "guard: the pre-existing Display contract must stay byte-identical"
21969 );
21970 assert!(
21971 !is_auth_failure_message(&error.to_string()),
21972 "guard: a generic provider key is not repaired by Parslee sign-in"
21973 );
21974 assert_eq!(
21975 classify_fallback_reason(&error),
21976 classify_fallback_reason(&InferenceError::InferenceFailed(message.clone())),
21977 "retyping must preserve prior fallback classification"
21978 );
21979 match error {
21980 InferenceError::ProviderKeyMissing {
21981 provider,
21982 model,
21983 env_vars,
21984 message: actual_message,
21985 } => {
21986 assert_eq!(provider, "test");
21987 assert_eq!(model, model_id);
21988 assert_eq!(env_vars, vec![KEY_ENV, EXTRA_KEY_ENV_1, EXTRA_KEY_ENV_2]);
21989 assert_eq!(actual_message, message);
21990 }
21991 other => panic!("expected ProviderKeyMissing, got {other:?}"),
21992 }
21993 }
21994
21995 #[test]
21999 fn credential_unavailable_outcomes_do_not_degrade_models_or_open_breakers() {
22000 for reason in [
22001 CredentialFailure::Expired { expires_at: 42 },
22002 CredentialFailure::SignedOut,
22003 CredentialFailure::StoreUnreadable,
22004 CredentialFailure::EnvVarMissing {
22005 env_var: "TEST_PROVIDER_KEY".into(),
22006 },
22007 CredentialFailure::RaceRetryable,
22008 ] {
22009 let model_id = "parslee/test-credential-outcome";
22010 let error = InferenceError::CredentialUnavailable {
22011 provider: "parslee".into(),
22012 model: model_id.into(),
22013 reason: reason.clone(),
22014 detail: "credential resolution stopped before dispatch".into(),
22015 };
22016 let mut tracker = OutcomeTracker::new();
22017 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
22018 record_dispatch_failure(&mut tracker, &trace, &error);
22019
22020 assert!(
22021 tracker.profile(model_id).is_none(),
22022 "{reason:?} changed the model profile: {:?}",
22023 tracker.profile(model_id)
22024 );
22025 let receipts = tracker.drain_ledger();
22026 assert_eq!(receipts.len(), 1, "{reason:?}");
22027 assert_eq!(receipts[0].success, None, "{reason:?}");
22028 assert_eq!(receipts[0].quality, None, "{reason:?}");
22029
22030 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
22031 if error_counts_against_circuit_breaker(&error) {
22032 breakers.record_failure(model_id);
22033 }
22034 assert!(
22035 breakers.state(model_id).is_none(),
22036 "{reason:?} created per-model breaker state"
22037 );
22038 }
22039 }
22040
22041 #[tokio::test(flavor = "current_thread")]
22042 async fn signed_out_managed_attempts_leave_health_and_breaker_untouched() {
22043 if !crate::run_in_isolated_test_process(
22044 "tests::signed_out_managed_attempts_leave_health_and_breaker_untouched",
22045 "CAR_SIGNED_OUT_MODEL_HEALTH_CHILD",
22046 ) {
22047 return;
22048 }
22049
22050 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
22051 let home = crate::openrouter::StateRootScope::new();
22052 let _restore = RestoredEnvironment::capture(&[
22053 "CAR_SECRETS_FILE_DIR",
22054 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
22055 car_auth::PARSLEE_API_BASE_KEY,
22056 ]);
22057 unsafe {
22058 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
22059 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
22060 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
22061 }
22062 crate::openrouter::clear_gateway_unconfigured();
22063 crate::parslee_credential::clear_credential_rejected();
22064
22065 let store = car_secrets::SecretStore::new();
22066 store
22067 .publish(
22068 &car_secrets::SecretRef::with_default_service(
22069 car_secrets::PARSLEE_AUTH_STATE_V2_KEY,
22070 ),
22071 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
22072 )
22073 .unwrap();
22074
22075 let models_dir = home.path().join("models");
22076 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
22077 let schema = crate::openrouter::curated_schemas()
22078 .into_iter()
22079 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
22080 .expect("managed alias fixture");
22081 let model_id = schema.id.clone();
22082 engine.unified_registry_mut().register_project_model(schema);
22083
22084 for _ in 0..3 {
22088 let error = engine
22089 .generate_tracked(GenerateRequest {
22090 prompt: "must stop before provider dispatch".into(),
22091 model: Some(model_id.clone()),
22092 params: GenerateParams {
22093 strict_model: true,
22094 ..Default::default()
22095 },
22096 ..Default::default()
22097 })
22098 .await
22099 .expect_err("signed-out managed inference must be rejected");
22100 assert!(
22101 matches!(
22102 &error,
22103 InferenceError::CredentialUnavailable {
22104 reason: CredentialFailure::SignedOut,
22105 ..
22106 }
22107 ),
22108 "unexpected signed-out error: {error:?}"
22109 );
22110 }
22111
22112 let tracker = engine.outcome_tracker();
22113 assert!(
22114 tracker.read().await.profile(&model_id).is_none(),
22115 "signed-out attempts must not create a model-health profile"
22116 );
22117 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
22121 receipts.extend(tracker.write().await.drain_ledger());
22122 let signed_out_receipts = receipts
22123 .iter()
22124 .filter(|entry| entry.model_id == model_id)
22125 .collect::<Vec<_>>();
22126 assert_eq!(signed_out_receipts.len(), 3);
22127 assert!(signed_out_receipts
22128 .iter()
22129 .all(|entry| entry.success.is_none() && entry.quality.is_none()));
22130 assert!(
22131 engine
22132 .adaptive_router
22133 .circuit_breakers
22134 .lock()
22135 .unwrap()
22136 .state(&model_id)
22137 .is_none(),
22138 "signed-out attempts must not create per-model breaker state"
22139 );
22140 }
22141
22142 fn admission_refusals() -> Vec<InferenceError> {
22146 use resource_policy::{LocalLoadVerdict, ModelMemoryEstimate, ModelResourceEvidence};
22147 [
22148 LocalLoadVerdict::LiveMemoryUnknown,
22149 LocalLoadVerdict::DisabledByPolicy,
22150 LocalLoadVerdict::ExceedsConfiguredCeiling,
22151 LocalLoadVerdict::InsufficientLiveMemory,
22152 LocalLoadVerdict::ModelMaintenance,
22153 LocalLoadVerdict::PendingTeardown,
22154 ]
22155 .into_iter()
22156 .map(|verdict| {
22157 InferenceError::from(resource_policy::LocalAdmissionError {
22158 preflight: resource_policy::LocalLoadPreflight {
22159 model_id: "vllm-mlx/big:4bit".into(),
22160 estimate: ModelMemoryEstimate {
22161 weights_mb: 16_000,
22162 runtime_overhead_mb: 1_000,
22163 context_overhead_mb: 2_000,
22164 transient_margin_mb: 1_000,
22165 estimated_peak_mb: 20_000,
22166 evidence: ModelResourceEvidence::CatalogExact,
22167 },
22168 configured_ceiling_mb: 26_214,
22169 resident_model_mb: 0,
22170 active_reservations_mb: 0,
22171 estimated_incremental_mb: 20_000,
22172 accelerator_total_mb: None,
22173 accelerator_resident_mb: None,
22174 accelerator_incremental_mb: None,
22175 live_available_mb: Some(4_000),
22176 emergency_reserve_mb: 6_553,
22177 verdict,
22178 },
22179 })
22180 })
22181 .collect()
22182 }
22183
22184 #[test]
22188 fn local_admission_refusal_is_not_a_model_failure() {
22189 let model_id = "vllm-mlx/big:4bit";
22190 for refusal in admission_refusals() {
22191 assert!(matches!(
22192 refusal,
22193 InferenceError::LocalResourceBlocked { .. }
22194 ));
22195 let mut tracker = OutcomeTracker::new();
22196 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
22197 record_dispatch_failure(&mut tracker, &trace, &refusal);
22198 assert!(
22199 tracker.profile(model_id).is_none(),
22200 "admission refusal changed the model profile: {refusal}"
22201 );
22202 let receipt = tracker.drain_ledger().pop().expect("receipt kept");
22203 assert_eq!(receipt.success, None, "{refusal}");
22204 assert_eq!(receipt.quality, None, "{refusal}");
22205 assert!(!error_counts_against_circuit_breaker(&refusal), "{refusal}");
22206 }
22207 }
22208
22209 #[test]
22210 fn content_refusal_closes_its_parenthetical_in_every_shape() {
22211 let render = |kind: Option<&str>, code: Option<&str>| {
22212 InferenceError::ContentRefused {
22213 provider: "p".into(),
22214 kind: kind.map(Into::into),
22215 code: code.map(Into::into),
22216 message: "m".into(),
22217 }
22218 .to_string()
22219 };
22220 assert_eq!(
22221 render(Some("k"), Some("c")),
22222 "p refused this request on content grounds (type=k, code=c): m"
22223 );
22224 assert_eq!(
22225 render(Some("k"), None),
22226 "p refused this request on content grounds (type=k): m"
22227 );
22228 assert_eq!(
22229 render(None, Some("c")),
22230 "p refused this request on content grounds (code=c): m"
22231 );
22232 assert_eq!(
22233 render(None, None),
22234 "p refused this request on content grounds: m"
22235 );
22236 }
22237
22238 fn legacy_unattributed_errors() -> Vec<String> {
22243 let mut legacy: Vec<String> = admission_refusals()
22244 .iter()
22245 .map(ToString::to_string)
22246 .collect();
22247 legacy.push(
22248 InferenceError::ProviderKeyMissing {
22249 provider: "anthropic".into(),
22250 model: "m".into(),
22251 env_vars: vec!["ANTHROPIC_API_KEY".into()],
22252 message: "no API keys available for endpoint https://api.anthropic.com (checked env vars: [\"ANTHROPIC_API_KEY\"])".into(),
22253 }
22254 .to_string(),
22255 );
22256 legacy.push(
22257 InferenceError::InferenceFailed(
22258 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required".into(),
22259 )
22260 .to_string(),
22261 );
22262 legacy.push(
22263 InferenceError::InferenceFailed(
22264 "FoundationModels declined the request (guardrailViolation): unsafe".into(),
22265 )
22266 .to_string(),
22267 );
22268 legacy.push(
22269 InferenceError::InferenceFailed(
22270 InferenceError::UnsupportedMode {
22271 mode: "apple-foundation-models",
22272 backend: "remote",
22273 reason: "no",
22274 }
22275 .to_string(),
22276 )
22277 .to_string(),
22278 );
22279 legacy
22280 }
22281
22282 fn write_legacy_ledger(
22286 ledger: &std::path::Path,
22287 real: &str,
22288 ghost: &str,
22289 ) -> Vec<outcome::ModelProfile> {
22290 let mut writer = OutcomeTracker::new();
22291 for error in legacy_unattributed_errors() {
22292 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
22293 writer.record_failure(&t, &error);
22294 }
22295 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
22296 writer.record_failure(&t, GENUINE_FAILURE);
22297 let t = writer.record_start(ghost, InferenceTask::Generate, "explicit model");
22298 writer.record_failure(
22299 &t,
22300 &InferenceError::InferenceFailed(
22301 InferenceError::ModelNotFound(ghost.into()).to_string(),
22302 )
22303 .to_string(),
22304 );
22305 outcome::append_ledger_entries(ledger, &writer.drain_ledger()).unwrap();
22306 writer.export_profiles()
22307 }
22308
22309 fn reconcile_for_test(
22312 ledger: &std::path::Path,
22313 tracker: &mut OutcomeTracker,
22314 is_registered: &dyn Fn(&str) -> bool,
22315 ) -> std::io::Result<outcome::LedgerReconciliation> {
22316 let plan = outcome::plan_reconcile(
22317 ledger,
22318 tracker,
22319 is_registered,
22320 &std::collections::HashMap::new(),
22321 );
22322 if let Some(entries) = &plan.ledger {
22323 outcome::write_reconciled_ledger(ledger, entries)?;
22324 }
22325 if let Some(profiles) = plan.profiles {
22326 tracker.commit_reconciled(profiles);
22327 }
22328 Ok(plan.report)
22329 }
22330
22331 const GENUINE_FAILURE: &str =
22332 "inference failed: forward: \"[metal::malloc] Attempting to allocate 41846087808 bytes\"";
22333
22334 fn profile_json(tracker: &OutcomeTracker, id: &str) -> serde_json::Value {
22335 let mut v = serde_json::to_value(tracker.profile(id).unwrap()).unwrap();
22336 v.as_object_mut().unwrap().remove("updated_at");
22337 v
22338 }
22339
22340 #[test]
22345 fn reconcile_takes_back_misattributed_failures_and_their_quality_evidence() {
22346 let dir = tempfile::tempdir().unwrap();
22347 let ledger = dir.path().join("outcome_ledger.jsonl");
22348 let (real, ghost) = ("mlx/real:4bit", "nonexistent-model-id-xyz");
22349 let profiles = write_legacy_ledger(&ledger, real, ghost);
22350 let legacy = legacy_unattributed_errors().len();
22351
22352 let mut tracker = OutcomeTracker::new();
22353 tracker.import_profiles(profiles);
22354 let before = tracker.profile(real).unwrap().clone();
22355 assert_eq!(before.fail_count, legacy as u64 + 1);
22356 assert!(
22357 before.quality_observations > 1,
22358 "the legacy rows moved the EMA"
22359 );
22360 assert!(
22361 before.ema_quality < 0.4 - 1e-9,
22362 "buried below one failure's worth"
22363 );
22364
22365 let is_registered = |id: &str| id == real;
22366 let report = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
22367 assert_eq!(report.reclassified, legacy);
22368 assert_eq!(report.unregistered, 1);
22369 assert_eq!(report.profiles_rebuilt, 1);
22370 assert_eq!(report.profiles_removed, 1);
22371
22372 let p = tracker.profile(real).unwrap();
22373 assert_eq!((p.fail_count, p.total_calls, p.success_count), (1, 1, 0));
22374 assert_eq!(p.quality_observations, 1);
22375 assert!(
22376 (p.ema_quality - 0.4).abs() < 1e-12,
22377 "0.5 stepped once toward 0: {}",
22378 p.ema_quality
22379 );
22380 let ts = &p.task_stats["generate"];
22381 assert_eq!((ts.failures, ts.calls, ts.quality_observations), (1, 0, 1));
22383 assert!(tracker.profile(ghost).is_none());
22384
22385 let rows = outcome::read_ledger(&ledger, 0);
22386 assert_eq!(rows.len(), legacy + 2, "nothing is deleted");
22387 for row in &rows {
22388 match row.error.as_deref() {
22389 Some(GENUINE_FAILURE) => {
22390 assert_eq!(row.success, Some(false));
22391 assert_eq!(row.attribution, None);
22392 }
22393 _ if row.model_id == ghost => {
22394 assert_eq!(row.success, None);
22395 assert_eq!(
22396 row.attribution,
22397 Some(outcome::UnattributedReason::Unregistered)
22398 );
22399 }
22400 Some(error) => {
22401 use outcome::UnattributedReason as R;
22402 assert_eq!(row.success, None);
22403 assert_eq!(row.quality, None);
22404 let expected = if error.contains("memory")
22406 || error.contains("allocation")
22407 || error.contains("maintained")
22408 || error.contains("previous local model")
22409 {
22410 R::ResourceBlocked
22411 } else if error.contains("API keys") || error.contains("org lookup") {
22412 R::Account
22413 } else if error.contains("declined") {
22414 R::ContentRefused
22415 } else {
22416 R::Capability
22417 };
22418 assert_eq!(row.attribution, Some(expected), "{error}");
22419 }
22420 None => panic!("every legacy row carries its error"),
22421 }
22422 }
22423 #[cfg(unix)]
22424 {
22425 use std::os::unix::fs::PermissionsExt;
22426 let mode = std::fs::metadata(&ledger).unwrap().permissions().mode() & 0o777;
22427 assert_eq!(mode, 0o600, "a rewritten ledger stays owner-only");
22428 }
22429
22430 let rebuilt = profile_json(&tracker, real);
22432 let again = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
22433 assert_eq!(
22434 again.reclassified + again.unregistered + again.profiles_removed,
22435 0
22436 );
22437 assert_eq!(profile_json(&tracker, real), rebuilt);
22438 }
22439
22440 #[test]
22446 fn reconcile_repairs_a_ledger_the_first_migration_already_rewrote() {
22447 let dir = tempfile::tempdir().unwrap();
22448 let ledger = dir.path().join("outcome_ledger.jsonl");
22449 let real = "mlx/real:4bit";
22450 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
22451
22452 let mut clean = OutcomeTracker::new();
22454 clean.import_profiles(profiles.clone());
22455 let clean_ledger = dir.path().join("clean.jsonl");
22456 std::fs::copy(&ledger, &clean_ledger).unwrap();
22457 reconcile_for_test(&clean_ledger, &mut clean, &|id| id == real).unwrap();
22458 let expected = profile_json(&clean, real);
22459
22460 let rows: Vec<_> = outcome::read_ledger(&ledger, 0)
22463 .into_iter()
22464 .map(|mut r| {
22465 if r.model_id == real && r.error.as_deref() != Some(GENUINE_FAILURE) {
22466 r.success = None;
22467 r.quality = None;
22468 }
22469 r
22470 })
22471 .collect();
22472 std::fs::remove_file(&ledger).unwrap();
22473 outcome::append_ledger_entries(&ledger, &rows).unwrap();
22474 let mut v2 = profiles;
22475 for p in &mut v2 {
22476 if p.model_id == real {
22477 p.fail_count = 1;
22478 p.total_calls = 1;
22479 p.task_stats.get_mut("generate").unwrap().failures = 1;
22480 }
22481 }
22482 let mut tracker = OutcomeTracker::new();
22483 tracker.import_profiles(v2);
22484 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
22485 assert_eq!(profile_json(&tracker, real), expected);
22486 }
22487
22488 #[test]
22493 fn reconcile_clamps_a_broken_profile_and_keeps_ids_that_ran() {
22494 let dir = tempfile::tempdir().unwrap();
22495 let ledger = dir.path().join("outcome_ledger.jsonl");
22496 let (model, renamed) = ("parslee/reasoning", "openai/gpt-5.4");
22497 let mut writer = OutcomeTracker::new();
22498 for id in [model, renamed] {
22499 for _ in 0..4 {
22500 let t = writer.record_start(id, InferenceTask::Generate, "r");
22501 writer.record_failure(
22502 &t,
22503 "inference failed: Parslee inference returned no content",
22504 );
22505 }
22506 }
22507 let mut rows = writer.drain_ledger();
22508 let mut ok = rows[0].clone();
22509 ok.model_id = renamed.into();
22510 ok.success = Some(true);
22511 ok.error = None;
22512 rows.push(ok);
22513 outcome::append_ledger_entries(&ledger, &rows).unwrap();
22514
22515 let mut profiles = writer.export_profiles();
22516 for p in &mut profiles {
22517 if p.model_id == model {
22518 p.success_count = 1522;
22519 p.total_calls = 1519;
22520 p.fail_count = 0;
22521 p.task_stats.get_mut("generate").unwrap().failures = 0;
22522 }
22523 }
22524 let mut tracker = OutcomeTracker::new();
22525 tracker.import_profiles(profiles);
22526 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == model).unwrap();
22527 assert_eq!(report.unregistered, 0, "an id that ran is not unregistered");
22528 let p = tracker.profile(model).unwrap();
22529 assert_eq!(
22530 (p.success_count, p.fail_count, p.total_calls),
22531 (1522, 0, 1522)
22532 );
22533 assert_eq!(tracker.profile(renamed).unwrap().fail_count, 4);
22534 assert!(outcome::read_ledger(&ledger, 0)
22535 .iter()
22536 .filter(|r| r.model_id == renamed && r.success != Some(true))
22537 .all(|r| r.success == Some(false)));
22538 }
22539
22540 #[test]
22546 fn reconcile_keeps_successes_the_ledger_never_saw() {
22547 let dir = tempfile::tempdir().unwrap();
22548 let ledger = dir.path().join("outcome_ledger.jsonl");
22549 let model = "mlx/qwen3-30b-a3b:4bit";
22550 let mut writer = OutcomeTracker::new();
22551 for _ in 0..20 {
22552 let t = writer.record_start(model, InferenceTask::Generate, "r");
22553 writer.record_complete(&t, 100, 10, 20);
22554 }
22555 for refusal in admission_refusals().iter().take(3) {
22556 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
22557 writer.record_failure(&t, &refusal.to_string());
22558 }
22559 let written = writer.drain_ledger();
22561 assert!(written.iter().all(|r| r.success == Some(false)));
22562 outcome::append_ledger_entries(&ledger, &written).unwrap();
22563 let mut tracker = OutcomeTracker::new();
22564 tracker.import_profiles(writer.export_profiles());
22565 let before = tracker.profile(model).unwrap().clone();
22566 assert_eq!((before.success_count, before.fail_count), (20, 3));
22567
22568 reconcile_for_test(&ledger, &mut tracker, &|_| true).unwrap();
22569 let p = tracker.profile(model).unwrap();
22570 assert_eq!((p.success_count, p.fail_count), (20, 0));
22571 assert_eq!(p.total_calls, before.total_calls - 3);
22572 assert_eq!(p.quality_observations, 0);
22573 assert_eq!(p.ema_quality, 0.5);
22574 }
22575
22576 #[test]
22580 fn reconcile_replays_a_seeded_profile_from_its_benchmark_prior() {
22581 let dir = tempfile::tempdir().unwrap();
22582 let ledger = dir.path().join("outcome_ledger.jsonl");
22583 let model = "mlx/qwen3-4b:4bit";
22584 let mut seeded = outcome::ModelProfile::new(model.into());
22585 seeded.ema_quality = 0.7;
22586 seeded.prior_sample_size = 3;
22587 let mut writer = OutcomeTracker::new();
22588 writer.import_profiles(vec![seeded]);
22589 for refusal in admission_refusals().iter().take(2) {
22590 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
22591 writer.record_failure(&t, &refusal.to_string());
22592 }
22593 outcome::append_ledger_entries(&ledger, &writer.drain_ledger()).unwrap();
22594 let mut tracker = OutcomeTracker::new();
22595 tracker.import_profiles(writer.export_profiles());
22596 let before = tracker.profile(model).unwrap().clone();
22597 assert_eq!(before.quality_observations, 2);
22598 assert!(before.ema_quality < 0.7 * 0.8 * 0.8 + 1e-9);
22599
22600 let seeds = std::collections::HashMap::from([(
22601 model.to_string(),
22602 outcome::QualitySeed {
22603 overall: Some(0.7),
22604 tasks: std::collections::HashMap::new(),
22605 },
22606 )]);
22607 let plan = outcome::plan_reconcile(&ledger, &tracker, &|_| true, &seeds);
22608 assert_eq!(plan.report.profiles_rebuilt, 1);
22609 tracker.commit_reconciled(plan.profiles.expect("a corrected profile set"));
22610 let p = tracker.profile(model).unwrap();
22611 assert_eq!(p.quality_observations, 0);
22612 assert!(
22613 (p.ema_quality - 0.7).abs() < 1e-12,
22614 "back to the seed: {}",
22615 p.ema_quality
22616 );
22617 assert_eq!(p.prior_sample_size, 3, "the prior's weight is the prior's");
22618 }
22619
22620 #[test]
22624 fn reconcile_is_quiet_in_steady_state() {
22625 let dir = tempfile::tempdir().unwrap();
22626 let ledger = dir.path().join("outcome_ledger.jsonl");
22627 let real = "mlx/real:4bit";
22628 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
22629 let mut tracker = OutcomeTracker::new();
22630 tracker.import_profiles(profiles);
22631 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
22632
22633 let t = tracker.record_start(real, InferenceTask::Generate, "r");
22635 tracker.record_complete(&t, 120, 10, 20);
22636 let t = tracker.record_start(real, InferenceTask::Generate, "r");
22637 tracker.record_complete(&t, 80, 10, 0);
22638 let t = tracker.record_start(real, InferenceTask::Generate, "r");
22639 tracker.record_failure(&t, GENUINE_FAILURE);
22640 outcome::append_ledger_entries(&ledger, &tracker.drain_ledger()).unwrap();
22641 let settled = profile_json(&tracker, real);
22642
22643 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
22644 assert_eq!(report, outcome::LedgerReconciliation::default());
22645 assert_eq!(profile_json(&tracker, real), settled);
22646
22647 outcome::prune_ledger(&ledger, 2).unwrap();
22650 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
22651 assert_eq!(report, outcome::LedgerReconciliation::default());
22652 assert_eq!(profile_json(&tracker, real), settled);
22653 }
22654
22655 #[test]
22659 fn neutral_profile_is_not_backfilled_on_load() {
22660 let dir = tempfile::tempdir().unwrap();
22661 let path = dir.path().join("profiles.json");
22662 let mut writer = OutcomeTracker::new();
22663 let t = writer.record_start("m", InferenceTask::Generate, "r");
22664 writer.record_failure(&t, "daemon read timeout on infer after 30s");
22665 let p = writer.profile("m").unwrap();
22666 assert_eq!((p.fail_count, p.quality_observations), (1, 0));
22667 assert_eq!(p.ema_quality, 0.5);
22668 writer.save_to_file(&path).unwrap();
22669 let mut reader = OutcomeTracker::new();
22670 reader.load_from_file(&path).unwrap();
22671 assert_eq!(reader.profile("m").unwrap().quality_observations, 0);
22672 }
22673
22674 #[tokio::test]
22678 async fn reconcile_refuses_when_profiles_could_not_be_read() {
22679 let root = tempfile::tempdir().unwrap();
22680 let state = root.path().join("state");
22681 let models = state.join("models");
22682 std::fs::create_dir_all(&models).unwrap();
22683 let ledger = models.join("outcome_ledger.jsonl");
22684 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
22685 std::fs::write(models.join("outcome_profiles.json"), b"{ not json").unwrap();
22686 let before = std::fs::read(&ledger).unwrap();
22687 let engine = InferenceEngine::new(InferenceConfig {
22688 state_root: state.clone(),
22689 models_dir: root.path().join("weights"),
22690 ..InferenceConfig::default()
22691 });
22692 let report = engine.reconcile_outcome_ledger().await.unwrap();
22693 assert_eq!(report, outcome::LedgerReconciliation::default());
22694 assert_eq!(std::fs::read(&ledger).unwrap(), before, "ledger untouched");
22695 assert_eq!(
22696 std::fs::read(models.join("outcome_profiles.json")).unwrap(),
22697 b"{ not json",
22698 "profiles untouched"
22699 );
22700 }
22701
22702 #[cfg(unix)]
22705 #[tokio::test]
22706 async fn retire_plan_includes_the_projection_and_engine_refusals() {
22707 let _serial = retire_test_scope().await;
22708 let root = tempfile::tempdir().unwrap();
22709 let root_path = root.path().canonicalize().unwrap();
22710 let hub = root_path.join("hub");
22711 let models = root_path.join("weights");
22712 let engine = InferenceEngine::new(InferenceConfig {
22713 state_root: root_path.join("state"),
22714 models_dir: models.clone(),
22715 ..InferenceConfig::default()
22716 });
22717 let schema = engine
22718 .unified_registry()
22719 .all()
22720 .find(|m| m.id == plain_mlx_id(&engine))
22721 .unwrap()
22722 .clone();
22723 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22724 unreachable!()
22725 };
22726 let repo = hf_cache::repo_dir_in(&hub, hf_repo);
22727 std::fs::create_dir_all(repo.join("snapshots/s")).unwrap();
22728 std::fs::create_dir_all(repo.join("blobs")).unwrap();
22729 std::fs::write(repo.join("blobs/w"), vec![0u8; 64]).unwrap();
22730 std::os::unix::fs::symlink(repo.join("blobs/w"), repo.join("snapshots/s/w")).unwrap();
22731 std::fs::create_dir_all(&models).unwrap();
22732 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22733
22734 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
22735 assert!(!plan.is_refused(), "{plan:?}");
22736 assert_eq!(
22737 plan.freed_bytes, 64,
22738 "the link frees nothing; the repo frees its blob"
22739 );
22740 assert_eq!(
22741 plan.projections,
22742 vec![retire::Projection {
22743 path: models.join(&schema.name),
22744 receipt_for: None,
22745 }]
22746 );
22747 assert!(matches!(
22748 &plan.deletions[..],
22749 [retire::HubDeletion::Repo { .. }]
22750 ));
22751
22752 let guard = crate::download::acquire_model_lock(&schema.id).await;
22753 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
22754 assert!(plan.refusals.contains(&retire::RetireRefusal::Downloading));
22755 drop(guard);
22756
22757 engine.lane_defaults_cache.write().unwrap().set(
22758 None,
22759 crate::intent::UseCase::Coding,
22760 schema.id.clone(),
22761 1,
22762 );
22763 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
22764 assert!(plan
22765 .refusals
22766 .iter()
22767 .any(|r| matches!(r, retire::RetireRefusal::LaneDefault { .. })));
22768 }
22769
22770 #[cfg(unix)]
22773 #[tokio::test]
22774 async fn retire_plan_contains_projections_and_sees_sibling_downloads() {
22775 let _serial = retire_test_scope().await;
22776 let root = tempfile::tempdir().unwrap();
22777 let root_path = root.path().canonicalize().unwrap();
22778 let hub = root_path.join("hub");
22779 std::fs::create_dir_all(&hub).unwrap();
22780 let mut engine = InferenceEngine::new(InferenceConfig {
22781 state_root: root_path.join("state"),
22782 models_dir: root_path.join("weights"),
22783 ..InferenceConfig::default()
22784 });
22785 let mut escaping = engine
22786 .unified_registry()
22787 .get("mlx/qwen3-4b:4bit")
22788 .unwrap()
22789 .clone();
22790 escaping.id = "user/escape".into();
22791 escaping.name = "../../outside".into();
22792 std::fs::create_dir_all(root_path.join("outside")).unwrap();
22793 engine
22794 .unified_registry_mut()
22795 .register_project_model(escaping);
22796 let plan = engine.retire_plan_in("user/escape", &hub).unwrap();
22797 assert!(plan.projections.is_empty(), "{plan:?}");
22798 assert!(plan
22799 .refusals
22800 .iter()
22801 .any(|r| matches!(r, retire::RetireRefusal::UnsafePath { .. })));
22802
22803 let guard = crate::download::acquire_model_lock("vllm-mlx/gemma-4-12b-it").await;
22805 let plan = engine
22806 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
22807 .unwrap();
22808 assert!(
22809 plan.refusals.contains(&retire::RetireRefusal::Downloading),
22810 "{plan:?}"
22811 );
22812 assert_eq!(
22814 plan.also_retires,
22815 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
22816 );
22817 drop(guard);
22818 let plan = engine
22819 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
22820 .unwrap();
22821 assert!(!plan.refusals.contains(&retire::RetireRefusal::Downloading));
22822 }
22823
22824 #[cfg(unix)]
22827 #[test]
22828 fn discovered_rows_are_retired_alone_and_their_files_are_kept() {
22829 let root = tempfile::tempdir().unwrap();
22830 let root_path = root.path().canonicalize().unwrap();
22831 let hub = root_path.join("hub");
22832 let models = root_path.join("weights");
22833 std::fs::create_dir_all(&hub).unwrap();
22834 let mut engine = InferenceEngine::new(InferenceConfig {
22835 state_root: root_path.join("state"),
22836 models_dir: models.clone(),
22837 ..InferenceConfig::default()
22838 });
22839 let base = engine
22840 .unified_registry()
22841 .get("mlx/qwen3-4b:4bit")
22842 .unwrap()
22843 .clone();
22844 for (id, name) in [("disk/one", "One-MLX"), ("disk/two", "Two-MLX")] {
22845 let mut row = base.clone();
22846 row.id = id.into();
22847 row.name = name.into();
22848 row.source = ModelSource::Mlx {
22849 hf_repo: String::new(),
22850 hf_weight_file: None,
22851 };
22852 engine.unified_registry_mut().register_project_model(row);
22853 std::fs::create_dir_all(models.join(name)).unwrap();
22854 std::fs::write(models.join(name).join("model.safetensors"), b"user").unwrap();
22855 }
22856 let plan = engine.retire_plan_in("disk/one", &hub).unwrap();
22857 assert!(plan.also_retires.is_empty(), "{plan:?}");
22858 assert!(plan.projections.is_empty(), "{plan:?}");
22859 assert_eq!(plan.user_content, vec![models.join("One-MLX")]);
22860 assert_eq!(plan.freed_bytes, 0);
22861 }
22862
22863 static RETIRE_LOCKS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
22867
22868 async fn retire_test_scope() -> (
22877 tokio::sync::MutexGuard<'static, ()>,
22878 tokio::sync::MutexGuard<'static, ()>,
22879 ) {
22880 let environment = crate::openrouter::test_environment_scope_async().await;
22881 let serial = RETIRE_LOCKS.lock().await;
22882 (environment, serial)
22883 }
22884
22885 #[cfg(unix)]
22889 fn plain_mlx_id(engine: &InferenceEngine) -> String {
22890 let protected = engine.protected_models(&engine.unified_registry(), true);
22891 [
22892 "mlx/qwen3-1.7b:3bit",
22893 "mlx/qwen3-0.6b:6bit",
22894 "mlx/qwen3-30b-a3b:4bit",
22895 ]
22896 .into_iter()
22897 .find(|id| !protected.contains_key(*id))
22898 .unwrap()
22899 .to_string()
22900 }
22901
22902 #[cfg(unix)]
22904 fn hub_repo(hub: &Path, repo: &str, files: &[(&str, usize)]) -> PathBuf {
22905 let dir = hf_cache::repo_dir_in(hub, repo);
22906 std::fs::create_dir_all(dir.join("snapshots/s")).unwrap();
22907 std::fs::create_dir_all(dir.join("blobs")).unwrap();
22908 for (i, (path, size)) in files.iter().enumerate() {
22909 let blob = dir.join("blobs").join(format!("b{i}"));
22910 std::fs::write(&blob, vec![0u8; *size]).unwrap();
22911 let entry = dir.join("snapshots/s").join(path);
22912 std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
22913 std::os::unix::fs::symlink(&blob, &entry).unwrap();
22914 }
22915 dir
22916 }
22917
22918 fn isolated_engine(root: &Path) -> (InferenceEngine, PathBuf, PathBuf) {
22919 let hub = root.join("hub");
22920 let models = root.join("weights");
22921 std::fs::create_dir_all(&hub).unwrap();
22922 std::fs::create_dir_all(&models).unwrap();
22923 let engine = InferenceEngine::new(InferenceConfig {
22924 state_root: root.join("state"),
22925 models_dir: models.clone(),
22926 ..InferenceConfig::default()
22927 });
22928 (engine, hub, models)
22929 }
22930
22931 #[tokio::test]
22937 async fn a_swapped_signed_catalog_is_one_generation_for_every_reader() {
22938 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22939 let root = tempfile::tempdir().unwrap();
22940 let (mut engine, _hub, _models) = isolated_engine(root.path());
22941 let builtin = crate::registry::builtin_catalog()
22942 .into_iter()
22943 .find(|m| !m.is_local())
22944 .expect("a remote builtin row");
22945 let mut added = builtin.clone();
22946 added.id = "signed/swapped-in".into();
22947 added.name = "swapped-in".into();
22948 added.tags.retain(|t| t != "builtin");
22949 let mut redefined = builtin.clone();
22950 redefined.context_length = builtin.context_length + 1;
22951 let mut kept = added.clone();
22953 kept.id = "project/registered-in-memory".into();
22954 kept.name = "registered-in-memory".into();
22955 assert!(engine
22956 .unified_registry_mut()
22957 .register_project_model(kept.clone()));
22958
22959 let before = engine.unified_registry();
22960 let (first, _key) = crate::catalog::signed_test_catalog(
22961 crate::catalog::CatalogDoc {
22962 revoked: Vec::new(),
22963 version: 2,
22964 models: vec![added.clone(), redefined],
22965 },
22966 7,
22967 );
22968 assert_eq!(engine.swap_signed_catalog(first), 2);
22970 assert!(engine.list_schemas().iter().any(|m| m.id == added.id));
22971 assert!(engine.unified_registry().get(&added.id).is_some());
22972 assert_eq!(
22973 engine.model_schema("swapped-in").map(|m| m.id),
22974 Some(added.id.clone())
22975 );
22976 assert_eq!(
22977 engine
22978 .unified_registry()
22979 .get(&builtin.id)
22980 .unwrap()
22981 .context_length,
22982 builtin.context_length,
22983 "a signed row never redefines a compiled builtin's identity"
22984 );
22985 assert!(
22986 before.get(&added.id).is_none(),
22987 "an older generation is unchanged"
22988 );
22989 assert!(engine.unified_registry().get(&kept.id).is_some());
22990
22991 let (second, _key) = crate::catalog::signed_test_catalog(
22992 crate::catalog::CatalogDoc {
22993 revoked: Vec::new(),
22994 version: 3,
22995 models: vec![],
22996 },
22997 7,
22998 );
22999 assert_eq!(engine.swap_signed_catalog(second), 0);
23000 assert!(
23001 engine.unified_registry().get(&added.id).is_none(),
23002 "withdrawn"
23003 );
23004 assert!(engine.model_schema("swapped-in").is_none());
23005 assert!(engine.unified_registry().get(&builtin.id).is_some());
23006 assert!(engine.unified_registry().get(&kept.id).is_some());
23007 }
23008
23009 #[tokio::test]
23012 async fn a_withdrawn_row_a_lane_defaults_to_is_kept_deprecated() {
23013 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23014 let root = tempfile::tempdir().unwrap();
23015 let (engine, _hub, _models) = isolated_engine(root.path());
23016 let mut row = crate::registry::builtin_catalog()
23017 .into_iter()
23018 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
23019 .unwrap();
23020 row.id = "signed/lane-default".into();
23021 row.name = "lane-default".into();
23022 let catalog = |models, version| {
23023 crate::catalog::signed_test_catalog(
23024 crate::catalog::CatalogDoc {
23025 revoked: Vec::new(),
23026 version,
23027 models,
23028 },
23029 13,
23030 )
23031 .0
23032 };
23033 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
23034 engine
23035 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
23036 .unwrap();
23037 engine.swap_signed_catalog(catalog(vec![], 3));
23038 let kept = engine
23039 .unified_registry()
23040 .get(&row.id)
23041 .cloned()
23042 .expect("kept");
23043 assert!(kept.deprecated);
23044 }
23045
23046 #[tokio::test]
23049 async fn a_row_withdrawn_mid_pull_is_kept_and_recorded() {
23050 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23051 let root = tempfile::tempdir().unwrap();
23052 let (engine, _hub, _models) = isolated_engine(root.path());
23053 let mut row = crate::registry::builtin_catalog()
23054 .into_iter()
23055 .find(|m| !m.is_local())
23056 .unwrap();
23057 row.id = "signed/mid-pull".into();
23058 row.name = "mid-pull".into();
23059 let catalog = |models, version| {
23060 crate::catalog::signed_test_catalog(
23061 crate::catalog::CatalogDoc {
23062 revoked: Vec::new(),
23063 version,
23064 models,
23065 },
23066 17,
23067 )
23068 .0
23069 };
23070 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
23071 let retained = catalog::retained_path(&root.path().join("state"));
23072 engine
23073 .with_pinned_registry(async {
23074 engine.swap_signed_catalog(catalog(vec![], 3));
23075 assert!(engine.current_registry().get(&row.id).is_none());
23076 assert!(!retained.exists(), "nothing was kept yet");
23077 engine.keep_pulled_row_if_withdrawn("mid-pull");
23078 })
23079 .await;
23080 let kept = engine
23081 .unified_registry()
23082 .get(&row.id)
23083 .cloned()
23084 .expect("kept");
23085 assert!(kept.deprecated);
23086 let recorded = std::fs::read_to_string(&retained).unwrap();
23087 assert!(recorded.contains("signed/mid-pull"), "{recorded}");
23088 }
23089
23090 #[tokio::test]
23094 async fn the_scheduled_refresh_loads_only_a_newer_verified_catalog() {
23095 use std::io::{Read, Write};
23096 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23097 let root = tempfile::tempdir().unwrap();
23098 let (engine, _hub, _models) = isolated_engine(root.path());
23099 let mut row = crate::registry::builtin_catalog()
23100 .into_iter()
23101 .find(|m| !m.is_local())
23102 .unwrap();
23103 row.id = "signed/served".into();
23104 row.name = "served".into();
23105 let (published, key) = crate::catalog::signed_test_catalog(
23106 crate::catalog::CatalogDoc {
23107 revoked: Vec::new(),
23108 version: 5,
23109 models: vec![row],
23110 },
23111 31,
23112 );
23113 let serving = std::sync::Arc::new(std::sync::Mutex::new(published.envelope()));
23114 let served = serving.clone();
23115 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
23116 let url = format!("http://{}/catalog.json", listener.local_addr().unwrap());
23117 std::thread::spawn(move || {
23118 for stream in listener.incoming() {
23119 let Ok(mut stream) = stream else { return };
23120 let mut request = [0u8; 2048];
23121 let n = stream.read(&mut request).unwrap_or(0);
23122 let request = String::from_utf8_lossy(&request[..n]);
23123 let envelope = served.lock().unwrap().clone();
23124 let body = if request.starts_with("GET /catalog.json.sig ") {
23125 envelope.signature.clone()
23126 } else {
23127 envelope.signed_body.clone()
23128 };
23129 let _ = write!(
23130 stream,
23131 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
23132 body.len(),
23133 body
23134 );
23135 }
23136 });
23137 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(Some(1)));
23138 assert!(engine.unified_registry().get("signed/served").is_some());
23139 assert_eq!(
23140 engine.refresh_catalog_from(&url, &key).await,
23141 Ok(None),
23142 "nothing newer is not an error"
23143 );
23144 let status_path = catalog::status_path(&engine.config.state_root);
23146 let status = catalog::load_status(&status_path);
23147 assert_eq!(status.source.as_deref(), Some(url.as_str()));
23148 assert_eq!(status.verified_version, Some(5));
23149 assert!(status.last_verified_at.is_some() && status.failing_since.is_none());
23150 let (_, other_key) = crate::catalog::signed_test_catalog(
23151 crate::catalog::CatalogDoc {
23152 revoked: Vec::new(),
23153 version: 1,
23154 models: vec![],
23155 },
23156 32,
23157 );
23158 assert!(engine.refresh_catalog_from(&url, &other_key).await.is_err());
23159 let status = catalog::load_status(&status_path);
23162 assert!(status.failing_since.is_some(), "{status:?}");
23163 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Rejected));
23164 assert!(matches!(
23165 engine.catalog_freshness(),
23166 catalog::CatalogFreshness::Current { .. }
23167 ));
23168
23169 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
23172 assert!(catalog::load_status(&status_path).failing_since.is_none());
23173 let (older, _) = crate::catalog::signed_test_catalog(
23174 crate::catalog::CatalogDoc {
23175 revoked: Vec::new(),
23176 version: 4,
23177 models: vec![],
23178 },
23179 31,
23180 );
23181 *serving.lock().unwrap() = older.envelope();
23182 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
23183 let status = catalog::load_status(&status_path);
23184 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Older));
23185 assert!(
23186 status
23187 .last_error
23188 .as_deref()
23189 .is_some_and(|e| e.contains("older than the cached v5")),
23190 "{status:?}"
23191 );
23192
23193 let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
23195 let gone = format!("http://{}/catalog.json", closed.local_addr().unwrap());
23196 drop(closed);
23197 assert!(engine.refresh_catalog_from(&gone, &key).await.is_err());
23198 assert_eq!(
23199 catalog::load_status(&status_path).last_failure,
23200 Some(catalog::CheckFailure::Unreachable)
23201 );
23202 }
23203
23204 #[cfg(unix)]
23208 #[tokio::test]
23209 async fn a_revoked_installed_model_is_unlinked_and_not_rediscovered() {
23210 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23211 let root = tempfile::tempdir().unwrap();
23212 let root_path = root.path().canonicalize().unwrap();
23213 let (engine, _hub, models) = isolated_engine(&root_path);
23214 let mut row = crate::registry::builtin_catalog()
23215 .into_iter()
23216 .find(|m| matches!(m.source, ModelSource::Local { .. }))
23217 .expect("a downloadable local row");
23218 row.id = "signed/revoked-local".into();
23219 row.name = "signed-revoked-local".into();
23220 row.tags.retain(|t| t != "builtin");
23221 let state = root_path.join("state");
23222 let path = catalog::cache_path(&state);
23223 let doc = |models, revoked, version| catalog::CatalogDoc {
23224 version,
23225 models,
23226 revoked,
23227 };
23228 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 61);
23229 engine
23230 .adopt_verified_catalog(&path, &v2, &key)
23231 .await
23232 .unwrap();
23233
23234 let shared = root_path.join("hub-snapshot");
23236 std::fs::create_dir_all(&shared).unwrap();
23237 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
23238 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
23239 std::os::unix::fs::symlink(&shared, models.join(&row.name)).unwrap();
23240 engine.adopt_model_into_car(&row.id).await.unwrap();
23241
23242 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 61);
23243 engine
23244 .adopt_verified_catalog(&path, &v3, &key)
23245 .await
23246 .unwrap();
23247 assert!(engine.unified_registry().get(&row.id).is_none());
23248 assert!(
23249 std::fs::symlink_metadata(models.join(&row.name)).is_err(),
23250 "CAR's linkage to revoked weights is gone"
23251 );
23252 assert!(
23253 shared.join("model.gguf").exists(),
23254 "the bytes are left to the orphan rules"
23255 );
23256 let restarted =
23257 crate::registry::UnifiedRegistry::new_with_state_root(state, models.clone());
23258 assert!(
23259 restarted.list().iter().all(|m| m.name != row.name),
23260 "nothing on disk brings it back"
23261 );
23262 }
23263
23264 #[cfg(unix)]
23270 #[tokio::test]
23271 async fn revoked_models_without_receipts_never_come_back_and_keep_their_bytes() {
23272 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23273 let root = tempfile::tempdir().unwrap();
23274 let root_path = root.path().canonicalize().unwrap();
23275 let (engine, _hub, models) = isolated_engine(&root_path);
23276 let template = crate::registry::builtin_catalog()
23277 .into_iter()
23278 .find(|m| matches!(m.source, ModelSource::Local { .. }))
23279 .expect("a downloadable local row");
23280 let signed = |id: &str, name: &str| {
23281 let mut m = template.clone();
23282 m.id = id.into();
23283 m.name = name.into();
23284 m.tags.retain(|t| t != "builtin");
23285 m
23286 };
23287 let lazy = signed("signed/lazy", "signed-lazy");
23288 let copied = signed("signed/copied", "signed-copied");
23289 let state = root_path.join("state");
23290 let path = catalog::cache_path(&state);
23291 let doc = |models, revoked, version| catalog::CatalogDoc {
23292 version,
23293 models,
23294 revoked,
23295 };
23296 let (v2, key) =
23297 catalog::signed_test_catalog(doc(vec![lazy.clone(), copied.clone()], vec![], 2), 62);
23298 engine
23299 .adopt_verified_catalog(&path, &v2, &key)
23300 .await
23301 .unwrap();
23302
23303 let shared = root_path.join("hub-snapshot");
23304 std::fs::create_dir_all(&shared).unwrap();
23305 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
23306 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
23307 std::os::unix::fs::symlink(&shared, models.join(&lazy.name)).unwrap();
23308 let copy = models.join(&copied.name);
23309 std::fs::create_dir_all(©).unwrap();
23310 std::fs::write(copy.join("model.gguf"), b"copied weights").unwrap();
23311 std::fs::write(copy.join("tokenizer.json"), b"{}").unwrap();
23312
23313 let (v3, _) = catalog::signed_test_catalog(
23314 doc(vec![], vec![lazy.id.clone(), copied.id.clone()], 3),
23315 62,
23316 );
23317 engine
23318 .adopt_verified_catalog(&path, &v3, &key)
23319 .await
23320 .unwrap();
23321 assert!(
23322 std::fs::symlink_metadata(models.join(&lazy.name)).is_err(),
23323 "links unlinked"
23324 );
23325 assert!(
23326 copy.join("model.gguf").exists(),
23327 "a copy's bytes are never deleted"
23328 );
23329 let restarted =
23330 crate::registry::UnifiedRegistry::new_with_state_root(state.clone(), models.clone());
23331 assert!(
23332 restarted
23333 .list()
23334 .iter()
23335 .all(|m| m.name != lazy.name && m.name != copied.name),
23336 "neither is rediscovered"
23337 );
23338
23339 let want = vec![retire::RevokedCopy {
23340 model_id: copied.id.clone(),
23341 path: copy.clone(),
23342 bytes: (b"copied weights".len() + b"{}".len()) as u64,
23343 }];
23344 assert_eq!(retire::revoked_copies(&state, &models), want);
23345 let hub = root_path.join("hub");
23346 assert_eq!(
23347 engine
23348 .portfolio_in(&hub, &HardwareInfo::detect())
23349 .await
23350 .revoked_copies,
23351 want
23352 );
23353 let opts = doctor::DoctorOptions {
23354 deep: false,
23355 repair: false,
23356 };
23357 let report = doctor::diagnose_at_isolated(&state, &models, &hub, &opts);
23358 assert_eq!(report.revoked_copies, want);
23359 let without = doctor::DoctorReport {
23360 revoked_copies: Vec::new(),
23361 ..report.clone()
23362 };
23363 assert_eq!(
23364 report.is_healthy(),
23365 without.is_healthy(),
23366 "informational only"
23367 );
23368
23369 let outside = root_path.join("outside");
23372 std::fs::create_dir_all(&outside).unwrap();
23373 std::fs::write(outside.join("private"), b"user data").unwrap();
23374 let names_path = catalog::revoked_names_path(&state);
23375 let mut names = catalog::load_revoked_names(&names_path);
23376 names.insert("../outside".into(), copied.id.clone());
23377 catalog::save_revoked_names(&names_path, &names).unwrap();
23378 assert_eq!(retire::revoked_copies(&state, &models), want);
23379
23380 std::fs::remove_dir_all(©).unwrap();
23382 std::fs::create_dir_all(©).unwrap();
23383 std::os::unix::fs::symlink(shared.join("model.gguf"), copy.join("model.gguf")).unwrap();
23384 assert!(retire::revoked_copies(&state, &models).is_empty());
23385 }
23386
23387 #[tokio::test]
23392 async fn agents_pinning_a_revoked_model_are_reported_by_id_or_name() {
23393 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23394 let root = tempfile::tempdir().unwrap();
23395 let root_path = root.path().canonicalize().unwrap();
23396 let (engine, _hub, _models) = isolated_engine(&root_path);
23397 let mut row = crate::registry::builtin_catalog()
23398 .into_iter()
23399 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
23400 .unwrap();
23401 row.id = "signed/revoked-pinned".into();
23402 row.name = "revoked-pinned".into();
23403 let registry = engine.current_registry();
23404 let builtin = crate::registry::builtin_catalog()
23405 .into_iter()
23406 .find(|m| {
23407 !m.is_local()
23408 && registry.find_by_name(&m.name).map(|f| f.id.as_str()) == Some(m.id.as_str())
23409 })
23410 .unwrap();
23411 let fine = crate::registry::builtin_catalog()
23412 .into_iter()
23413 .find(|m| m.id != builtin.id && !m.is_local())
23414 .unwrap();
23415 let path = catalog::cache_path(&root_path.join("state"));
23416 let doc = |models, revoked, version| catalog::CatalogDoc {
23417 version,
23418 models,
23419 revoked,
23420 };
23421 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 65);
23422 engine
23423 .adopt_verified_catalog(&path, &v2, &key)
23424 .await
23425 .unwrap();
23426 let pins = std::collections::BTreeMap::from([
23427 (row.id.clone(), vec!["by-id".to_string()]),
23428 (
23429 row.name.clone(),
23430 vec!["monthly".to_string(), "digest".to_string()],
23431 ),
23432 (builtin.name.clone(), vec!["weekly".to_string()]),
23433 (fine.id.clone(), vec!["untouched".to_string()]),
23434 ]);
23435 engine.set_external_pins(pins.clone());
23436 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
23437
23438 let (v3, _) = catalog::signed_test_catalog(
23439 doc(vec![], vec![row.id.clone(), builtin.id.clone()], 3),
23440 65,
23441 );
23442 engine
23443 .adopt_verified_catalog(&path, &v3, &key)
23444 .await
23445 .unwrap();
23446 let expect = |agents: &[&str], pin: &str, model_id: &str| portfolio::RevokedPin {
23447 pin: pin.to_string(),
23448 model_id: model_id.to_string(),
23449 agents: agents.iter().map(|a| a.to_string()).collect(),
23450 };
23451 let mut want = vec![
23452 expect(&["by-id"], &row.id, &row.id),
23453 expect(&["digest", "monthly"], &row.name, &row.id),
23454 expect(&["weekly"], &builtin.name, &builtin.id),
23455 ];
23456 want.sort_by(|a, b| a.pin.cmp(&b.pin));
23457 assert_eq!(engine.revoked_pins(&engine.current_registry()), want);
23458 let state = root_path.join("state");
23461 assert!(
23462 catalog::load_revoked_names(&catalog::revoked_models_path(&state))
23463 .contains_key(&row.name)
23464 );
23465 assert!(
23466 !catalog::load_revoked_names(&catalog::revoked_names_path(&state))
23467 .contains_key(&row.name)
23468 );
23469 let hub = root_path.join("hub");
23471 let hw = HardwareInfo::detect();
23472 assert_eq!(engine.portfolio_in(&hub, &hw).await.revoked_pins, want);
23473 assert_eq!(engine.concierge_status(false).await.revoked_pins, want);
23474
23475 drop(engine);
23478 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
23479 let (restarted, _hub, _models) = isolated_engine(&root_path);
23480 std::env::remove_var("CAR_CATALOG_PUBKEY");
23481 restarted.set_external_pins(pins);
23482 assert_eq!(restarted.revoked_pins(&restarted.current_registry()), want);
23483 }
23484
23485 #[tokio::test]
23489 async fn a_pin_by_a_name_a_live_row_holds_is_not_reported() {
23490 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23491 let root = tempfile::tempdir().unwrap();
23492 let root_path = root.path().canonicalize().unwrap();
23493 let (engine, _hub, _models) = isolated_engine(&root_path);
23494 let base = crate::registry::builtin_catalog()
23495 .into_iter()
23496 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
23497 .unwrap();
23498 let named = |id: &str| {
23499 let mut row = base.clone();
23500 row.id = id.into();
23501 row.name = "shared-name".into();
23502 row
23503 };
23504 let (old, fresh) = (named("signed/old"), named("signed/fresh"));
23505 let path = catalog::cache_path(&root_path.join("state"));
23506 let doc = |models, revoked, version| catalog::CatalogDoc {
23507 version,
23508 models,
23509 revoked,
23510 };
23511 let (v2, key) = catalog::signed_test_catalog(doc(vec![old.clone()], vec![], 2), 66);
23512 engine
23513 .adopt_verified_catalog(&path, &v2, &key)
23514 .await
23515 .unwrap();
23516 engine.set_external_pins(std::collections::BTreeMap::from([(
23517 "shared-name".to_string(),
23518 vec!["monthly".to_string()],
23519 )]));
23520
23521 let (v3, _) =
23522 catalog::signed_test_catalog(doc(vec![fresh.clone()], vec![old.id.clone()], 3), 66);
23523 engine
23524 .adopt_verified_catalog(&path, &v3, &key)
23525 .await
23526 .unwrap();
23527 assert!(
23528 engine.revoked_pins(&engine.current_registry()).is_empty(),
23529 "the name belongs to a live row"
23530 );
23531
23532 let (v4, _) = catalog::signed_test_catalog(doc(vec![], vec![old.id.clone()], 4), 66);
23535 engine
23536 .adopt_verified_catalog(&path, &v4, &key)
23537 .await
23538 .unwrap();
23539 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
23540 }
23541
23542 #[tokio::test]
23545 async fn a_revoked_lane_default_is_cleared_and_recorded() {
23546 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23547 let root = tempfile::tempdir().unwrap();
23548 let root_path = root.path().canonicalize().unwrap();
23549 let (engine, _hub, _models) = isolated_engine(&root_path);
23550 let mut row = crate::registry::builtin_catalog()
23551 .into_iter()
23552 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
23553 .unwrap();
23554 row.id = "signed/revoked-default".into();
23555 row.name = "revoked-default".into();
23556 let other = crate::registry::builtin_catalog()
23557 .into_iter()
23558 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Code))
23559 .unwrap();
23560 let path = catalog::cache_path(&root_path.join("state"));
23561 let doc = |models, revoked, version| catalog::CatalogDoc {
23562 version,
23563 models,
23564 revoked,
23565 };
23566 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 63);
23567 engine
23568 .adopt_verified_catalog(&path, &v2, &key)
23569 .await
23570 .unwrap();
23571 engine
23572 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
23573 .unwrap();
23574 engine
23575 .set_lane_default(None, crate::intent::UseCase::Coding, &other.id)
23576 .unwrap();
23577
23578 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 63);
23579 engine
23580 .adopt_verified_catalog(&path, &v3, &key)
23581 .await
23582 .unwrap();
23583 assert_eq!(
23584 engine.lane_default(None, crate::intent::UseCase::Assistant),
23585 None
23586 );
23587 assert_eq!(
23588 engine.lane_default(None, crate::intent::UseCase::Coding),
23589 Some(other.id.clone()),
23590 "a lane on another model is untouched"
23591 );
23592 let cleared = engine
23593 .concierge_actions(0)
23594 .into_iter()
23595 .find(|a| a.kind == action_ledger::ConciergeActionKind::RevokedLaneCleared)
23596 .expect("the clearing is recorded");
23597 assert_eq!(cleared.model_id, row.id);
23598 assert!(cleared.detail.contains("revoked"), "{}", cleared.detail);
23599 let now = std::time::SystemTime::now()
23601 .duration_since(std::time::UNIX_EPOCH)
23602 .unwrap()
23603 .as_secs();
23604 let shown = engine
23605 .recent_concierge_actions(now)
23606 .into_iter()
23607 .find(|a| a.kind == "lane_cleared")
23608 .expect("the clearing is in recent actions");
23609 assert_eq!(shown.subject, row.id);
23610 assert_eq!(shown.undo, None);
23611 let refused = engine
23613 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
23614 .unwrap_err();
23615 assert!(refused.contains("revoked"), "{refused}");
23616 }
23617
23618 #[tokio::test]
23621 async fn a_revocation_cached_before_a_restart_still_clears_its_lane() {
23622 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23623 let root = tempfile::tempdir().unwrap();
23624 let root_path = root.path().canonicalize().unwrap();
23625 let mut row = crate::registry::builtin_catalog()
23626 .into_iter()
23627 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
23628 .unwrap();
23629 row.id = "signed/revoked-while-down".into();
23630 row.name = "revoked-while-down".into();
23631 let state = root_path.join("state");
23632 let path = catalog::cache_path(&state);
23633 let doc = |models, revoked, version| catalog::CatalogDoc {
23634 version,
23635 models,
23636 revoked,
23637 };
23638 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 64);
23639 {
23640 let (engine, _hub, _models) = isolated_engine(&root_path);
23641 engine
23642 .adopt_verified_catalog(&path, &v2, &key)
23643 .await
23644 .unwrap();
23645 engine
23646 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
23647 .unwrap();
23648 }
23649 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 64);
23651 catalog::save_verified(&path, &v3).unwrap();
23652 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
23653 let (engine, _hub, _models) = isolated_engine(&root_path);
23654 std::env::remove_var("CAR_CATALOG_PUBKEY");
23655 assert_eq!(
23656 engine.lane_default(None, crate::intent::UseCase::Assistant),
23657 Some(row.id.clone())
23658 );
23659 engine.enforce_catalog_revocations().await;
23660 assert_eq!(
23661 engine.lane_default(None, crate::intent::UseCase::Assistant),
23662 None
23663 );
23664 }
23665
23666 #[tokio::test]
23669 async fn a_request_reads_one_registry_generation_across_a_swap() {
23670 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23671 let root = tempfile::tempdir().unwrap();
23672 let (engine, _hub, _models) = isolated_engine(root.path());
23673 let mut added = crate::registry::builtin_catalog()
23674 .into_iter()
23675 .find(|m| !m.is_local())
23676 .unwrap();
23677 added.id = "signed/mid-request".into();
23678 added.name = "mid-request".into();
23679 let (catalog, _key) = crate::catalog::signed_test_catalog(
23680 crate::catalog::CatalogDoc {
23681 revoked: Vec::new(),
23682 version: 2,
23683 models: vec![added],
23684 },
23685 9,
23686 );
23687 engine
23688 .with_pinned_registry(async {
23689 assert!(engine.model_schema("mid-request").is_none());
23690 engine.swap_signed_catalog(catalog);
23691 assert!(
23692 engine.model_schema("mid-request").is_none(),
23693 "the request's own generation is unchanged"
23694 );
23695 assert!(engine
23696 .current_registry()
23697 .get("signed/mid-request")
23698 .is_some());
23699 })
23700 .await;
23701 assert!(engine.model_schema("mid-request").is_some());
23702 }
23703
23704 #[tokio::test]
23707 async fn a_refreshed_catalog_loads_live_and_a_stale_one_changes_nothing() {
23708 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
23709 let root = tempfile::tempdir().unwrap();
23710 let (engine, _hub, _models) = isolated_engine(root.path());
23711 let template = crate::registry::builtin_catalog()
23712 .into_iter()
23713 .find(|m| !m.is_local())
23714 .unwrap();
23715 let row = |id: &str| {
23716 let mut m = template.clone();
23717 m.id = id.into();
23718 m.name = id.replace('/', "-");
23719 m
23720 };
23721 let path = crate::catalog::cache_path(&root.path().join("state"));
23722 let (v3, key) = crate::catalog::signed_test_catalog(
23723 crate::catalog::CatalogDoc {
23724 revoked: Vec::new(),
23725 version: 3,
23726 models: vec![row("signed/v3")],
23727 },
23728 11,
23729 );
23730 let (v2, _) = crate::catalog::signed_test_catalog(
23731 crate::catalog::CatalogDoc {
23732 revoked: Vec::new(),
23733 version: 2,
23734 models: vec![row("signed/v2")],
23735 },
23736 11,
23737 );
23738 assert_eq!(engine.adopt_verified_catalog(&path, &v3, &key).await, Ok(1));
23739 assert!(engine.unified_registry().get("signed/v3").is_some());
23740 assert!(engine
23741 .adopt_verified_catalog(&path, &v2, &key)
23742 .await
23743 .is_err());
23744 assert!(engine.unified_registry().get("signed/v3").is_some());
23745 assert!(engine.unified_registry().get("signed/v2").is_none());
23746 }
23747
23748 #[cfg(unix)]
23752 #[tokio::test]
23753 async fn retiring_a_whole_repo_model_frees_it_and_leaves_nothing_behind() {
23754 let _serial = retire_test_scope().await;
23755 let root = tempfile::tempdir().unwrap();
23756 let root_path = root.path().canonicalize().unwrap();
23757 let (engine, hub, models) = isolated_engine(&root_path);
23758 let id = plain_mlx_id(&engine);
23759 let id = id.as_str();
23760 let schema = engine.unified_registry().get(id).unwrap().clone();
23761 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23762 unreachable!()
23763 };
23764 let repo = hub_repo(
23765 &hub,
23766 hf_repo,
23767 &[("config.json", 10), ("model.safetensors", 500)],
23768 );
23769 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
23770
23771 let dry = engine.retire_model_in(id, &hub, true, None).await.unwrap();
23772 assert!(!dry.executed);
23773 assert!(repo.exists(), "a dry run deletes nothing");
23774
23775 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23776 assert!(outcome.executed, "{outcome:?}");
23777 assert!(outcome.errors.is_empty(), "{outcome:?}");
23778 assert!(!repo.exists());
23779 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
23780 let leftovers: Vec<_> = std::fs::read_dir(&hub)
23781 .unwrap()
23782 .filter_map(Result::ok)
23783 .collect();
23784 assert!(leftovers.is_empty(), "no quarantine left: {leftovers:?}");
23785 let journals = engine
23786 .model_management_store()
23787 .management_state_dir()
23788 .join("hub-removals");
23789 assert!(std::fs::read_dir(&journals).map_or(true, |mut d| d.next().is_none()));
23790 assert!(engine
23791 .concierge_actions(10)
23792 .iter()
23793 .any(|a| a.kind == action_ledger::ConciergeActionKind::Retire && a.model_id == id));
23794
23795 let again = engine.retire_plan_in(id, &hub).unwrap();
23796 assert!(
23797 again.deletions.is_empty() && again.projections.is_empty(),
23798 "{again:?}"
23799 );
23800 }
23801
23802 #[cfg(unix)]
23805 #[tokio::test]
23806 async fn retiring_a_gguf_model_leaves_the_users_checkout_alone() {
23807 let _serial = retire_test_scope().await;
23808 let root = tempfile::tempdir().unwrap();
23809 let root_path = root.path().canonicalize().unwrap();
23810 let (engine, hub, _models) = isolated_engine(&root_path);
23811 let schema = engine
23812 .unified_registry()
23813 .all()
23814 .find(|m| m.id == "qwen/qwen3-8b:q4_k_m")
23815 .unwrap()
23816 .clone();
23817 let ModelSource::Local {
23818 hf_repo,
23819 hf_filename,
23820 tokenizer_repo,
23821 } = &schema.source
23822 else {
23823 unreachable!()
23824 };
23825 let quant = hub_repo(&hub, hf_repo, &[(hf_filename.as_str(), 300)]);
23826 let base = hub_repo(
23827 &hub,
23828 tokenizer_repo,
23829 &[("tokenizer.json", 7), ("model.safetensors", 900)],
23830 );
23831
23832 let outcome = engine
23833 .retire_model_in(&schema.id, &hub, false, None)
23834 .await
23835 .unwrap();
23836 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
23837 assert!(std::fs::symlink_metadata(quant.join("snapshots/s").join(hf_filename)).is_err());
23838 assert!(!quant.join("blobs/b0").exists());
23839 assert!(base.join("snapshots/s/tokenizer.json").exists());
23840 assert!(base.join("snapshots/s/model.safetensors").exists());
23841 assert_eq!(std::fs::metadata(base.join("blobs/b1")).unwrap().len(), 900);
23842 }
23843
23844 #[cfg(unix)]
23848 #[tokio::test]
23849 async fn retirement_refuses_a_changed_plan_and_a_running_download() {
23850 let _serial = retire_test_scope().await;
23851 let root = tempfile::tempdir().unwrap();
23852 let root_path = root.path().canonicalize().unwrap();
23853 let (engine, hub, _models) = isolated_engine(&root_path);
23854 let id = plain_mlx_id(&engine);
23855 let id = id.as_str();
23856 let ModelSource::Mlx { hf_repo, .. } =
23857 engine.unified_registry().get(id).unwrap().source.clone()
23858 else {
23859 unreachable!()
23860 };
23861 let repo = hub_repo(&hub, &hf_repo, &[("w", 10)]);
23862 let reviewed = engine.retire_model_in(id, &hub, true, None).await.unwrap();
23863 assert!(!reviewed.plan.digest.is_empty());
23864
23865 std::fs::write(repo.join("blobs/late"), vec![0u8; 99]).unwrap();
23866 let changed = engine
23867 .retire_model_in(id, &hub, false, Some(&reviewed.plan.digest))
23868 .await
23869 .unwrap();
23870 assert!(!changed.executed);
23871 assert!(changed
23872 .plan
23873 .refusals
23874 .contains(&retire::RetireRefusal::StateChanged));
23875 assert!(repo.join("blobs/late").exists() && repo.join("blobs/b0").exists());
23876
23877 let busy = crate::download::try_acquire_model_lock(id).unwrap();
23878 let blocked = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23879 assert!(!blocked.executed);
23880 assert!(blocked
23881 .plan
23882 .refusals
23883 .contains(&retire::RetireRefusal::Downloading));
23884 assert!(repo.exists());
23885 drop(busy);
23886
23887 let current = engine.retire_model_in(id, &hub, true, None).await.unwrap();
23888 let done = engine
23889 .retire_model_in(id, &hub, false, Some(¤t.plan.digest))
23890 .await
23891 .unwrap();
23892 assert!(done.executed && done.errors.is_empty(), "{done:?}");
23893 assert!(!repo.exists());
23894 }
23895
23896 #[cfg(unix)]
23899 #[tokio::test]
23900 async fn a_shared_auxiliary_does_not_make_twins() {
23901 let _serial = retire_test_scope().await;
23902 let root = tempfile::tempdir().unwrap();
23903 let root_path = root.path().canonicalize().unwrap();
23904 let (mut engine, hub, _models) = isolated_engine(&root_path);
23905 let ltx = engine
23906 .unified_registry()
23907 .get("mlx/ltx-2.3:q4")
23908 .unwrap()
23909 .clone();
23910 let ModelSource::Mlx {
23911 hf_repo: ltx_repo, ..
23912 } = ltx.source.clone()
23913 else {
23914 unreachable!()
23915 };
23916 let mut second = ltx.clone();
23917 second.id = "user/ltx-q8".into();
23918 second.name = "ltx-q8".into();
23919 second.source = ModelSource::Mlx {
23920 hf_repo: "someone/ltx-q8".into(),
23921 hf_weight_file: None,
23922 };
23923 engine.unified_registry_mut().register_project_model(second);
23924 let ltx_dir = hub_repo(&hub, <x_repo, &[("config.json", 5), ("w", 50)]);
23925 hub_repo(&hub, "someone/ltx-q8", &[("config.json", 5), ("w", 70)]);
23926 hub_repo(
23927 &hub,
23928 crate::registry::LTX_TEXT_ENCODER,
23929 &[("config.json", 5), ("w", 90)],
23930 );
23931 let plan = engine.retire_plan_in("mlx/ltx-2.3:q4", &hub).unwrap();
23932 assert!(plan.also_retires.is_empty(), "{plan:?}");
23933 assert!(
23934 plan.deletions.iter().all(|d| !matches!(
23935 d,
23936 retire::HubDeletion::Repo { repo, .. } if repo == "someone/ltx-q8"
23937 )),
23938 "{plan:?}"
23939 );
23940 assert!(ltx_dir.exists());
23941 }
23942
23943 #[cfg(unix)]
23945 #[tokio::test]
23946 async fn twins_sharing_a_repo_retire_together() {
23947 let _serial = retire_test_scope().await;
23948 let root = tempfile::tempdir().unwrap();
23949 let root_path = root.path().canonicalize().unwrap();
23950 let (engine, hub, _models) = isolated_engine(&root_path);
23951 let id = "mlx/gemma-4-12b-it:4bit";
23952 let ModelSource::Mlx { hf_repo, .. } =
23953 engine.unified_registry().get(id).unwrap().source.clone()
23954 else {
23955 unreachable!()
23956 };
23957 let repo = hub_repo(&hub, &hf_repo, &[("config.json", 5), ("w", 50)]);
23958 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23959 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
23960 assert_eq!(
23961 outcome.plan.also_retires,
23962 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
23963 );
23964 assert!(!repo.exists());
23965 let retired: Vec<String> = engine
23966 .concierge_actions(10)
23967 .into_iter()
23968 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
23969 .map(|a| a.model_id)
23970 .collect();
23971 assert!(retired.contains(&id.to_string()), "{retired:?}");
23972 assert!(
23973 retired.contains(&"vllm-mlx/gemma-4-12b-it".to_string()),
23974 "{retired:?}"
23975 );
23976
23977 let again = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23979 assert!(!again.executed);
23980 assert!(again
23981 .plan
23982 .refusals
23983 .contains(&retire::RetireRefusal::NothingToRetire));
23984 assert_eq!(
23985 engine
23986 .concierge_actions(10)
23987 .iter()
23988 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
23989 .count(),
23990 2
23991 );
23992 }
23993
23994 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
23999 #[tokio::test]
24000 async fn a_receipted_projection_retires_and_the_row_stays_fetchable() {
24001 let _serial = retire_test_scope().await;
24002 let root = tempfile::tempdir().unwrap();
24003 let root_path = root.path().canonicalize().unwrap();
24004 let (engine, hub, models) = isolated_engine(&root_path);
24005 let id = plain_mlx_id(&engine);
24006 let id = id.as_str();
24007 let schema = engine.unified_registry().get(id).unwrap().clone();
24008 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
24009 unreachable!()
24010 };
24011 let repo = hub_repo(
24012 &hub,
24013 hf_repo,
24014 &[("config.json", 10), ("model.safetensors", 500)],
24015 );
24016 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
24017 engine.adopt_model_into_car(id).await.unwrap();
24018 let store = engine.model_management_store();
24019 assert!(store.receipt_path(id).exists());
24020
24021 let plan = engine.retire_plan_in(id, &hub).unwrap();
24022 assert!(
24023 plan.projections
24024 .iter()
24025 .any(|p| p.receipt_for.as_deref() == Some(id)),
24026 "{plan:?}"
24027 );
24028 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
24029 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
24030 assert!(!repo.exists());
24031 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
24032 assert!(!store.receipt_path(id).exists());
24033 assert!(!store.tombstone_path(id).exists(), "retired, not disabled");
24034 assert!(store.car_enabled(id).unwrap());
24035 let intents = store.management_state_dir().join("retiring");
24036 assert!(std::fs::read_dir(&intents).map_or(true, |mut d| d.next().is_none()));
24037 }
24038
24039 #[cfg(unix)]
24041 #[test]
24042 fn a_projection_holding_a_real_file_is_not_removed() {
24043 let root = tempfile::tempdir().unwrap();
24044 let models = root.path().canonicalize().unwrap().join("weights");
24045 let projection = models.join("Some-MLX");
24046 std::fs::create_dir_all(&projection).unwrap();
24047 std::os::unix::fs::symlink("/nowhere", projection.join("link")).unwrap();
24048 std::fs::write(projection.join("notes.txt"), b"mine").unwrap();
24049 assert!(retire::remove_links_only_tree(&projection, &models).is_err());
24050 assert!(std::fs::symlink_metadata(projection.join("link")).is_ok());
24051 assert!(projection.join("notes.txt").exists());
24052 let elsewhere = root.path().canonicalize().unwrap().join("elsewhere");
24054 std::fs::create_dir_all(&elsewhere).unwrap();
24055 assert!(retire::remove_links_only_tree(&elsewhere, &models).is_err());
24056 assert!(elsewhere.exists());
24057 }
24058
24059 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
24064 #[tokio::test]
24065 async fn portfolio_keeps_defaults_and_waits_for_evidence() {
24066 let _serial = retire_test_scope().await;
24067 let root = tempfile::tempdir().unwrap();
24068 let root_path = root.path().canonicalize().unwrap();
24069 let (engine, hub, models) = isolated_engine(&root_path);
24070 let hw = HardwareInfo {
24071 total_ram_mb: 64 * 1024,
24072 ..HardwareInfo::detect()
24073 };
24074 let protected = engine.protected_models(&engine.unified_registry(), true);
24077 let idle_id = [
24078 "mlx/qwen3-30b-a3b:4bit",
24079 "mlx/qwen3-1.7b:3bit",
24080 "mlx/qwen3-0.6b:6bit",
24081 ]
24082 .into_iter()
24083 .find(|id| !protected.contains_key(*id))
24084 .unwrap();
24085 let mut installed = Vec::new();
24086 for id in [idle_id, "mlx/qwen3-8b:4bit"] {
24087 let schema = engine.unified_registry().get(id).unwrap().clone();
24088 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
24089 unreachable!()
24090 };
24091 let repo = hub_repo(
24092 &hub,
24093 hf_repo,
24094 &[("config.json", 10), ("model.safetensors", 500)],
24095 );
24096 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
24097 .unwrap();
24098 installed.push(id);
24099 }
24100 engine.lane_defaults_cache.write().unwrap().set(
24101 None,
24102 crate::intent::UseCase::Assistant,
24103 "mlx/qwen3-8b:4bit".to_string(),
24104 1,
24105 );
24106
24107 let fresh = engine.portfolio_in(&hub, &hw).await;
24108 let ids: Vec<&str> = fresh.models.iter().map(|m| m.model_id.as_str()).collect();
24109 for id in &installed {
24110 assert!(ids.contains(id), "{ids:?}");
24111 }
24112 let verdict = |p: &portfolio::Portfolio, id: &str| {
24113 p.models.iter().find(|m| m.model_id == id).unwrap().clone()
24114 };
24115 assert_eq!(
24116 verdict(&fresh, "mlx/qwen3-8b:4bit").verdict,
24117 portfolio::Verdict::Protected
24118 );
24119 let small = verdict(&fresh, idle_id);
24120 assert_eq!(small.verdict, portfolio::Verdict::Unknown, "{small:?}");
24121 assert_eq!(small.action, portfolio::PortfolioAction::Keep);
24122
24123 engine
24125 .model_management_store()
24126 .set_usage_tracking_since_for_test(1);
24127 let aged = engine.portfolio_in(&hub, &hw).await;
24128 let small = verdict(&aged, idle_id);
24129 assert_eq!(small.verdict, portfolio::Verdict::Idle, "{small:?}");
24130 assert_eq!(small.action, portfolio::PortfolioAction::Retire);
24131 assert!(aged.reclaimable_bytes >= 510, "{aged:?}");
24132 let digest = small.retire.unwrap().digest;
24133 let done = engine
24134 .retire_model_in(idle_id, &hub, false, Some(&digest))
24135 .await
24136 .unwrap();
24137 assert!(done.executed && done.errors.is_empty(), "{done:?}");
24138 assert_eq!(
24139 verdict(&aged, "mlx/qwen3-8b:4bit").verdict,
24140 portfolio::Verdict::Protected
24141 );
24142
24143 let pinned = plain_mlx_id(&engine);
24146 engine.set_external_pins(std::collections::BTreeMap::from([(
24147 pinned.clone(),
24148 vec!["monthly".to_string()],
24149 )]));
24150 let plan = engine.retire_plan_in(&pinned, &hub).unwrap();
24151 assert!(
24152 plan.refusals.iter().any(|r| matches!(
24153 r,
24154 retire::RetireRefusal::Protected { why } if why.contains("monthly")
24155 )),
24156 "{plan:?}"
24157 );
24158 engine.set_external_pins(Default::default());
24159
24160 let tool = engine
24162 .retire_plan_in(LOCAL_TOOL_MODEL_PULL_ID, &hub)
24163 .unwrap();
24164 assert!(
24165 tool.refusals
24166 .iter()
24167 .any(|r| matches!(r, retire::RetireRefusal::Protected { .. })),
24168 "{tool:?}"
24169 );
24170 }
24171
24172 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
24179 #[tokio::test]
24180 async fn maintenance_retires_under_pressure_and_respects_keep_and_churn() {
24181 use crate::update_prefs::UpdatePolicy;
24182 let _serial = retire_test_scope().await;
24183 let root = tempfile::tempdir().unwrap();
24184 let root_path = root.path().canonicalize().unwrap();
24185 let (engine, hub, _models) = isolated_engine(&root_path);
24186 let hw = HardwareInfo {
24187 total_ram_mb: 64 * 1024,
24188 ..HardwareInfo::detect()
24189 };
24190 let id = plain_mlx_id(&engine);
24191 let ModelSource::Mlx { hf_repo, .. } =
24192 engine.unified_registry().get(&id).unwrap().source.clone()
24193 else {
24194 unreachable!()
24195 };
24196 let layout = || hub_repo(&hub, &hf_repo, &[("config.json", 10), ("w", 500)]);
24197 let repo = layout();
24198 engine
24199 .model_management_store()
24200 .set_usage_tracking_since_for_test(1);
24201 let gb = 1_000_000_000;
24202 let roomy = Some(maintenance::DiskSpace {
24203 free_bytes: 500 * gb,
24204 total_bytes: 1_000 * gb,
24205 });
24206 let tight = Some(maintenance::DiskSpace {
24207 free_bytes: 50 * gb,
24208 total_bytes: 1_000 * gb,
24209 });
24210 let none = crate::nudge::NudgeState::default();
24211
24212 let calm = engine
24213 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
24214 .await;
24215 assert!(calm.retired.is_none(), "{calm:?}");
24216 assert!(calm.plan.held.iter().any(|h| h.model_id == id), "{calm:?}");
24217 assert!(repo.exists());
24218
24219 let notify = engine
24220 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, tight)
24221 .await;
24222 assert!(notify.retired.is_none());
24223 assert!(
24224 notify.plan.suggest.iter().any(|d| d.model_id == id),
24225 "{notify:?}"
24226 );
24227 assert!(repo.exists(), "Notify never acts");
24228
24229 let off = engine
24230 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Off, &none, tight)
24231 .await;
24232 assert_eq!(off.plan, maintenance::MaintenancePlan::default());
24233
24234 let mut kept = crate::nudge::NudgeState::default();
24235 kept.dismiss_labeled(
24236 maintenance::keep_key(&id),
24237 crate::concierge::DismissReason::Wrong,
24238 1,
24239 );
24240 let held = engine
24241 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &kept, tight)
24242 .await;
24243 assert!(held.retired.is_none());
24244 assert!(repo.exists(), "kept");
24245
24246 let pressed = engine
24247 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
24248 .await;
24249 let retired = pressed.retired.as_ref().expect("retired under pressure");
24250 assert_eq!(retired.model_id, id);
24251 assert!(!repo.exists());
24252 let record = engine.portfolio_in(&hub, &hw).await.recent_actions;
24254 let entry = record
24255 .iter()
24256 .find(|a| a.kind == "retire" && a.subject == id)
24257 .expect("the retirement is on record");
24258 assert_eq!(
24259 entry.undo.as_deref(),
24260 Some(format!("car models pull {id}").as_str())
24261 );
24262 assert!(!entry.interrupted);
24263 let kinds: Vec<_> = engine
24264 .concierge_actions(10)
24265 .into_iter()
24266 .filter(|a| a.model_id == id)
24267 .map(|a| a.kind)
24268 .collect();
24269 assert!(kinds.contains(&action_ledger::ConciergeActionKind::RetireStarted));
24270 assert!(kinds.contains(&action_ledger::ConciergeActionKind::Retire));
24271
24272 let repo = layout();
24274 let again = engine
24275 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
24276 .await;
24277 assert!(again.retired.is_none(), "{again:?}");
24278 assert!(again
24279 .plan
24280 .held
24281 .iter()
24282 .any(|h| h.model_id == id && h.why.contains("fetched back")));
24283 assert!(repo.exists());
24284
24285 let old = repo.join("blobs/aaaa.incomplete");
24287 let fresh = repo.join("blobs/bbbb.incomplete");
24288 std::fs::write(&old, b"half").unwrap();
24289 std::fs::write(&fresh, b"half").unwrap();
24290 let two_days_ago =
24291 std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 24 * 60 * 60);
24292 std::fs::File::options()
24293 .write(true)
24294 .open(&old)
24295 .unwrap()
24296 .set_modified(two_days_ago)
24297 .unwrap();
24298 let swept = engine
24299 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
24300 .await;
24301 assert_eq!(swept.partials_discarded, vec![old.clone()], "{swept:?}");
24302 assert!(!old.exists() && fresh.exists());
24303 }
24304
24305 #[cfg(unix)]
24308 #[tokio::test]
24309 async fn maintenance_retires_an_orphan_car_fetched_only_under_pressure() {
24310 use crate::update_prefs::UpdatePolicy;
24311 let _serial = retire_test_scope().await;
24312 let root = tempfile::tempdir().unwrap();
24313 let root_path = root.path().canonicalize().unwrap();
24314 let (engine, hub, models) = isolated_engine(&root_path);
24315 let hw = HardwareInfo::detect();
24316 let orphan = hub_repo(&hub, "dropped/model", &[("config.json", 10), ("w", 900)]);
24317 let theirs = hub_repo(
24318 &hub,
24319 "someone/checkout",
24320 &[("tokenizer.json", 5), ("w", 50)],
24321 );
24322 let fetched = |repo: &str, files: &[&str]| -> Vec<retire::Fetched> {
24323 files
24324 .iter()
24325 .map(|f| (repo.to_string(), f.to_string(), Some("s".to_string())))
24326 .collect()
24327 };
24328 retire::record_download(
24329 &root_path.join("state"),
24330 &hub,
24331 "old/row",
24332 &fetched("dropped/model", &["config.json", "w"]),
24333 );
24334 retire::record_download(
24335 &root_path.join("state"),
24336 &hub,
24337 "old/row",
24338 &fetched("someone/checkout", &["tokenizer.json"]),
24339 );
24340 let gb = 1_000_000_000;
24341 let none = crate::nudge::NudgeState::default();
24342 let roomy = Some(maintenance::DiskSpace {
24343 free_bytes: 500 * gb,
24344 total_bytes: 1_000 * gb,
24345 });
24346 let calm = engine
24347 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
24348 .await;
24349 assert!(calm.orphan_retired.is_none());
24350 assert!(orphan.exists());
24351
24352 let tight = Some(maintenance::DiskSpace {
24353 free_bytes: 10 * gb,
24354 total_bytes: 1_000 * gb,
24355 });
24356 let link = models.join("Dropped-Model");
24359 std::os::unix::fs::symlink(orphan.join("snapshots/s"), &link).unwrap();
24360 let blocked = engine
24361 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
24362 .await;
24363 assert!(blocked.orphan_retired.is_none(), "{blocked:?}");
24364 assert!(
24365 blocked
24366 .errors
24367 .iter()
24368 .any(|e| e.contains("resolves into it")),
24369 "{blocked:?}"
24370 );
24371 assert!(orphan.exists());
24372 std::fs::remove_file(&link).unwrap();
24373
24374 let pressed = engine
24375 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
24376 .await;
24377 assert_eq!(
24378 pressed.orphan_retired.as_ref().map(|o| o.repo.as_str()),
24379 Some("dropped/model"),
24380 "{pressed:?}"
24381 );
24382 assert!(!orphan.exists());
24383 assert!(theirs.exists(), "a user's checkout is never touched");
24384 let again = engine
24385 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
24386 .await;
24387 assert!(again.orphan_retired.is_none(), "{again:?}");
24388 assert!(theirs.exists());
24389 }
24390
24391 #[tokio::test]
24395 async fn maintenance_raises_an_unchosen_memory_policy_on_evidence() {
24396 use crate::resource_policy::{
24397 FileResourcePolicyRepository, ResourcePolicy, ResourcePolicyRepository, ResourceProfile,
24398 };
24399 use crate::update_prefs::UpdatePolicy;
24400 let _serial = retire_test_scope().await;
24401 let root = tempfile::tempdir().unwrap();
24402 let root_path = root.path().canonicalize().unwrap();
24403 let hub = root_path.join("hub");
24404 std::fs::create_dir_all(&hub).unwrap();
24405 let engine = InferenceEngine::new(InferenceConfig {
24406 state_root: root_path.join("state"),
24407 models_dir: root_path.join("weights"),
24408 ..InferenceConfig::default()
24409 });
24410 let hw = HardwareInfo {
24411 total_ram_mb: 64 * 1024,
24412 ..HardwareInfo::detect()
24413 };
24414 let big = engine
24416 .unified_registry()
24417 .all()
24418 .find(|m| {
24419 m.is_local()
24420 && recommend::model_fit(
24421 m,
24422 &hw,
24423 Some(&ResourcePolicy::everyday()),
24424 &CheckpointGeometry::default(),
24425 )
24426 .fit == recommend::ModelFitStatus::TooBig
24427 && recommend::model_fit(
24428 m,
24429 &hw,
24430 Some(&ResourcePolicy::local_focused()),
24431 &CheckpointGeometry::default(),
24432 )
24433 .fit == recommend::ModelFitStatus::Fits
24434 })
24435 .expect("a row only Local-focused admits")
24436 .id
24437 .clone();
24438 let ledger = engine
24439 .config
24440 .state_models_dir()
24441 .join("outcome_ledger.jsonl");
24442 std::fs::create_dir_all(ledger.parent().unwrap()).unwrap();
24443 let now = std::time::SystemTime::now()
24444 .duration_since(std::time::UNIX_EPOCH)
24445 .unwrap()
24446 .as_secs();
24447 let lines: String = (0..3)
24448 .map(|i| {
24449 format!(
24450 "{}\n",
24451 serde_json::json!({
24452 "trace_id": format!("t{i}"), "model_id": big, "task": "generate",
24453 "routing_reason": "explicit model", "latency_ms": 0,
24454 "input_tokens": 0, "output_tokens": 0, "error": "refused for memory",
24455 "attribution": "resource_blocked",
24456 "timestamp": now - 60 - i * 24 * 60 * 60
24457 })
24458 )
24459 })
24460 .collect();
24461 std::fs::write(&ledger, lines).unwrap();
24462 let none = crate::nudge::NudgeState::default();
24463 let repository = FileResourcePolicyRepository::new(root_path.join("state"));
24464
24465 let unobserved = engine
24467 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
24468 .await;
24469 assert!(unobserved.resource_policy.is_none(), "{unobserved:?}");
24470
24471 engine
24472 .model_management_store()
24473 .set_usage_tracking_since_for_test(now - 4 * 24 * 60 * 60);
24474 let notify = engine
24475 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, None)
24476 .await;
24477 let suggested = notify.resource_policy.expect("suggested under Notify");
24478 assert!(!suggested.applied);
24479 assert!(!repository.path().exists(), "Notify saves nothing");
24480
24481 let auto = engine
24482 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
24483 .await;
24484 assert!(
24485 auto.resource_policy.as_ref().is_some_and(|p| p.applied),
24486 "{auto:?}"
24487 );
24488 assert_eq!(repository.load().unwrap(), ResourcePolicy::local_focused());
24489 assert_eq!(
24490 repository.concierge_marker(),
24491 Some(ResourcePolicy::local_focused())
24492 );
24493 assert_eq!(
24494 engine.active_local_resource_policy().policy.profile,
24495 ResourceProfile::LocalFocused
24496 );
24497
24498 let small = engine
24501 .unified_registry()
24502 .all()
24503 .find(|m| {
24504 m.is_local()
24505 && recommend::model_fit(
24506 m,
24507 &hw,
24508 Some(&ResourcePolicy::everyday()),
24509 &CheckpointGeometry::default(),
24510 )
24511 .fit == recommend::ModelFitStatus::Fits
24512 })
24513 .unwrap()
24514 .id
24515 .clone();
24516 let later = std::time::SystemTime::now()
24517 .duration_since(std::time::UNIX_EPOCH)
24518 .unwrap()
24519 .as_secs()
24520 + 1;
24521 let mut text = std::fs::read_to_string(&ledger).unwrap();
24522 for i in 0..3 {
24523 text.push_str(&format!(
24524 "{}\n",
24525 serde_json::json!({
24526 "trace_id": format!("live{i}"), "model_id": small, "task": "generate",
24527 "routing_reason": "explicit model", "latency_ms": 0,
24528 "input_tokens": 0, "output_tokens": 0, "error": "not enough memory free",
24529 "attribution": "resource_blocked", "timestamp": later
24530 })
24531 ));
24532 }
24533 std::fs::write(&ledger, text).unwrap();
24534 let reverted = engine
24535 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
24536 .await;
24537 let undo = reverted.resource_policy.expect("reverted");
24538 assert!(
24539 undo.applied && undo.to == ResourceProfile::Everyday,
24540 "{undo:?}"
24541 );
24542 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
24543 assert!(repository.concierge_choice().unwrap().reverted);
24544 let settled = engine
24545 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
24546 .await;
24547 assert!(settled.resource_policy.is_none(), "{settled:?}");
24548
24549 repository.save(&ResourcePolicy::everyday()).unwrap();
24551 repository.clear_concierge_marker().unwrap();
24552 engine.apply_local_resource_policy(ResourcePolicy::everyday());
24553 let chosen = engine
24554 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
24555 .await;
24556 assert!(chosen.resource_policy.is_none(), "{chosen:?}");
24557 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
24558 }
24559
24560 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
24566 #[tokio::test]
24567 async fn a_curated_upgrade_switches_the_lane_that_used_the_old_model() {
24568 let _serial = retire_test_scope().await;
24569 let root = tempfile::tempdir().unwrap();
24570 let root_path = root.path().canonicalize().unwrap();
24571 let (engine, hub, models) = isolated_engine(&root_path);
24572 let from = plain_mlx_id(&engine);
24573 let to = "mlx/qwen3-8b:4bit".to_string();
24574 for id in [&from, &to] {
24575 let schema = engine.unified_registry().get(id).unwrap().clone();
24576 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
24577 unreachable!()
24578 };
24579 let repo = hub_repo(
24580 &hub,
24581 hf_repo,
24582 &[("config.json", 10), ("model.safetensors", 500)],
24583 );
24584 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
24585 .unwrap();
24586 }
24587 let finding = |source| crate::upgrade::UpgradeFinding {
24588 from_id: from.clone(),
24589 from_name: from.clone(),
24590 to_id: to.clone(),
24591 to_name: to.clone(),
24592 reason: "newer".into(),
24593 trust_tier: crate::schema::TrustTier::Curated,
24594 source,
24595 target_pullable: true,
24596 };
24597 let now = 1;
24598 let none = crate::nudge::NudgeState::default();
24599
24600 let idle = engine
24602 .apply_one_curated_upgrade_from(
24603 vec![finding(crate::upgrade::UpgradeSource::Curated)],
24604 &hub,
24605 &none,
24606 now,
24607 )
24608 .await
24609 .unwrap();
24610 assert!(idle.is_none());
24611
24612 engine
24613 .set_lane_default(None, crate::intent::UseCase::Assistant, &from)
24614 .unwrap();
24615 let community = engine
24617 .apply_one_curated_upgrade_from(
24618 vec![finding(crate::upgrade::UpgradeSource::Upstream)],
24619 &hub,
24620 &none,
24621 now,
24622 )
24623 .await
24624 .unwrap();
24625 assert!(community.is_none());
24626 let mut dismissed = crate::nudge::NudgeState::default();
24627 dismissed.dismiss(&format!("{from}=>{to}"));
24628 assert!(engine
24629 .apply_one_curated_upgrade_from(
24630 vec![finding(crate::upgrade::UpgradeSource::Curated)],
24631 &hub,
24632 &dismissed,
24633 now,
24634 )
24635 .await
24636 .unwrap()
24637 .is_none());
24638
24639 let upgraded = engine
24640 .apply_one_curated_upgrade_from(
24641 vec![finding(crate::upgrade::UpgradeSource::Curated)],
24642 &hub,
24643 &none,
24644 now,
24645 )
24646 .await
24647 .unwrap()
24648 .expect("upgraded");
24649 assert_eq!(upgraded.lanes, vec!["assistant".to_string()]);
24650 assert_eq!(
24651 engine
24652 .lane_default(None, crate::intent::UseCase::Assistant)
24653 .as_deref(),
24654 Some(to.as_str())
24655 );
24656 let record = engine.recent_concierge_actions(now);
24657 let entry = record
24658 .iter()
24659 .find(|a| a.kind == "upgrade")
24660 .expect("on record");
24661 assert_eq!(
24662 entry.undo.as_deref(),
24663 Some("car concierge rollback assistant")
24664 );
24665 assert_eq!(entry.use_case.as_deref(), Some("assistant"));
24666 assert_eq!(entry.prior.as_deref(), Some(from.as_str()));
24667 }
24668
24669 #[tokio::test]
24672 async fn concierge_ask_without_a_question_states_the_facts_without_a_model() {
24673 let _serial = retire_test_scope().await;
24674 let root = tempfile::tempdir().unwrap();
24675 let engine = InferenceEngine::new(InferenceConfig {
24676 state_root: root.path().join("state"),
24677 models_dir: root.path().join("weights"),
24678 ..InferenceConfig::default()
24679 });
24680 let answer = engine.concierge_ask(None).await.unwrap();
24681 assert!(answer.model_used.is_none());
24682 assert_eq!(answer.answer, answer.summary);
24683 assert!(
24684 answer.summary.contains("Installed local models:"),
24685 "{}",
24686 answer.summary
24687 );
24688 assert!(answer.summary.contains("Memory policy for local models:"));
24689 }
24690
24691 #[cfg(unix)]
24693 #[tokio::test]
24694 async fn a_refused_retirement_deletes_nothing() {
24695 let _serial = retire_test_scope().await;
24696 let root = tempfile::tempdir().unwrap();
24697 let root_path = root.path().canonicalize().unwrap();
24698 let (engine, hub, _models) = isolated_engine(&root_path);
24699 let id = "mlx/qwen3-4b:4bit";
24700 let schema = engine.unified_registry().get(id).unwrap().clone();
24701 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
24702 unreachable!()
24703 };
24704 let repo = hub_repo(&hub, hf_repo, &[("w", 10)]);
24705 engine.lane_defaults_cache.write().unwrap().set(
24706 None,
24707 crate::intent::UseCase::Assistant,
24708 id.to_string(),
24709 1,
24710 );
24711 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
24712 assert!(!outcome.executed);
24713 assert!(repo.exists());
24714 }
24715
24716 #[test]
24719 fn retire_plan_refuses_a_model_stored_outside_the_hub() {
24720 let root = tempfile::tempdir().unwrap();
24721 let engine = InferenceEngine::new(InferenceConfig {
24722 state_root: root.path().join("state"),
24723 models_dir: root.path().join("weights"),
24724 ..InferenceConfig::default()
24725 });
24726 let whisper = engine
24727 .unified_registry()
24728 .all()
24729 .find(|m| matches!(m.source, ModelSource::WhisperCpp { .. }))
24730 .expect("a whisper.cpp row")
24731 .id
24732 .clone();
24733 let plan = engine.retire_plan_in(&whisper, root.path()).unwrap();
24734 assert!(
24735 matches!(
24736 &plan.refusals[..],
24737 [retire::RetireRefusal::Unsupported { .. }]
24738 ),
24739 "{plan:?}"
24740 );
24741 }
24742
24743 #[test]
24745 fn downloads_are_recorded_per_hub() {
24746 let root = tempfile::tempdir().unwrap();
24747 let fetched = [(
24748 "renamed/repo".to_string(),
24749 "model.safetensors".to_string(),
24750 Some("r1".to_string()),
24751 )];
24752 let hub = root.path().join("hub");
24753 retire::record_download(root.path(), &hub, "old/row", &fetched);
24754 let recorded = retire::recorded_files(root.path(), &hub);
24755 assert_eq!(
24756 recorded.get("renamed/repo"),
24757 Some(&retire::Owned::Files(std::collections::BTreeSet::from([
24758 retire::OwnedFile {
24759 path: "model.safetensors".into(),
24760 revision: Some("r1".into()),
24761 }
24762 ])))
24763 );
24764 assert!(retire::recorded_files(root.path(), Path::new("/elsewhere")).is_empty());
24765 }
24766
24767 #[cfg(unix)]
24772 #[tokio::test]
24773 async fn a_failed_ledger_write_leaves_the_profiles_untouched_and_retries_once() {
24774 use std::os::unix::fs::PermissionsExt;
24775 let root = tempfile::tempdir().unwrap();
24776 let state = root.path().join("state");
24777 let models = state.join("models");
24778 std::fs::create_dir_all(&models).unwrap();
24779 let ledger = models.join("outcome_ledger.jsonl");
24780 let real = "mlx/qwen3-4b:4bit";
24781 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
24782 let mut seed = OutcomeTracker::new();
24783 seed.import_profiles(profiles);
24784 seed.save_to_file(&models.join("outcome_profiles.json"))
24785 .unwrap();
24786 let engine = InferenceEngine::new(InferenceConfig {
24787 state_root: state.clone(),
24788 models_dir: root.path().join("weights"),
24789 ..InferenceConfig::default()
24790 });
24791 let before = profile_json(&*engine.outcome_tracker().read().await, real);
24792 let dirty_before = engine.outcome_tracker().read().await.is_dirty();
24793 assert!(
24794 engine
24795 .outcome_tracker()
24796 .read()
24797 .await
24798 .profile(real)
24799 .unwrap()
24800 .fail_count
24801 > 1
24802 );
24803
24804 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o555)).unwrap();
24806 let failed = engine.reconcile_outcome_ledger().await;
24807 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o755)).unwrap();
24808 assert!(failed.is_err(), "the write was refused");
24809 {
24810 let tracker = engine.outcome_tracker();
24811 let tracker = tracker.read().await;
24812 assert_eq!(profile_json(&tracker, real), before, "profiles untouched");
24813 assert_eq!(
24814 tracker.is_dirty(),
24815 dirty_before,
24816 "the failed reconcile left nothing new for the periodic save to persist"
24817 );
24818 }
24819
24820 engine.reconcile_outcome_ledger().await.unwrap();
24822 engine.reconcile_outcome_ledger().await.unwrap();
24823 let p = engine
24824 .outcome_tracker()
24825 .read()
24826 .await
24827 .profile(real)
24828 .unwrap()
24829 .clone();
24830 assert_eq!(
24831 p.fail_count, 1,
24832 "only the genuine failure remains, taken back once"
24833 );
24834 }
24835
24836 #[test]
24839 fn engine_construction_never_rewrites_the_ledger() {
24840 let root = tempfile::tempdir().unwrap();
24841 let state = root.path().join("state");
24842 let models = state.join("models");
24843 std::fs::create_dir_all(&models).unwrap();
24844 let ledger = models.join("outcome_ledger.jsonl");
24845 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
24846 let before = std::fs::read(&ledger).unwrap();
24847 let _engine = InferenceEngine::new(InferenceConfig {
24848 state_root: state,
24849 models_dir: root.path().join("weights"),
24850 ..InferenceConfig::default()
24851 });
24852 assert_eq!(std::fs::read(&ledger).unwrap(), before);
24853 }
24854
24855 #[test]
24856 fn dispatch_outcome_guards_keep_real_failures_and_existing_rejections_distinct() {
24857 let model_id = "test/model";
24858
24859 let real_failure = InferenceError::InferenceFailed("provider returned HTTP 500".into());
24860 let mut tracker = OutcomeTracker::new();
24861 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
24862 record_dispatch_failure(&mut tracker, &trace, &real_failure);
24863 let profile = tracker.profile(model_id).expect("real failure profile");
24864 assert_eq!(profile.success_count, 0);
24865 assert_eq!(profile.fail_count, 1);
24866 let receipt = tracker.drain_ledger().pop().unwrap();
24867 assert_eq!(receipt.success, Some(false));
24868 assert!(error_counts_against_circuit_breaker(&real_failure));
24869 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
24870 breakers.record_failure(model_id);
24871 assert_eq!(
24872 breakers.state(model_id),
24873 Some(crate::routing_ext::CircuitState::Open)
24874 );
24875
24876 for rejection in [
24877 InferenceError::GatewayUnconfigured {
24878 provider: "parslee".into(),
24879 namespace: "parslee/openrouter/".into(),
24880 status: 503,
24881 message: "not configured".into(),
24882 },
24883 InferenceError::ContentRefused {
24884 provider: "parslee".into(),
24885 kind: Some("invalid_request_error".into()),
24886 code: Some("content_policy_violation".into()),
24887 message: "content refused".into(),
24888 },
24889 ] {
24890 let mut tracker = OutcomeTracker::new();
24891 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
24892 record_dispatch_failure(&mut tracker, &trace, &rejection);
24893 assert!(
24894 tracker.profile(model_id).is_none(),
24895 "existing rejection changed model profile: {rejection}"
24896 );
24897 let receipt = tracker.drain_ledger().pop().unwrap();
24898 assert_eq!(receipt.success, None, "{rejection}");
24899 assert_eq!(receipt.quality, None, "{rejection}");
24900 assert!(!error_counts_against_circuit_breaker(&rejection));
24901 }
24902 }
24903
24904 #[tokio::test(flavor = "current_thread")]
24917 async fn openrouter_out_of_credits_does_not_degrade_the_model() {
24918 use wiremock::matchers::{method, path};
24919 use wiremock::{Mock, MockServer, ResponseTemplate};
24920
24921 let _credential_scope = crate::openrouter::test_credential_scope();
24922 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
24923 let server = MockServer::start().await;
24924 Mock::given(method("POST"))
24925 .and(path("/v1/chat/completions"))
24926 .respond_with(
24927 ResponseTemplate::new(402)
24928 .set_body_string(r#"{"error":{"code":402,"message":"Insufficient credits"}}"#),
24929 )
24930 .mount(&server)
24931 .await;
24932
24933 let tmp = TempDir::new().unwrap();
24934 let models_dir = tmp.path().join("models");
24935 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
24936 let mut schema = crate::openrouter::curated_schemas()
24937 .into_iter()
24938 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
24939 .unwrap();
24940 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
24941 *endpoint = server.uri();
24942 }
24943 let model_id = schema.id.clone();
24944 engine.unified_registry_mut().register_project_model(schema);
24945
24946 let err = engine
24947 .generate_tracked(GenerateRequest {
24948 prompt: "bill me".into(),
24949 model: Some(model_id.clone()),
24950 params: GenerateParams {
24951 strict_model: true,
24952 ..Default::default()
24953 },
24954 ..Default::default()
24955 })
24956 .await
24957 .expect_err("402 must not succeed");
24958
24959 match &err {
24960 InferenceError::ProviderAccount {
24961 provider, status, ..
24962 } => {
24963 assert_eq!(status, &402);
24964 assert_eq!(provider, "openrouter");
24965 }
24966 other => panic!("expected ProviderAccount, got {other:?}"),
24967 }
24968 assert!(
24969 !error_counts_against_circuit_breaker(&err),
24970 "an account rejection must not feed the per-model breaker"
24971 );
24972
24973 let tracker_handle = engine.outcome_tracker();
24975 let tracker = tracker_handle.read().await;
24976 let profile = tracker.profile(&model_id).cloned();
24977 assert!(
24978 profile.as_ref().is_none_or(|p| p.fail_count == 0),
24979 "account rejection degraded the model profile: {profile:?}"
24980 );
24981 drop(tracker);
24982
24983 let ledger = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
24987 let entry = ledger
24988 .iter()
24989 .find(|e| e.model_id == model_id)
24990 .expect("the attempt must still produce a receipt");
24991 assert!(
24992 entry.success.is_none() && entry.quality.is_none(),
24993 "receipt must record the attempt without a verdict: {entry:?}"
24994 );
24995
24996 assert!(
24998 engine
24999 .adaptive_router
25000 .circuit_breakers
25001 .lock()
25002 .unwrap()
25003 .state(&model_id)
25004 .is_none(),
25005 "account rejection must not create breaker state for the model"
25006 );
25007 }
25008
25009 #[tokio::test(flavor = "current_thread")]
25010 async fn openrouter_stream_account_error_is_unattributed_and_never_completes_successfully() {
25011 use wiremock::matchers::{method, path};
25012 use wiremock::{Mock, MockServer, ResponseTemplate};
25013
25014 let _credential_scope = crate::openrouter::test_credential_scope();
25015 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
25016 let server = MockServer::start().await;
25017 Mock::given(method("POST"))
25018 .and(path("/v1/chat/completions"))
25019 .respond_with(ResponseTemplate::new(200).set_body_raw(
25020 "data: {\"error\":{\"code\":402,\"message\":\"private balance details\"}}\n\n",
25021 "text/event-stream",
25022 ))
25023 .mount(&server)
25024 .await;
25025
25026 let tmp = TempDir::new().unwrap();
25027 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25028 let mut schema = crate::openrouter::curated_schemas()
25029 .into_iter()
25030 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
25031 .unwrap();
25032 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
25033 *endpoint = server.uri();
25034 }
25035 let model_id = schema.id.clone();
25036 engine.unified_registry_mut().register_project_model(schema);
25037
25038 let mut handle = engine
25039 .generate_tracked_stream(GenerateRequest {
25040 prompt: "fail after headers".into(),
25041 model: Some(model_id.clone()),
25042 ..Default::default()
25043 })
25044 .await
25045 .unwrap();
25046 let mut events = Vec::new();
25047 while let Some(event) = handle.events.recv().await {
25048 events.push(event);
25049 }
25050 assert!(matches!(
25051 events.as_slice(),
25052 [StreamEvent::Error(message)] if message == "OpenRouter account is out of credits"
25053 ));
25054 assert!(!events
25055 .iter()
25056 .any(|event| matches!(event, StreamEvent::Done { .. })));
25057
25058 let mut receipts = Vec::new();
25061 for _ in 0..50 {
25062 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
25063 if !receipts.is_empty() {
25064 break;
25065 }
25066 tokio::task::yield_now().await;
25067 }
25068 let [receipt] = receipts.as_slice() else {
25069 panic!("expected one receipt, got {receipts:?}");
25070 };
25071 assert_eq!(receipt.success, None);
25072 assert_eq!(
25073 receipt.attribution,
25074 Some(crate::outcome::UnattributedReason::Account)
25075 );
25076 assert!(
25077 engine
25078 .outcome_tracker()
25079 .read()
25080 .await
25081 .profile(&model_id)
25082 .is_none(),
25083 "an account rejection must not touch the model's profile"
25084 );
25085 }
25086
25087 #[tokio::test(flavor = "current_thread")]
25090 async fn tracked_stream_dropped_by_the_caller_is_cancelled_not_a_failure() {
25091 let _offload_guard = crate::offload::test_offload_lock().lock().await;
25092 let tmp = TempDir::new().unwrap();
25093 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25094 pin_test_local_admission_memory(&mut engine);
25095 let model_id = install_small_local_fixture(&engine);
25096 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
25097
25098 let stream = engine
25099 .generate_tracked_stream(GenerateRequest {
25100 prompt: "interrupted".into(),
25101 model: Some(model_id.clone()),
25102 params: GenerateParams {
25103 strict_model: true,
25104 ..Default::default()
25105 },
25106 ..Default::default()
25107 })
25108 .await
25109 .unwrap();
25110 drop(stream);
25111
25112 let mut receipts = Vec::new();
25113 for _ in 0..200 {
25114 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
25115 if !receipts.is_empty() {
25116 break;
25117 }
25118 tokio::task::yield_now().await;
25119 }
25120 crate::offload::set_local_offload(None);
25121 let [receipt] = receipts.as_slice() else {
25122 panic!("expected one receipt, got {receipts:?}");
25123 };
25124 assert_eq!(receipt.success, None);
25125 assert_eq!(
25126 receipt.attribution,
25127 Some(crate::outcome::UnattributedReason::Cancelled)
25128 );
25129 assert!(engine
25130 .outcome_tracker()
25131 .read()
25132 .await
25133 .profile(&model_id)
25134 .is_none_or(|p| p.fail_count == 0));
25135 }
25136
25137 #[test]
25141 fn an_apple_guardrail_decline_does_not_end_the_fallback_chain() {
25142 let refusal = |provider: &str| InferenceError::ContentRefused {
25143 provider: provider.into(),
25144 kind: Some("guardrailViolation".into()),
25145 code: None,
25146 message: "declined".into(),
25147 };
25148 assert!(!error_ends_fallback_chain(&refusal("apple")));
25149 assert!(error_ends_fallback_chain(&refusal("parslee")));
25150 assert!(!error_counts_against_circuit_breaker(&refusal("apple")));
25151 }
25152
25153 #[tokio::test(flavor = "current_thread")]
25154 async fn tracked_stream_without_done_records_failure_not_success() {
25155 let _offload_guard = crate::offload::test_offload_lock().lock().await;
25156 let tmp = TempDir::new().unwrap();
25157 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25158 pin_test_local_admission_memory(&mut engine);
25159 let model_id = install_small_local_fixture(&engine);
25160 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: false })));
25161
25162 let mut stream = engine
25163 .generate_tracked_stream(GenerateRequest {
25164 prompt: "must not count as success".into(),
25165 model: Some(model_id.clone()),
25166 params: GenerateParams {
25167 strict_model: true,
25168 ..Default::default()
25169 },
25170 ..Default::default()
25171 })
25172 .await
25173 .unwrap();
25174 while stream.events.recv().await.is_some() {}
25175 crate::offload::set_local_offload(None);
25176
25177 for _ in 0..50 {
25178 if engine
25179 .outcome_tracker()
25180 .read()
25181 .await
25182 .profile(&model_id)
25183 .is_some_and(|profile| profile.fail_count == 1)
25184 {
25185 break;
25186 }
25187 tokio::task::yield_now().await;
25188 }
25189 let profile = engine
25190 .outcome_tracker()
25191 .read()
25192 .await
25193 .profile(&model_id)
25194 .cloned()
25195 .unwrap();
25196 assert_eq!(profile.fail_count, 1);
25197 assert_eq!(profile.success_count, 0);
25198 }
25199
25200 #[tokio::test(flavor = "current_thread")]
25201 async fn google_vertex_terminal_matrix_records_only_deliberate_finishes_as_success() {
25202 use wiremock::matchers::{method, path};
25203 use wiremock::{Mock, MockServer, ResponseTemplate};
25204
25205 let _provider_env = crate::openrouter::test_environment_scope_async().await;
25206 let _env = ENV_MUTEX.lock().await;
25207 unsafe { std::env::set_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY", "matrix-key") };
25208
25209 for (protocol, reason, should_succeed) in [
25210 (schema::ApiProtocol::Google, "STOP", true),
25211 (schema::ApiProtocol::Google, "SAFETY", false),
25212 (schema::ApiProtocol::VertexAi, "MAX_TOKENS", true),
25213 (
25214 schema::ApiProtocol::VertexAi,
25215 "MALFORMED_FUNCTION_CALL",
25216 false,
25217 ),
25218 ] {
25219 let server = MockServer::start().await;
25220 let expected_path = match protocol {
25221 schema::ApiProtocol::Google => "/v1beta/models/gemini-test:streamGenerateContent",
25222 schema::ApiProtocol::VertexAi => {
25223 "/publishers/google/models/gemini-test:streamGenerateContent"
25224 }
25225 _ => unreachable!(),
25226 };
25227 Mock::given(method("POST"))
25228 .and(path(expected_path))
25229 .respond_with(ResponseTemplate::new(200).set_body_raw(
25230 format!(
25231 "data: {{\"candidates\":[{{\"content\":{{\"parts\":[{{\"text\":\"matrix\"}}]}},\"finishReason\":\"{reason}\"}}]}}\n\n"
25232 ),
25233 "text/event-stream",
25234 ))
25235 .mount(&server)
25236 .await;
25237
25238 let id = format!("test/{protocol:?}-{reason}");
25239 let tmp = TempDir::new().unwrap();
25240 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25241 engine.register_model(remote_stream_fixture_schema(
25242 &id,
25243 server.uri(),
25244 protocol,
25245 "CAR_GOOGLE_OUTCOME_MATRIX_KEY",
25246 ));
25247 let mut stream = engine
25248 .generate_tracked_stream(GenerateRequest {
25249 prompt: "matrix".into(),
25250 model: Some(id.clone()),
25251 params: GenerateParams {
25252 strict_model: true,
25253 ..Default::default()
25254 },
25255 ..Default::default()
25256 })
25257 .await
25258 .unwrap();
25259 let mut events = Vec::new();
25260 while let Some(event) = stream.events.recv().await {
25261 events.push(event);
25262 }
25263 for _ in 0..50 {
25264 if engine
25265 .outcome_tracker()
25266 .read()
25267 .await
25268 .profile(&id)
25269 .is_some_and(|profile| profile.total_calls == 1)
25270 {
25271 break;
25272 }
25273 tokio::task::yield_now().await;
25274 }
25275 let profile = engine
25276 .outcome_tracker()
25277 .read()
25278 .await
25279 .profile(&id)
25280 .cloned()
25281 .unwrap();
25282 assert_eq!(
25283 (profile.success_count, profile.fail_count),
25284 if should_succeed { (1, 0) } else { (0, 1) },
25285 "{protocol:?}/{reason}: {events:?}"
25286 );
25287 }
25288
25289 unsafe { std::env::remove_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY") };
25290 }
25291
25292 #[tokio::test(flavor = "current_thread")]
25293 async fn remote_primary_stream_setup_failure_falls_back_to_installed_local_dispatch() {
25294 use wiremock::matchers::{method, path};
25295 use wiremock::{Mock, MockServer, ResponseTemplate};
25296
25297 let _offload_guard = crate::offload::test_offload_lock().lock().await;
25298 let _env = ENV_MUTEX.lock().await;
25299 let server = MockServer::start().await;
25300 Mock::given(method("POST"))
25301 .and(path("/v1/chat/completions"))
25302 .respond_with(ResponseTemplate::new(503))
25303 .mount(&server)
25304 .await;
25305 unsafe { std::env::set_var("CAR_STREAM_FALLBACK_TEST_KEY", "fixture") };
25306
25307 let tmp = TempDir::new().unwrap();
25308 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25309 pin_test_local_admission_memory(&mut engine);
25310 let _local_id = install_small_local_fixture(&engine);
25311 let remote_id = "test/remote-primary";
25312 engine.register_model(ModelSchema {
25313 id: remote_id.into(),
25314 name: "remote-primary".into(),
25315 provider: "test".into(),
25316 family: "test".into(),
25317 version: "1".into(),
25318 capabilities: vec![ModelCapability::Generate],
25319 context_length: 8_192,
25320 max_output_tokens: Some(1_024),
25321 param_count: String::new(),
25322 quantization: None,
25323 performance: Default::default(),
25324 cost: Default::default(),
25325 source: ModelSource::RemoteApi {
25326 endpoint: server.uri(),
25327 api_key_env: "CAR_STREAM_FALLBACK_TEST_KEY".into(),
25328 api_key_envs: vec![],
25329 api_version: None,
25330 protocol: schema::ApiProtocol::OpenAiCompat,
25331 },
25332 tags: vec![],
25333 supported_params: vec![],
25334 public_benchmarks: vec![],
25335 trust_tier: TrustTier::Community,
25336 deprecated: false,
25337 available: true,
25338 weights_ready: true,
25339 });
25340 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
25341
25342 let mut stream = engine
25343 .generate_tracked_stream(GenerateRequest {
25344 prompt: "fall back".into(),
25345 model: Some(remote_id.into()),
25346 ..Default::default()
25347 })
25348 .await
25349 .expect("compatible installed local model should be dispatched");
25350 assert_ne!(stream.model_used, remote_id);
25351 assert!(
25352 engine
25353 .unified_registry()
25354 .get(&stream.model_used)
25355 .is_some_and(ModelSchema::is_local),
25356 "fallback must stay on a compatible local model: {}",
25357 stream.model_used
25358 );
25359 let mut saw_done = false;
25360 while let Some(event) = stream.events.recv().await {
25361 saw_done |= matches!(event, StreamEvent::Done { .. });
25362 }
25363 assert!(saw_done);
25364
25365 crate::offload::set_local_offload(None);
25366 unsafe { std::env::remove_var("CAR_STREAM_FALLBACK_TEST_KEY") };
25367 }
25368
25369 #[tokio::test(flavor = "current_thread")]
25370 async fn exact_model_id_rejects_display_name_without_dispatch() {
25371 let tmp = TempDir::new().unwrap();
25372 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25373 let mut first = remote_stream_fixture_schema(
25374 "test/ambiguous-a:1",
25375 "http://127.0.0.1:9".into(),
25376 schema::ApiProtocol::OpenAiCompat,
25377 "CAR_AMBIGUOUS_EXACT_KEY",
25378 );
25379 first.name = "Shared Alias".into();
25380 let mut second = first.clone();
25381 second.id = "test/ambiguous-b:1".into();
25382 engine.register_model(first);
25383 engine.register_model(second);
25384 let mut request = GenerateRequest {
25385 prompt: "must not dispatch".into(),
25386 ..Default::default()
25387 };
25388 pin_exact_model_id(&mut request, "Shared Alias".into()).unwrap();
25389
25390 let error = engine
25391 .generate_tracked(request)
25392 .await
25393 .expect_err("an exact-id pin must never resolve a display name");
25394 assert!(matches!(error, InferenceError::ModelNotFound(_)));
25395 }
25396
25397 #[tokio::test(flavor = "current_thread")]
25398 async fn exact_openai_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
25399 let _env = ENV_MUTEX.lock().await;
25400 assert_remote_model_identity_contract(
25401 schema::ApiProtocol::OpenAiCompat,
25402 "openai/newsroom-gpt-5.5-2026-04-23:test",
25403 "newsroom-gpt-5.5-2026-04-23-test",
25404 "CAR_OPENAI_IDENTITY_TEST_KEY",
25405 )
25406 .await;
25407 }
25408
25409 #[tokio::test(flavor = "current_thread")]
25410 async fn exact_anthropic_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
25411 let _env = ENV_MUTEX.lock().await;
25412 assert_remote_model_identity_contract(
25413 schema::ApiProtocol::Anthropic,
25414 "anthropic/newsroom-claude-opus-4-8:test",
25415 "newsroom-claude-opus-4-8-test",
25416 "CAR_ANTHROPIC_IDENTITY_TEST_KEY",
25417 )
25418 .await;
25419 }
25420
25421 async fn book_tool_only_turn(
25426 usage: Option<serde_json::Value>,
25427 api_key_env: &str,
25428 ) -> (u64, u64, u64, u64) {
25429 use wiremock::matchers::{method, path};
25430 use wiremock::{Mock, MockServer, ResponseTemplate};
25431
25432 let server = MockServer::start().await;
25433 let mut response = serde_json::json!({
25434 "choices": [{
25435 "message": {
25436 "content": null,
25437 "tool_calls": [{
25438 "id": "call_1",
25439 "type": "function",
25440 "function": {
25441 "name": "get_weather",
25442 "arguments": "{\"city\":\"Paris\",\"days\":3}"
25443 }
25444 }]
25445 },
25446 "finish_reason": "tool_calls"
25447 }]
25448 });
25449 if let Some(usage) = usage {
25450 response["usage"] = usage;
25451 }
25452 Mock::given(method("POST"))
25453 .and(path("/v1/chat/completions"))
25454 .respond_with(ResponseTemplate::new(200).set_body_json(response))
25455 .mount(&server)
25456 .await;
25457 unsafe { std::env::set_var(api_key_env, "fixture") };
25458
25459 let tmp = TempDir::new().unwrap();
25460 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25461 let model_id = "openai/tool-usage:test";
25462 let mut schema = remote_stream_fixture_schema(
25463 model_id,
25464 server.uri(),
25465 schema::ApiProtocol::OpenAiCompat,
25466 api_key_env,
25467 );
25468 schema.capabilities.push(ModelCapability::ToolUse);
25469 engine.register_model(schema);
25470
25471 let mut request = GenerateRequest {
25472 prompt: "what is the weather in Paris?".into(),
25473 tools: Some(vec![serde_json::json!({
25474 "name": "get_weather",
25475 "description": "forecast",
25476 "parameters": {"type": "object"}
25477 })]),
25478 ..Default::default()
25479 };
25480 pin_exact_model_id(&mut request, model_id.into()).unwrap();
25481 let result = engine.generate_tracked(request).await.unwrap();
25482 assert!(result.text.is_empty(), "control: the turn has no text");
25483 assert_eq!(result.tool_calls.len(), 1, "control: one tool call");
25484
25485 let (profile_output, successes) = {
25486 let tracker = engine.outcome_tracker.read().await;
25487 let profile = tracker.profile(model_id).expect("the call was profiled");
25488 (profile.total_output_tokens, profile.success_count)
25489 };
25490 let keys: Vec<_> = engine
25491 .remote_backend
25492 .key_pool
25493 .all_stats()
25494 .await
25495 .into_values()
25496 .flatten()
25497 .collect();
25498 let key_input = keys.iter().map(|stats| stats.total_input_tokens).sum();
25499 let key_output = keys.iter().map(|stats| stats.total_output_tokens).sum();
25500 unsafe { std::env::remove_var(api_key_env) };
25501 (profile_output, successes, key_input, key_output)
25502 }
25503
25504 #[tokio::test(flavor = "current_thread")]
25510 async fn tool_only_turn_without_provider_usage_books_its_tool_call_as_output() {
25511 let _env = ENV_MUTEX.lock().await;
25512 let (profile_output, successes, _, key_output) =
25513 book_tool_only_turn(None, "CAR_TOOL_USAGE_ESTIMATE_TEST_KEY").await;
25514 assert!(
25515 profile_output > 0,
25516 "the tool call is output: profile booked {profile_output}"
25517 );
25518 assert_eq!(successes, 1, "a tool-only answer is a mechanical success");
25519 assert!(
25520 key_output > 0,
25521 "the key pool books the tool call too, got {key_output}"
25522 );
25523 }
25524
25525 #[tokio::test(flavor = "current_thread")]
25530 async fn tool_only_turn_with_provider_usage_books_the_reported_output() {
25531 let _env = ENV_MUTEX.lock().await;
25532 let usage = serde_json::json!({
25533 "prompt_tokens": 40, "completion_tokens": 17, "total_tokens": 57,
25534 "prompt_tokens_details": {"cached_tokens": 30}
25535 });
25536 let (profile_output, successes, key_input, key_output) =
25537 book_tool_only_turn(Some(usage), "CAR_TOOL_USAGE_REPORTED_TEST_KEY").await;
25538 assert_eq!(profile_output, 17, "the profile books the provider count");
25539 assert_eq!(successes, 1);
25540 assert_eq!(key_output, 17, "the key pool books the provider count");
25541 assert_eq!(
25542 key_input, 40,
25543 "the key pool books the provider's whole input, cached part included, \
25544 in the same unit as its output"
25545 );
25546 }
25547
25548 #[tokio::test(flavor = "current_thread")]
25552 async fn tool_only_stream_without_provider_usage_books_its_tool_call_as_output() {
25553 use wiremock::matchers::{method, path};
25554 use wiremock::{Mock, MockServer, ResponseTemplate};
25555
25556 let _env = ENV_MUTEX.lock().await;
25557 let server = MockServer::start().await;
25558 let body = concat!(
25559 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",",
25560 "\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]}}]}\n\n",
25561 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":",
25562 "{\"arguments\":\"{\\\"city\\\":\\\"Paris\\\",\\\"days\\\":3}\"}}]},",
25563 "\"finish_reason\":\"tool_calls\"}]}\n\n",
25564 "data: [DONE]\n\n",
25565 );
25566 Mock::given(method("POST"))
25567 .and(path("/v1/chat/completions"))
25568 .respond_with(ResponseTemplate::new(200).set_body_raw(body, "text/event-stream"))
25569 .mount(&server)
25570 .await;
25571 let api_key_env = "CAR_TOOL_USAGE_STREAM_TEST_KEY";
25572 unsafe { std::env::set_var(api_key_env, "fixture") };
25573
25574 let tmp = TempDir::new().unwrap();
25575 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25576 let model_id = "openai/tool-usage-stream:test";
25577 let mut schema = remote_stream_fixture_schema(
25578 model_id,
25579 server.uri(),
25580 schema::ApiProtocol::OpenAiCompat,
25581 api_key_env,
25582 );
25583 schema.capabilities.push(ModelCapability::ToolUse);
25584 engine.register_model(schema);
25585
25586 let request = GenerateRequest {
25587 prompt: "what is the weather in Paris?".into(),
25588 tools: Some(vec![serde_json::json!({
25589 "name": "get_weather",
25590 "description": "forecast",
25591 "parameters": {"type": "object"}
25592 })]),
25593 model: Some(model_id.into()),
25594 params: GenerateParams {
25595 strict_model: true,
25596 ..Default::default()
25597 },
25598 ..Default::default()
25599 };
25600 let mut handle = engine.generate_tracked_stream(request).await.unwrap();
25601 let mut accumulator = StreamAccumulator::default();
25602 while let Some(event) = handle.events.recv().await {
25603 accumulator.push(&event);
25604 }
25605 let (text, tool_calls, usage, _) = accumulator.finish_with_usage();
25606 assert!(text.is_empty(), "control: the turn has no text");
25607 assert_eq!(tool_calls.len(), 1, "control: one tool call");
25608 assert!(usage.is_none(), "control: the provider reported no usage");
25609
25610 let mut profile_output = None;
25611 for _ in 0..50 {
25612 profile_output = engine
25613 .outcome_tracker()
25614 .read()
25615 .await
25616 .profile(model_id)
25617 .map(|profile| profile.total_output_tokens);
25618 if profile_output.is_some() {
25619 break;
25620 }
25621 tokio::task::yield_now().await;
25622 }
25623 unsafe { std::env::remove_var(api_key_env) };
25624 let profile_output = profile_output.expect("the stream was profiled");
25625 assert!(
25626 profile_output > 1,
25627 "the tool call is output, not a one-token floor: booked {profile_output}"
25628 );
25629
25630 let keys: Vec<_> = engine
25633 .remote_backend
25634 .key_pool
25635 .all_stats()
25636 .await
25637 .into_values()
25638 .flatten()
25639 .collect();
25640 assert_eq!(keys.len(), 1, "control: one pooled key");
25641 assert_eq!(
25642 keys[0].active_requests, 0,
25643 "a finished stream leaves no request in flight on its key"
25644 );
25645 assert_eq!(
25646 keys[0].successes, 1,
25647 "the finished stream is booked a success"
25648 );
25649 assert!(
25650 keys[0].total_output_tokens > 0,
25651 "the stream's tool call is booked against its key, got {}",
25652 keys[0].total_output_tokens
25653 );
25654 }
25655
25656 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
25661 #[tokio::test(flavor = "current_thread")]
25662 #[ignore = "requires macOS 26.4+ with Apple Intelligence enabled"]
25663 async fn foundation_models_tool_stream_reports_usage_before_done() {
25664 let tmp = TempDir::new().unwrap();
25665 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25666 let mut request = GenerateRequest {
25667 prompt: "What is the weather in Paris? Use the tool.".into(),
25668 model: Some("apple/foundation:default".into()),
25669 tools: Some(vec![serde_json::json!({
25670 "name": "get_weather",
25671 "description": "Get the weather forecast for a city",
25672 "parameters": {
25673 "type": "object",
25674 "properties": {"city": {"type": "string"}},
25675 "required": ["city"]
25676 }
25677 })]),
25678 ..Default::default()
25679 };
25680 request.params.strict_model = true;
25681 request.params.tool_choice = Some("required".into());
25682 let mut handle = engine.generate_tracked_stream(request).await.unwrap();
25683 let mut events = Vec::new();
25684 while let Some(event) = handle.events.recv().await {
25685 events.push(event);
25686 }
25687 let done = events
25688 .iter()
25689 .position(|event| matches!(event, StreamEvent::Done { .. }))
25690 .unwrap_or_else(|| panic!("the stream completes: {events:?}"));
25691 let (usage_at, output) = events
25692 .iter()
25693 .enumerate()
25694 .find_map(|(at, event)| match event {
25695 StreamEvent::Usage { output_tokens, .. } => Some((at, *output_tokens)),
25696 _ => None,
25697 })
25698 .unwrap_or_else(|| panic!("the stream reports usage: {events:?}"));
25699 assert!(usage_at < done, "usage arrives before Done: {events:?}");
25700 assert!(output > 0, "the captured call is output, got {output}");
25701 }
25702
25703 #[tokio::test(flavor = "current_thread")]
25707 async fn refused_stream_returns_its_key_lease() {
25708 use wiremock::matchers::{method, path};
25709 use wiremock::{Mock, MockServer, ResponseTemplate};
25710
25711 let _env = ENV_MUTEX.lock().await;
25712 let server = MockServer::start().await;
25713 Mock::given(method("POST"))
25714 .and(path("/v1/chat/completions"))
25715 .respond_with(ResponseTemplate::new(500).set_body_string("upstream broke"))
25716 .mount(&server)
25717 .await;
25718 let api_key_env = "CAR_REFUSED_STREAM_LEASE_TEST_KEY";
25719 unsafe { std::env::set_var(api_key_env, "fixture") };
25720
25721 let tmp = TempDir::new().unwrap();
25722 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25723 let model_id = "openai/refused-stream:test";
25724 engine.register_model(remote_stream_fixture_schema(
25725 model_id,
25726 server.uri(),
25727 schema::ApiProtocol::OpenAiCompat,
25728 api_key_env,
25729 ));
25730 let request = GenerateRequest {
25731 prompt: "refuse me".into(),
25732 model: Some(model_id.into()),
25733 params: GenerateParams {
25734 strict_model: true,
25735 ..Default::default()
25736 },
25737 ..Default::default()
25738 };
25739 let _ = engine.generate_tracked_stream(request).await;
25740 unsafe { std::env::remove_var(api_key_env) };
25741
25742 let keys: Vec<_> = engine
25743 .remote_backend
25744 .key_pool
25745 .all_stats()
25746 .await
25747 .into_values()
25748 .flatten()
25749 .collect();
25750 assert_eq!(keys.len(), 1, "control: one pooled key");
25751 assert!(keys[0].total_requests >= 1, "control: the key was leased");
25752 assert_eq!(
25753 keys[0].active_requests, 0,
25754 "a refused stream leaves no request in flight on its key"
25755 );
25756 assert_eq!(
25757 keys[0].failures, keys[0].total_requests,
25758 "each refusal is booked"
25759 );
25760 }
25761
25762 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
25763 #[tokio::test(flavor = "current_thread")]
25764 async fn exact_model_id_nonstream_bypasses_mlx_equivalent_substitution() {
25765 let _offload_guard = crate::offload::test_offload_lock().lock().await;
25766 let tmp = TempDir::new().unwrap();
25767 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25768 let exact_id = install_exact_pin_equivalent_fixture(&engine);
25769 let offload = Arc::new(ExactPinCaptureOffload::new());
25770 crate::offload::set_local_offload(Some(offload.clone()));
25771
25772 let mut request = GenerateRequest {
25773 prompt: "use only the immutable row I selected".into(),
25774 ..Default::default()
25775 };
25776 pin_exact_model_id(&mut request, exact_id.clone()).unwrap();
25777 let error = engine.generate_tracked(request).await.unwrap_err();
25778
25779 assert!(offload.dispatched_models().is_empty());
25780 assert!(error.to_string().contains(&exact_id), "{error}");
25781 assert!(
25782 error
25783 .to_string()
25784 .contains("MLX-equivalent substitution is disabled for exact pins"),
25785 "{error}"
25786 );
25787 crate::offload::set_local_offload(None);
25788 }
25789
25790 #[tokio::test(flavor = "current_thread")]
25791 async fn catalog_identity_mismatch_rejects_before_provider_dispatch() {
25792 use wiremock::matchers::{method, path};
25793 use wiremock::{Mock, MockServer, ResponseTemplate};
25794
25795 let _env = ENV_MUTEX.lock().await;
25796 let server = MockServer::start().await;
25797 Mock::given(method("POST"))
25798 .and(path("/v1/chat/completions"))
25799 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
25800 "choices": [{"message": {"content": "must not run"}, "finish_reason": "stop"}],
25801 })))
25802 .mount(&server)
25803 .await;
25804 unsafe { std::env::set_var("CAR_PRECONDITION_TEST_KEY", "fixture") };
25805
25806 let tmp = TempDir::new().unwrap();
25807 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25808 let model_id = "test/catalog-precondition:1";
25809 engine.register_model(remote_stream_fixture_schema(
25810 model_id,
25811 server.uri(),
25812 schema::ApiProtocol::OpenAiCompat,
25813 "CAR_PRECONDITION_TEST_KEY",
25814 ));
25815 let snapshot = engine.catalog_snapshot().unwrap();
25816 let row_digest = snapshot
25817 .model_by_exact_id(model_id)
25818 .unwrap()
25819 .row_digest
25820 .clone();
25821
25822 for (expected_row_digest, expected_catalog_revision) in [
25823 (
25824 Some("0".repeat(64)),
25825 Some(snapshot.catalog_revision.clone()),
25826 ),
25827 (Some(row_digest.clone()), Some("f".repeat(64))),
25828 ] {
25829 let mut request = GenerateRequest {
25830 prompt: "must not dispatch".into(),
25831 expected_row_digest,
25832 expected_catalog_revision,
25833 ..Default::default()
25834 };
25835 pin_exact_model_id(&mut request, model_id.into()).unwrap();
25836 let error = engine.generate_tracked(request).await.unwrap_err();
25837 assert!(matches!(
25838 error,
25839 InferenceError::CatalogPreconditionMismatch { .. }
25840 ));
25841 }
25842 assert!(
25843 server.received_requests().await.unwrap().is_empty(),
25844 "catalog identity mismatches must fail before provider dispatch"
25845 );
25846
25847 unsafe { std::env::remove_var("CAR_PRECONDITION_TEST_KEY") };
25848 }
25849
25850 #[tokio::test(flavor = "current_thread")]
25851 async fn exact_nonstream_identity_survives_thinking_retry() {
25852 use std::sync::atomic::{AtomicUsize, Ordering};
25853 use wiremock::matchers::{method, path};
25854 use wiremock::{Mock, MockServer, ResponseTemplate};
25855
25856 let _env = ENV_MUTEX.lock().await;
25857 let server = MockServer::start().await;
25858 let calls = Arc::new(AtomicUsize::new(0));
25859 let response_calls = calls.clone();
25860 Mock::given(method("POST"))
25861 .and(path("/v1/chat/completions"))
25862 .respond_with(move |_request: &wiremock::Request| {
25863 let attempt = response_calls.fetch_add(1, Ordering::SeqCst);
25864 let content = if attempt == 0 { "" } else { "recovered" };
25865 ResponseTemplate::new(200).set_body_json(serde_json::json!({
25866 "choices": [{
25867 "message": {"content": content},
25868 "finish_reason": if attempt == 0 { "length" } else { "stop" },
25869 }],
25870 "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
25871 }))
25872 })
25873 .mount(&server)
25874 .await;
25875 unsafe { std::env::set_var("CAR_IDENTITY_RETRY_KEY", "fixture") };
25876
25877 let tmp = TempDir::new().unwrap();
25878 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25879 let model_id = "test/identity-retry:1";
25880 engine.register_model(remote_stream_fixture_schema(
25881 model_id,
25882 server.uri(),
25883 schema::ApiProtocol::OpenAiCompat,
25884 "CAR_IDENTITY_RETRY_KEY",
25885 ));
25886 let snapshot = engine.catalog_snapshot().unwrap();
25887 let expected_row = snapshot.model_by_exact_id(model_id).unwrap();
25888 let mut request = GenerateRequest {
25889 prompt: "retry once".into(),
25890 expected_row_digest: Some(expected_row.row_digest.clone()),
25891 expected_catalog_revision: Some(snapshot.catalog_revision.clone()),
25892 ..Default::default()
25893 };
25894 pin_exact_model_id(&mut request, model_id.into()).unwrap();
25895
25896 let result = engine.generate_tracked(request).await.unwrap();
25897 assert_eq!(calls.load(Ordering::SeqCst), 2, "must exercise retry path");
25898 assert_eq!(result.stop_reason.as_deref(), Some("thinking_recovered"));
25899 assert_eq!(
25900 result.model_identity.requested_model_id.as_deref(),
25901 Some(model_id)
25902 );
25903 assert_eq!(result.model_identity.resolved_model_id, model_id);
25904 assert_eq!(result.model_identity.row_digest, expected_row.row_digest);
25905 assert_eq!(
25906 result.model_identity.catalog_revision,
25907 snapshot.catalog_revision
25908 );
25909
25910 unsafe { std::env::remove_var("CAR_IDENTITY_RETRY_KEY") };
25911 }
25912
25913 #[tokio::test(flavor = "current_thread")]
25914 async fn managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns() {
25915 if !crate::run_in_isolated_test_process(
25916 "tests::managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns",
25917 "CAR_MANAGED_REASONING_ROUNDTRIP_CHILD",
25918 ) {
25919 return;
25920 }
25921 let _home = crate::openrouter::StateRootScope::new();
25922 use wiremock::matchers::{header, method, path};
25923 use wiremock::{Mock, MockServer, ResponseTemplate};
25924
25925 let _provider_env = crate::openrouter::test_environment_scope_async().await;
25926 let _env = ENV_MUTEX.lock().await;
25927 let bearer = "managed-reasoning-roundtrip-bearer";
25928
25929 let server = MockServer::start().await;
25930 unsafe {
25931 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
25932 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
25933 }
25934 Mock::given(method("GET"))
25935 .and(path("/api/v1/organizations/me"))
25936 .and(header("authorization", format!("Bearer {bearer}")))
25937 .respond_with(
25938 ResponseTemplate::new(200)
25939 .set_body_json(serde_json::json!({"organizationId": "org-roundtrip"})),
25940 )
25941 .mount(&server)
25942 .await;
25943 Mock::given(method("GET"))
25944 .and(path("/connect/session"))
25945 .respond_with(
25946 ResponseTemplate::new(200)
25947 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
25948 )
25949 .mount(&server)
25950 .await;
25951 Mock::given(method("POST"))
25952 .and(path("/api/v1/orgs/org-roundtrip/inference/responses"))
25953 .respond_with(ResponseTemplate::new(200).set_body_raw(
25954 include_str!("../tests/fixtures/parslee-openrouter-reasoning-roundtrip.sse"),
25955 "text/event-stream",
25956 ))
25957 .expect(2)
25958 .mount(&server)
25959 .await;
25960
25961 let tmp = TempDir::new().unwrap();
25962 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25963 let schema = crate::openrouter::curated_schemas()
25964 .into_iter()
25965 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
25966 .unwrap();
25967 engine.unified_registry_mut().register_project_model(schema);
25968
25969 let first = engine
25970 .generate_tracked(GenerateRequest {
25971 prompt: "first".into(),
25972 model: Some("parslee/openrouter/frontier-general".into()),
25973 ..Default::default()
25974 })
25975 .await
25976 .expect("first managed turn");
25977 let expected_reasoning = serde_json::json!({
25978 "type": "reasoning",
25979 "id": "rs_car_roundtrip",
25980 "status": "completed",
25981 "summary": [{"type": "summary_text", "text": "safe summary"}],
25982 "encrypted_content": "opaque-encrypted-reasoning",
25983 });
25984 assert_eq!(
25985 first.provider_output_items,
25986 vec![expected_reasoning.clone()]
25987 );
25988
25989 let mut history = vec![Message::User {
25990 content: "first".into(),
25991 }];
25992 first.append_assistant_history(&mut history, first.tool_calls.clone());
25993 history.push(Message::User {
25994 content: "continue".into(),
25995 });
25996
25997 let second = engine
25998 .generate_tracked(GenerateRequest {
25999 prompt: String::new(),
26000 model: Some("parslee/openrouter/frontier-general".into()),
26001 messages: Some(history),
26002 ..Default::default()
26003 })
26004 .await
26005 .expect("second managed turn");
26006 assert_eq!(second.text, "first answer");
26007
26008 let requests = server.received_requests().await.unwrap();
26009 let posts: Vec<serde_json::Value> = requests
26010 .iter()
26011 .filter(|request| {
26012 request.method.as_str() == "POST"
26013 && request.url.path() == "/api/v1/orgs/org-roundtrip/inference/responses"
26014 })
26015 .map(|request| serde_json::from_slice(&request.body).unwrap())
26016 .collect();
26017 assert_eq!(posts.len(), 2);
26018 for body in &posts {
26019 assert_eq!(body["store"], false);
26020 assert_eq!(
26021 body["include"],
26022 serde_json::json!(["reasoning.encrypted_content"])
26023 );
26024 }
26025 let second_input = posts[1]["input"].as_array().unwrap();
26026 let reasoning_index = second_input
26027 .iter()
26028 .position(|item| item == &expected_reasoning)
26029 .expect("second request must replay the exact reasoning item");
26030 let assistant_index = second_input
26031 .iter()
26032 .position(|item| item["role"] == "assistant")
26033 .expect("second request assistant turn");
26034 let user_index = second_input
26035 .iter()
26036 .position(|item| item["content"] == "continue")
26037 .expect("second request user turn");
26038 assert!(
26039 reasoning_index < assistant_index && assistant_index < user_index,
26040 "provider output order must be reasoning, assistant text, then the next user turn"
26041 );
26042
26043 unsafe {
26044 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
26045 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
26046 }
26047 }
26048
26049 #[tokio::test(flavor = "current_thread")]
26050 async fn managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures() {
26051 if !crate::run_in_isolated_test_process(
26052 "tests::managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures",
26053 "CAR_MANAGED_PARTIAL_EOF_CHILD",
26054 ) {
26055 return;
26056 }
26057 let _home = crate::openrouter::StateRootScope::new();
26058 use wiremock::matchers::{header, method, path};
26059 use wiremock::{Mock, MockServer, ResponseTemplate};
26060
26061 let _provider_env = crate::openrouter::test_environment_scope_async().await;
26062 let _env = ENV_MUTEX.lock().await;
26063 let bearer = "managed-partial-outcome-bearer";
26064
26065 let server = MockServer::start().await;
26066 unsafe {
26067 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
26068 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
26069 }
26070 Mock::given(method("GET"))
26071 .and(path("/api/v1/organizations/me"))
26072 .and(header("authorization", format!("Bearer {bearer}")))
26073 .respond_with(
26074 ResponseTemplate::new(200)
26075 .set_body_json(serde_json::json!({"organizationId": "org-partial-outcome"})),
26076 )
26077 .mount(&server)
26078 .await;
26079 Mock::given(method("GET"))
26080 .and(path("/connect/session"))
26081 .respond_with(
26082 ResponseTemplate::new(200)
26083 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
26084 )
26085 .mount(&server)
26086 .await;
26087 Mock::given(method("POST"))
26088 .and(path("/api/v1/orgs/org-partial-outcome/inference/responses"))
26089 .respond_with(ResponseTemplate::new(200).set_body_raw(
26090 "event: response.output_text.delta\ndata: {\"delta\":\"partial must fail\"}\n\n",
26091 "text/event-stream",
26092 ))
26093 .expect(2)
26094 .mount(&server)
26095 .await;
26096
26097 let tmp = TempDir::new().unwrap();
26098 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26099 let model_id = "parslee/openrouter/frontier-general";
26100 let schema = crate::openrouter::curated_schemas()
26101 .into_iter()
26102 .find(|schema| schema.id == model_id)
26103 .unwrap();
26104 engine.unified_registry_mut().register_project_model(schema);
26105
26106 let buffered_error = engine
26107 .generate_tracked(GenerateRequest {
26108 prompt: "buffered".into(),
26109 model: Some(model_id.into()),
26110 params: GenerateParams {
26111 strict_model: true,
26112 ..Default::default()
26113 },
26114 ..Default::default()
26115 })
26116 .await
26117 .expect_err("buffered partial EOF must fail");
26118 assert!(
26119 buffered_error.to_string().contains("response.completed"),
26120 "unexpected buffered error: {buffered_error}"
26121 );
26122
26123 let mut stream = engine
26124 .generate_tracked_stream(GenerateRequest {
26125 prompt: "streamed".into(),
26126 model: Some(model_id.into()),
26127 params: GenerateParams {
26128 strict_model: true,
26129 ..Default::default()
26130 },
26131 ..Default::default()
26132 })
26133 .await
26134 .expect("HTTP streaming request starts");
26135 let mut events = Vec::new();
26136 while let Some(event) = stream.events.recv().await {
26137 events.push(event);
26138 }
26139 assert!(
26140 matches!(events.last(), Some(StreamEvent::Error(message)) if message.contains("response.completed"))
26141 );
26142 assert!(!events
26143 .iter()
26144 .any(|event| matches!(event, StreamEvent::Done { .. })));
26145
26146 for _ in 0..50 {
26147 if engine
26148 .outcome_tracker()
26149 .read()
26150 .await
26151 .profile(model_id)
26152 .is_some_and(|profile| profile.fail_count == 2)
26153 {
26154 break;
26155 }
26156 tokio::task::yield_now().await;
26157 }
26158 let profile = engine
26159 .outcome_tracker()
26160 .read()
26161 .await
26162 .profile(model_id)
26163 .cloned()
26164 .unwrap();
26165 assert_eq!(profile.fail_count, 2);
26166 assert_eq!(profile.success_count, 0);
26167
26168 unsafe {
26169 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
26170 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
26171 }
26172 }
26173
26174 #[tokio::test]
26175 async fn tokenize_rejects_known_remote_model_with_unsupported_mode() {
26176 let tmp = TempDir::new().unwrap();
26181 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26182 let remote_id = engine
26183 .list_schemas()
26184 .into_iter()
26185 .find(|s| !s.is_local())
26186 .map(|s| s.id)
26187 .expect("built-in catalog should include at least one remote model schema");
26188
26189 let err = engine
26190 .tokenize(&remote_id, "hello")
26191 .await
26192 .expect_err("remote tokenize must error");
26193 match err {
26194 InferenceError::UnsupportedMode { mode, backend, .. } => {
26195 assert_eq!(mode, "tokenize/detokenize");
26196 assert_eq!(backend, "remote");
26197 }
26198 other => panic!("expected UnsupportedMode, got {other:?}"),
26199 }
26200
26201 let err = engine
26202 .detokenize(&remote_id, &[1, 2, 3])
26203 .await
26204 .expect_err("remote detokenize must error");
26205 assert!(
26206 matches!(err, InferenceError::UnsupportedMode { .. }),
26207 "expected UnsupportedMode, got {err:?}"
26208 );
26209 }
26210
26211 #[test]
26212 fn unsupported_mode_does_not_trip_circuit_breaker() {
26213 let unsupported = InferenceError::UnsupportedMode {
26219 mode: "structured-output-json-schema",
26220 backend: "anthropic",
26221 reason: "not wired under the pinned API version",
26222 };
26223 assert!(!error_counts_against_circuit_breaker(&unsupported));
26224 assert!(error_counts_against_circuit_breaker(
26225 &InferenceError::InferenceFailed("API returned 500".into())
26226 ));
26227 assert!(error_counts_against_circuit_breaker(
26228 &InferenceError::InferenceFailed("API returned 429".into())
26229 ));
26230 }
26231
26232 #[test]
26240 fn a_content_refusal_does_not_trip_the_circuit_breaker() {
26241 let refused = InferenceError::ContentRefused {
26242 provider: "parslee".into(),
26243 kind: Some("invalid_request_error".into()),
26244 code: Some("content_policy_violation".into()),
26245 message: "content refused".into(),
26246 };
26247 assert!(!error_counts_against_circuit_breaker(&refused));
26248
26249 let rendered = refused.to_string();
26252 assert!(rendered.contains("content grounds"), "{rendered}");
26253 assert!(rendered.contains("content_policy_violation"), "{rendered}");
26254
26255 assert!(error_counts_against_circuit_breaker(
26257 &InferenceError::InferenceFailed("managed inference failed".into())
26258 ));
26259 }
26260
26261 #[test]
26271 fn a_content_refusal_ends_the_fallback_chain() {
26272 let refused = InferenceError::ContentRefused {
26273 provider: "parslee".into(),
26274 kind: Some("invalid_request_error".into()),
26275 code: Some("content_policy_violation".into()),
26276 message: "content refused".into(),
26277 };
26278 assert!(error_ends_fallback_chain(&refused));
26279
26280 for still_advances in [
26284 InferenceError::InferenceFailed("managed inference failed".into()),
26285 InferenceError::InferenceFailed("API returned 503".into()),
26286 InferenceError::ModelNotFound("parslee/reasoning".into()),
26287 InferenceError::UnsupportedMode {
26288 mode: "json_schema",
26289 backend: "anthropic",
26290 reason: "structured output is not supported by this protocol",
26291 },
26292 InferenceError::GatewayUnconfigured {
26293 provider: "parslee".into(),
26294 namespace: "parslee/openrouter/".into(),
26295 status: 503,
26296 message: "not configured".into(),
26297 },
26298 ] {
26299 assert!(
26300 !error_ends_fallback_chain(&still_advances),
26301 "must keep advancing the chain: {still_advances}"
26302 );
26303 }
26304 }
26305
26306 #[test]
26315 fn recovery_hints_do_not_rewrite_a_content_refusal() {
26316 let refused = InferenceError::ContentRefused {
26317 provider: "parslee".into(),
26318 kind: Some("invalid_request_error".into()),
26319 code: Some("content_policy_violation".into()),
26320 message: "blocked: 403 forbidden by the content filter".into(),
26322 };
26323 let out = apply_exhaustion_recovery_hint(refused);
26324 assert!(
26325 matches!(out, InferenceError::ContentRefused { .. }),
26326 "{out:?}"
26327 );
26328
26329 let signed_out = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
26331 "no credential for proprietary provider 'parslee'".into(),
26332 ));
26333 assert!(
26334 matches!(signed_out, InferenceError::InferenceFailed(ref m) if m.contains("car auth")),
26335 "{signed_out:?}"
26336 );
26337
26338 let transient = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
26340 "API returned 500".into(),
26341 ));
26342 assert_eq!(transient.to_string(), "inference failed: API returned 500");
26343 }
26344
26345 #[test]
26353 fn unconfigured_gateway_does_not_trip_circuit_breaker() {
26354 let unconfigured = InferenceError::GatewayUnconfigured {
26355 provider: "parslee".into(),
26356 namespace: "parslee/openrouter/".into(),
26357 status: 503,
26358 message: "OpenRouter inference is not configured on this Parslee environment.".into(),
26359 };
26360 assert!(!error_counts_against_circuit_breaker(&unconfigured));
26361 let rendered = unconfigured.to_string();
26365 assert!(rendered.contains("parslee/openrouter/"), "{rendered}");
26366 assert!(rendered.contains("not configured"), "{rendered}");
26367 }
26368
26369 #[test]
26375 fn namespace_drop_spares_siblings_outside_the_prefix() {
26376 let namespace = "parslee/openrouter/";
26377 let mut queue: std::collections::VecDeque<String> = [
26378 "parslee/openrouter/open-fast",
26379 "parslee/reasoning",
26380 "parslee/openrouter/frontier-general",
26381 "parslee/advisor",
26382 "anthropic/claude-opus-4-8:latest",
26383 ]
26384 .into_iter()
26385 .map(String::from)
26386 .collect();
26387
26388 queue.retain(|id| !id.starts_with(namespace));
26389
26390 assert_eq!(
26391 queue.iter().collect::<Vec<_>>(),
26392 vec![
26393 "parslee/reasoning",
26394 "parslee/advisor",
26395 "anthropic/claude-opus-4-8:latest"
26396 ],
26397 "only the unconfigured namespace may be dropped"
26398 );
26399 }
26400
26401 #[test]
26402 fn engine_loads_benchmark_priors_on_startup() {
26403 let _env = ENV_MUTEX.blocking_lock();
26404 let tmp = TempDir::new().unwrap();
26405 let priors_path = tmp.path().join("benchmark_priors.json");
26406 std::fs::write(
26407 &priors_path,
26408 serde_json::json!({
26409 "model_id": "qwen/qwen3-8b:q4_k_m",
26410 "overall_score": 0.88
26411 })
26412 .to_string(),
26413 )
26414 .unwrap();
26415
26416 unsafe {
26417 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
26418 }
26419
26420 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26421 let tracker = engine.outcome_tracker.blocking_read();
26422 let profile = tracker
26423 .profile("qwen/qwen3-8b:q4_k_m")
26424 .expect("benchmark prior should create a profile");
26425 assert!((profile.ema_quality - 0.88).abs() < 0.01);
26426
26427 unsafe {
26428 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
26429 }
26430 }
26431
26432 #[test]
26433 fn benchmark_priors_do_not_override_observed_profiles() {
26434 let _env = ENV_MUTEX.blocking_lock();
26435 let tmp = TempDir::new().unwrap();
26436 let models_dir = tmp.path().join("models");
26437 std::fs::create_dir_all(&models_dir).unwrap();
26438
26439 let observed = vec![ModelProfile {
26440 model_id: "qwen/qwen3-8b:q4_k_m".into(),
26441 total_calls: 12,
26442 success_count: 3,
26443 fail_count: 9,
26444 total_latency_ms: 1200,
26445 total_input_tokens: 0,
26446 total_output_tokens: 0,
26447 total_cache_read_input_tokens: 0,
26448 total_cache_creation_input_tokens: 0,
26449 task_stats: std::collections::HashMap::new(),
26450 ema_quality: 0.21,
26451 prior_sample_size: 0,
26452 quality_observations: 0,
26453 quality_per_1k_tokens: 0.0,
26454 updated_at: 1,
26455 }];
26456 std::fs::write(
26457 models_dir.join("outcome_profiles.json"),
26458 serde_json::to_string(&observed).unwrap(),
26459 )
26460 .unwrap();
26461
26462 let priors_path = tmp.path().join("benchmark_priors.json");
26463 std::fs::write(
26464 &priors_path,
26465 serde_json::json!({
26466 "model_id": "qwen/qwen3-8b:q4_k_m",
26467 "overall_score": 0.95
26468 })
26469 .to_string(),
26470 )
26471 .unwrap();
26472
26473 unsafe {
26474 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
26475 }
26476
26477 let engine = InferenceEngine::new(test_config(models_dir));
26478 let tracker = engine.outcome_tracker.blocking_read();
26479 let profile = tracker
26480 .profile("qwen/qwen3-8b:q4_k_m")
26481 .expect("observed profile should remain present");
26482 assert!((profile.ema_quality - 0.21).abs() < 0.01);
26483 assert_eq!(profile.total_calls, 12);
26484
26485 unsafe {
26486 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
26487 }
26488 }
26489
26490 #[test]
26491 fn speech_runtime_package_spec_defaults_and_overrides() {
26492 let _env = ENV_MUTEX.blocking_lock();
26493 unsafe {
26494 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
26495 }
26496 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.2");
26497
26498 unsafe {
26499 std::env::set_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC", "mlx-audio==0.4.1");
26500 }
26501 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.1");
26502
26503 unsafe {
26504 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
26505 }
26506 }
26507
26508 #[test]
26509 fn speech_runtime_spacy_model_spec_defaults_and_overrides() {
26510 let _env = ENV_MUTEX.blocking_lock();
26511 unsafe {
26512 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
26513 }
26514 assert!(
26515 speech_runtime_spacy_model_spec().starts_with("en-core-web-sm @ https://github.com/")
26516 );
26517
26518 unsafe {
26519 std::env::set_var(
26520 "CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC",
26521 "en-core-web-sm==3.8.0",
26522 );
26523 }
26524 assert_eq!(speech_runtime_spacy_model_spec(), "en-core-web-sm==3.8.0");
26525
26526 unsafe {
26527 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
26528 }
26529 }
26530
26531 #[test]
26532 fn kokoro_runtime_fallback_defaults_on() {
26533 unsafe {
26534 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
26535 }
26536 assert!(kokoro_runtime_fallback_enabled());
26537
26538 unsafe {
26539 std::env::set_var("CAR_SPEECH_KOKORO_FALLBACK", "false");
26540 }
26541 assert!(!kokoro_runtime_fallback_enabled());
26542
26543 unsafe {
26544 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
26545 }
26546 }
26547
26548 #[test]
26549 fn preferred_local_tts_wins_over_builtin_rank() {
26550 let tmp = TempDir::new().unwrap();
26551 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26552 engine.set_speech_policy(SpeechPolicy {
26553 prefer_local: true,
26554 allow_remote_fallback: false,
26555 preferred_local_stt: None,
26556 preferred_local_tts: Some("Kokoro-82M-6bit".into()),
26557 preferred_remote_stt: None,
26558 preferred_remote_tts: None,
26559 });
26560
26561 let schema = engine
26562 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
26563 .expect("preferred local TTS should resolve");
26564 #[cfg(not(target_os = "windows"))]
26571 assert_eq!(schema.name, "Kokoro-82M-6bit");
26572 #[cfg(target_os = "windows")]
26573 assert_eq!(schema.name, "Windows-Speech");
26574 }
26575
26576 #[test]
26577 fn preferred_discovered_vllm_mlx_model_wins_generate_routing() {
26578 let tmp = TempDir::new().unwrap();
26579 let mut config = test_config(tmp.path().join("models"));
26580 config.preferred_generation_model =
26581 Some("vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit".into());
26582 let mut engine = InferenceEngine::new(config);
26583 let schema = crate::vllm_mlx::to_model_schema(
26584 &crate::vllm_mlx::DiscoveredModel {
26585 id: "mlx-community/gemma-3n-E2B-it-lm-4bit".into(),
26586 owned_by: Some("mlx-community".into()),
26587 },
26588 "http://127.0.0.1:8001",
26589 );
26590 engine.register_model(schema);
26591
26592 let rt = tokio::runtime::Runtime::new().unwrap();
26593 let decision = rt.block_on(engine.route_adaptive("say hello in one sentence"));
26594 assert_eq!(
26595 decision.model_id,
26596 "vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit"
26597 );
26598 assert_eq!(decision.strategy, RoutingStrategy::Explicit);
26599 assert_eq!(decision.reason, "preferred generation model override");
26600 }
26601
26602 #[test]
26611 fn tools_request_on_non_tool_model_terminates_not_spins() {
26612 let tmp = TempDir::new().unwrap();
26613 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26614 let schema = crate::vllm_mlx::to_model_schema(
26616 &crate::vllm_mlx::DiscoveredModel {
26617 id: "test-org/embed-only-model".into(),
26618 owned_by: None,
26619 },
26620 "http://127.0.0.1:1", );
26622 let model_id = schema.id.clone();
26623 engine.register_model(schema);
26624
26625 let mut req = GenerateRequest {
26626 prompt: "call a tool".to_string(),
26627 model: Some(model_id),
26628 tools: Some(vec![serde_json::json!({
26629 "name": "noop", "description": "n", "parameters": {"type": "object"}
26630 })]),
26631 ..Default::default()
26632 };
26633 req.params.strict_model = true;
26634
26635 let rt = tokio::runtime::Runtime::new().unwrap();
26636 let out = rt.block_on(async {
26637 tokio::time::timeout(
26638 std::time::Duration::from_secs(10),
26639 engine.generate_tracked(req),
26640 )
26641 .await
26642 });
26643 let completed = out.expect("fallback loop must terminate, not spin");
26646 assert!(completed.is_err());
26647 }
26648
26649 fn fake_ready_speech_runtime(root: &Path) {
26660 managed_venv::seed_ready_venv(root);
26661 let runtime = SpeechRuntime::new(root.to_path_buf());
26662 for program in [&runtime.stt_program, &runtime.tts_program] {
26663 std::fs::create_dir_all(program.parent().expect("program has a parent")).unwrap();
26664 std::fs::write(program, b"").unwrap();
26665 }
26666 }
26667
26668 #[tokio::test]
26695 async fn prepare_speech_runtime_returns_the_root_doctor_reports() {
26696 let _env = ENV_MUTEX.lock().await;
26697 let tmp = TempDir::new().unwrap();
26698 let runtime_root = tmp.path().join("speech-runtime");
26699 fake_ready_speech_runtime(&runtime_root);
26700 unsafe {
26701 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
26702 }
26703
26704 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26705 let root = engine
26706 .prepare_speech_runtime()
26707 .await
26708 .expect("prepare should succeed against a ready runtime");
26709 let health = engine.speech_health();
26710
26711 assert_eq!(
26712 root, health.runtime.root,
26713 "prepare returned a different root than doctor reports"
26714 );
26715 assert!(
26716 root.exists(),
26717 "prepare returned {} but it does not exist",
26718 root.display()
26719 );
26720 assert!(
26721 health.runtime.installed,
26722 "doctor should report a ready runtime as installed"
26723 );
26724 assert_eq!(
26726 engine
26727 .prepare_speech_runtime()
26728 .await
26729 .expect("second prepare should succeed"),
26730 root
26731 );
26732
26733 unsafe {
26734 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
26735 }
26736 }
26737
26738 #[tokio::test]
26752 async fn prepare_speech_runtime_fails_when_bootstrap_fails() {
26753 let _env = ENV_MUTEX.lock().await;
26754 let tmp = TempDir::new().unwrap();
26755 let runtime_root = tmp.path().join("speech-runtime");
26756 assert!(!runtime_root.exists(), "precondition: root absent");
26757 unsafe {
26758 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
26759 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
26760 }
26761
26762 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26763 let err = engine.prepare_speech_runtime().await.err();
26764 let health = engine.speech_health();
26765
26766 assert!(
26767 err.is_some(),
26768 "a failed bootstrap must fail the install, not return a root"
26769 );
26770 assert!(
26771 !health.runtime.installed,
26772 "doctor must not claim a runtime the bootstrap never built"
26773 );
26774
26775 unsafe {
26776 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
26777 std::env::remove_var("CAR_SPEECH_PYTHON");
26778 }
26779 }
26780
26781 #[cfg(target_os = "linux")]
26794 #[tokio::test]
26795 async fn a_runtime_that_cannot_be_built_no_longer_blocks_the_whisper_install() {
26796 let _env = ENV_MUTEX.lock().await;
26797 let tmp = TempDir::new().unwrap();
26798
26799 let cached = tmp
26803 .path()
26804 .join(".tokhn")
26805 .join("whisper")
26806 .join("ggml-large-v3-turbo-q5_0.bin");
26807 std::fs::create_dir_all(cached.parent().unwrap()).unwrap();
26808 std::fs::write(&cached, b"stand-in for the ggml weights").unwrap();
26809
26810 let previous_home = std::env::var_os("HOME");
26811 unsafe {
26812 std::env::set_var("HOME", tmp.path());
26813 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", tmp.path().join("speech-runtime"));
26814 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
26818 }
26819
26820 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26821 let installed = engine.install_curated_speech().await;
26822
26823 let restore = || unsafe {
26824 match &previous_home {
26825 Some(value) => std::env::set_var("HOME", value),
26826 None => std::env::remove_var("HOME"),
26827 }
26828 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
26829 std::env::remove_var("CAR_SPEECH_PYTHON");
26830 };
26831 let installed = match installed {
26832 Ok(installed) => installed,
26833 Err(error) => {
26834 restore();
26835 panic!("a runtime that cannot be built must not abort the install: {error}");
26836 }
26837 };
26838 let runtime_installed = engine.speech_health().runtime.installed;
26839 restore();
26840
26841 assert!(
26842 !runtime_installed,
26843 "precondition: the bootstrap must actually have failed, or this \
26844 test would pass without exercising anything"
26845 );
26846 let whisper = installed
26847 .iter()
26848 .find(|report| report.hf_repo == "ggerganov/whisper.cpp")
26849 .unwrap_or_else(|| {
26850 panic!(
26851 "the cross-platform whisper model must still be installed, got: {installed:?}"
26852 )
26853 });
26854 assert_eq!(whisper.snapshot_path, cached);
26855 assert!(
26856 installed
26857 .iter()
26858 .all(|report| report.hf_repo == "ggerganov/whisper.cpp"),
26859 "MLX weights are Apple-only and must not be pulled here — over a \
26860 gigabyte of them, for models this host can never load; got: {installed:?}"
26861 );
26862 }
26863
26864 #[test]
26872 fn alias_resolves_to_canonical_id_for_outcome_keying() {
26873 let tmp = TempDir::new().unwrap();
26874 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
26875
26876 let cases = [
26878 ("claude-sonnet-4-6", "anthropic/claude-sonnet-4-6:latest"),
26879 ("gpt-5.4", "openai/gpt-5.4:latest"),
26880 ("gemini-2.5-flash", "google/gemini-2.5-flash:latest"),
26881 ];
26882 for (alias, canonical) in cases {
26883 let resolved = engine
26886 .resolve_schema(alias)
26887 .map(|s| s.id)
26888 .unwrap_or_else(|| alias.to_string());
26889 assert_eq!(
26890 resolved, canonical,
26891 "alias `{alias}` must resolve to canonical `{canonical}` for outcome keying, got `{resolved}`"
26892 );
26893 assert_ne!(
26894 resolved, alias,
26895 "alias `{alias}` must NOT be recorded raw — that is the fragmentation bug"
26896 );
26897 }
26898 }
26899
26900 #[test]
26905 fn inference_result_serializes_with_full_shape() {
26906 use crate::tasks::generate::ToolCall;
26907 use std::collections::HashMap;
26908
26909 let mut args = HashMap::new();
26910 args.insert("path".to_string(), serde_json::json!("README.md"));
26911
26912 let result = InferenceResult {
26913 text: String::new(),
26914 bounding_boxes: Vec::new(),
26915 tool_calls: vec![ToolCall {
26916 id: None,
26917 name: "read_file".into(),
26918 arguments: args,
26919 }],
26920 trace_id: "trace-abc".into(),
26921 model_used: "test-model".into(),
26922 model_identity: InferenceModelIdentity {
26923 requested_model_id: Some("test/model:1".into()),
26924 resolved_model_id: "test/model:1".into(),
26925 row_digest: "a".repeat(64),
26926 catalog_revision: "b".repeat(64),
26927 },
26928 latency_ms: 1234,
26929 time_to_first_token_ms: Some(180),
26930 usage: Some(TokenUsage {
26931 prompt_tokens: 100,
26932 completion_tokens: 50,
26933 total_tokens: 150,
26934 context_window: 8192,
26935 ..Default::default()
26936 }),
26937 provider_output_items: Vec::new(),
26938 thinking: Vec::new(),
26939 stop_reason: Some("tool_use".into()),
26940 auth_fallback_from: None,
26941 local_last_resort: false,
26942 fallback_from: Vec::new(),
26943 };
26944
26945 let json = serde_json::to_value(&result).expect("serialize");
26946
26947 assert_eq!(json["stop_reason"].as_str(), Some("tool_use"));
26949
26950 assert_eq!(json["text"].as_str(), Some(""));
26952 assert_eq!(json["trace_id"].as_str(), Some("trace-abc"));
26953 assert_eq!(json["model_used"].as_str(), Some("test-model"));
26954 assert_eq!(json["requested_model_id"].as_str(), Some("test/model:1"));
26955 assert_eq!(json["resolved_model_id"].as_str(), Some("test/model:1"));
26956 assert_eq!(json["row_digest"].as_str().unwrap(), "a".repeat(64));
26957 assert_eq!(json["catalog_revision"].as_str().unwrap(), "b".repeat(64));
26958 assert_eq!(json["latency_ms"].as_u64(), Some(1234));
26959
26960 let tool_calls = json["tool_calls"].as_array().expect("tool_calls array");
26962 assert_eq!(tool_calls.len(), 1);
26963 assert_eq!(tool_calls[0]["name"].as_str(), Some("read_file"));
26964 assert_eq!(
26965 tool_calls[0]["arguments"]["path"].as_str(),
26966 Some("README.md")
26967 );
26968
26969 let usage = &json["usage"];
26971 assert_eq!(usage["prompt_tokens"].as_u64(), Some(100));
26972 assert_eq!(usage["completion_tokens"].as_u64(), Some(50));
26973 assert_eq!(usage["total_tokens"].as_u64(), Some(150));
26974 assert_eq!(usage["context_window"].as_u64(), Some(8192));
26975
26976 assert_eq!(json["time_to_first_token_ms"].as_u64(), Some(180));
26978 }
26979
26980 #[test]
26986 fn inference_result_top_level_keys_are_locked() {
26987 use std::collections::BTreeSet;
26988
26989 let result = InferenceResult {
26990 text: "anything".into(),
26991 bounding_boxes: Vec::new(),
26992 tool_calls: vec![],
26993 trace_id: "t".into(),
26994 model_used: "m".into(),
26995 model_identity: InferenceModelIdentity::default(),
26996 latency_ms: 0,
26997 time_to_first_token_ms: None,
26998 usage: None,
26999 provider_output_items: Vec::new(),
27000 thinking: Vec::new(),
27001 stop_reason: None,
27002 auth_fallback_from: None,
27003 local_last_resort: false,
27004 fallback_from: Vec::new(),
27005 };
27006
27007 let json = serde_json::to_value(&result).expect("serialize");
27008 let keys: BTreeSet<&str> = json
27009 .as_object()
27010 .expect("top-level object")
27011 .keys()
27012 .map(String::as_str)
27013 .collect();
27014
27015 let expected: BTreeSet<&str> = [
27016 "text",
27017 "tool_calls",
27018 "trace_id",
27019 "model_used",
27020 "requested_model_id",
27021 "resolved_model_id",
27022 "row_digest",
27023 "catalog_revision",
27024 "latency_ms",
27025 "time_to_first_token_ms",
27026 "usage",
27027 "stop_reason",
27028 ]
27029 .into_iter()
27030 .collect();
27031
27032 assert_eq!(
27033 keys, expected,
27034 "infer response top-level keys drifted -- update both the test \
27035 and the WebSocket protocol documentation if this is intentional"
27036 );
27037
27038 for key in &keys {
27040 assert!(
27041 !key.chars().any(|c| c.is_uppercase()) && !key.contains('-'),
27042 "key '{}' is not snake_case",
27043 key
27044 );
27045 }
27046 }
27047
27048 #[test]
27052 fn inference_result_serializes_plain_text_response() {
27053 let result = InferenceResult {
27054 text: "hello world".into(),
27055 bounding_boxes: Vec::new(),
27056 tool_calls: vec![],
27057 trace_id: "trace-xyz".into(),
27058 model_used: "test-model".into(),
27059 model_identity: InferenceModelIdentity::default(),
27060 latency_ms: 42,
27061 time_to_first_token_ms: None,
27062 usage: None,
27063 provider_output_items: Vec::new(),
27064 thinking: Vec::new(),
27065 stop_reason: None,
27066 auth_fallback_from: None,
27067 local_last_resort: false,
27068 fallback_from: Vec::new(),
27069 };
27070
27071 let json = serde_json::to_value(&result).expect("serialize");
27072 assert_eq!(json["text"], "hello world");
27073 assert!(json["stop_reason"].is_null());
27075 assert!(json["tool_calls"].is_array());
27076 assert_eq!(json["tool_calls"].as_array().unwrap().len(), 0);
27077 assert_eq!(json["model_used"], "test-model");
27078 assert!(json["usage"].is_null());
27079 assert!(json["time_to_first_token_ms"].is_null());
27082 }
27083
27084 #[test]
27085 fn append_assistant_history_preserves_responses_items_in_provider_order() {
27086 let reasoning = serde_json::json!({
27087 "type": "reasoning",
27088 "id": "rs_history",
27089 "status": "completed",
27090 "summary": [{"type": "summary_text", "text": "safe"}],
27091 "encrypted_content": "opaque",
27092 });
27093 let result: InferenceResult = serde_json::from_value(serde_json::json!({
27094 "text": "calling",
27095 "tool_calls": [{
27096 "id": "call_1",
27097 "name": "read_file",
27098 "arguments": {"path": "README.md"}
27099 }],
27100 "trace_id": "trace",
27101 "model_used": "gateway-alias",
27102 "resolved_model_id": "openrouter/anthropic/claude-sonnet-4.5",
27103 "local_last_resort": true,
27104 "latency_ms": 1,
27105 "provider_output_items": [reasoning.clone()],
27106 }))
27107 .unwrap();
27108 let mut history = vec![crate::tasks::generate::Message::User {
27109 content: "inspect".into(),
27110 }];
27111
27112 result.append_assistant_history(&mut history, result.tool_calls.clone());
27113
27114 assert!(matches!(
27115 &history[1],
27116 crate::tasks::generate::Message::ProviderOutputItems { protocol, items }
27117 if protocol == crate::protocol::OPENAI_RESPONSES_PROTOCOL
27118 && items == &vec![reasoning]
27119 ));
27120 assert!(matches!(
27121 &history[2],
27122 crate::tasks::generate::Message::Assistant {
27123 content,
27124 tool_calls,
27125 model_id,
27126 local_last_resort,
27127 ..
27128 } if content == "calling"
27129 && tool_calls[0].id.as_deref() == Some("call_1")
27130 && model_id.as_deref() == Some("openrouter/anthropic/claude-sonnet-4.5")
27131 && *local_last_resort
27132 ));
27133 }
27134
27135 #[test]
27136 fn append_assistant_history_leaves_personal_chat_history_unchanged() {
27137 let result: InferenceResult = serde_json::from_value(serde_json::json!({
27138 "text": "plain",
27139 "tool_calls": [],
27140 "trace_id": "trace",
27141 "model_used": "openrouter/openai/gpt-4.1-mini",
27142 "latency_ms": 1,
27143 }))
27144 .unwrap();
27145 let mut history = Vec::new();
27146
27147 result.append_assistant_history(&mut history, Vec::new());
27148
27149 assert_eq!(history.len(), 1);
27150 assert!(matches!(
27151 &history[0],
27152 crate::tasks::generate::Message::Assistant { content, .. } if content == "plain"
27153 ));
27154 }
27155
27156 #[test]
27168 fn generate_request_deserializes_intent_field_from_json_rpc_params() {
27169 use crate::intent::TaskHint;
27170 use crate::schema::ModelCapability;
27171
27172 let params = serde_json::json!({
27175 "prompt": "summarize this email",
27176 "intent": {
27177 "task": "chat",
27178 "prefer_local": true,
27179 "require": ["tool_use"],
27180 },
27181 });
27182
27183 let req: GenerateRequest =
27184 serde_json::from_value(params).expect("GenerateRequest deserialize");
27185
27186 let intent = req.intent.as_ref().expect("intent field deserialized");
27187 assert_eq!(intent.task, Some(TaskHint::Chat));
27188 assert!(intent.prefer_local);
27189 assert_eq!(intent.require, vec![ModelCapability::ToolUse]);
27190
27191 let back: serde_json::Value =
27195 serde_json::to_value(&req).expect("re-serialize GenerateRequest");
27196 assert_eq!(back["intent"]["task"], "chat");
27197 assert_eq!(back["intent"]["prefer_local"], true);
27198 assert_eq!(back["intent"]["require"][0], "tool_use");
27199
27200 let default_req: GenerateRequest = serde_json::from_value(serde_json::json!({
27205 "prompt": "x",
27206 "intent": {},
27207 }))
27208 .unwrap();
27209 let default_intent = default_req.intent.expect("present but empty");
27210 assert_eq!(default_intent.task, None);
27211 assert!(!default_intent.prefer_local);
27212 assert!(default_intent.require.is_empty());
27213
27214 let no_intent: GenerateRequest =
27217 serde_json::from_value(serde_json::json!({"prompt": "x"})).unwrap();
27218 assert!(no_intent.intent.is_none());
27219 }
27220
27221 #[test]
27222 fn rerank_prompt_matches_upstream_template_shape() {
27223 let p = rerank_prompt(
27224 "retrieve relevant passages",
27225 "who runs the treasury?",
27226 "doc x",
27227 );
27228 assert!(p.contains("<|im_start|>system"));
27229 assert!(p.contains("Note that the answer can only be \"yes\" or \"no\"."));
27230 assert!(p.contains("<|im_start|>user\n<Instruct>: retrieve relevant passages"));
27231 assert!(p.contains("<Query>: who runs the treasury?"));
27232 assert!(p.contains("<Document>: doc x<|im_end|>"));
27233 assert!(p.contains("<|im_start|>assistant\n<think>\n\n</think>\n\n"));
27234 }
27235
27236 #[test]
27237 fn rerank_score_yes_and_no_exactly() {
27238 assert_eq!(score_from_rerank_output("yes", "m"), 1.0);
27239 assert_eq!(score_from_rerank_output("no", "m"), 0.0);
27240 }
27241
27242 #[test]
27243 fn rerank_score_handles_case_leading_space_and_chat_sentinels() {
27244 assert_eq!(score_from_rerank_output(" Yes", "m"), 1.0);
27247 assert_eq!(score_from_rerank_output("\nno.", "m"), 0.0);
27248 assert_eq!(score_from_rerank_output("<|im_end|>yes", "m"), 1.0);
27249 }
27250
27251 #[test]
27252 fn rerank_score_scans_up_to_three_tokens() {
27253 assert_eq!(score_from_rerank_output("_bos_ yes", "m"), 1.0);
27256 }
27257
27258 #[test]
27259 fn rerank_score_unexpected_is_neutral() {
27260 assert_eq!(score_from_rerank_output("maybe", "m"), 0.5);
27263 assert_eq!(score_from_rerank_output("", "m"), 0.5);
27264 }
27265
27266 #[tokio::test]
27267 async fn pull_reuses_a_valid_directory_receipt_and_reports_removability() {
27268 let root = tempfile::tempdir().unwrap();
27269 let models_dir = root.path().join("models");
27270 std::fs::create_dir_all(&models_dir).unwrap();
27271 let engine = InferenceEngine::new(InferenceConfig {
27272 state_root: root.path().join("state"),
27273 models_dir: models_dir.clone(),
27274 ..InferenceConfig::default()
27275 });
27276 let schema = engine
27277 .unified_registry()
27278 .all()
27279 .find(|schema| matches!(schema.source, ModelSource::Local { .. }))
27280 .unwrap()
27281 .clone();
27282 let managed = models_dir.join(&schema.name);
27283 std::fs::create_dir_all(&managed).unwrap();
27284 std::fs::write(managed.join("model.gguf"), b"owned").unwrap();
27285 std::fs::write(managed.join("tokenizer.json"), b"{}").unwrap();
27286 engine
27287 .model_management
27288 .record_managed_artifact(
27289 &schema.id,
27290 model_source_identity(&schema),
27291 None,
27292 1,
27293 false,
27294 managed.clone(),
27295 )
27296 .unwrap();
27297 let row = engine
27298 .list_models_unified()
27299 .into_iter()
27300 .find(|row| row.id == schema.id)
27301 .unwrap();
27302 if model_management::directory_removal_supported() {
27303 assert!(engine.model_management.can_remove(&schema.id).unwrap());
27304 assert!(row.can_remove);
27305 assert_eq!(row.management_evidence.as_deref(), Some("install_receipt"));
27306 } else {
27307 assert!(!engine.model_management.can_remove(&schema.id).unwrap());
27308 assert!(!row.can_remove);
27309 assert_eq!(
27310 row.management_evidence.as_deref(),
27311 Some("install_receipt_directory_cleanup_unsupported")
27312 );
27313 }
27314
27315 let reused = engine.pull_model(&schema.id).await.unwrap();
27316 assert_eq!(reused, managed);
27317 }
27318
27319 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27323 fn foundation_models_request() -> GenerateRequest {
27324 GenerateRequest {
27325 prompt: "What is in this picture?".into(),
27326 model: Some("apple/foundation:default".into()),
27327 params: GenerateParams {
27328 strict_model: true,
27329 ..Default::default()
27330 },
27331 ..Default::default()
27332 }
27333 }
27334
27335 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27336 fn inline_png() -> ContentBlock {
27337 ContentBlock::ImageBase64 {
27338 data: "iVBORw0KGgo=".into(),
27339 media_type: "image/png".into(),
27340 }
27341 }
27342
27343 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27344 fn url_image() -> ContentBlock {
27345 ContentBlock::ImageUrl {
27346 url: "https://example.com/second.png".into(),
27347 detail: "auto".into(),
27348 }
27349 }
27350
27351 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27355 fn assert_foundation_models_refused(
27356 err: InferenceError,
27357 expected_mode: &str,
27358 reason_has: &str,
27359 ) {
27360 match err {
27361 InferenceError::UnsupportedMode {
27362 mode,
27363 backend,
27364 reason,
27365 } => {
27366 assert_eq!(backend, "foundation-models");
27367 assert_eq!(mode, expected_mode);
27368 assert!(
27369 reason.contains(reason_has),
27370 "expected reason containing {reason_has:?}, got {reason:?}"
27371 );
27372 }
27373 other => panic!("expected FoundationModels UnsupportedMode, got {other:?}"),
27374 }
27375 }
27376
27377 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27378 fn message_with_image() -> Vec<Message> {
27379 vec![Message::UserMultimodal {
27380 content: vec![
27381 ContentBlock::Text {
27382 text: "What is in this picture?".into(),
27383 },
27384 inline_png(),
27385 ],
27386 }]
27387 }
27388
27389 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27390 fn lookup_tool() -> serde_json::Value {
27391 serde_json::json!({
27392 "name": "lookup",
27393 "description": "Look something up",
27394 "parameters": {"type": "object", "properties": {}},
27395 })
27396 }
27397
27398 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27402 #[tokio::test]
27403 async fn foundation_models_refuses_images_inside_messages() {
27404 let tmp = TempDir::new().unwrap();
27405 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27406 let req = GenerateRequest {
27407 messages: Some(message_with_image()),
27408 ..foundation_models_request()
27409 };
27410 let err = engine
27411 .generate_tracked(req)
27412 .await
27413 .expect_err("a message-borne image must not be dropped");
27414 assert_foundation_models_refused(
27415 err,
27416 "multimodal-content",
27417 "images only from the request's `images` field",
27418 );
27419 }
27420
27421 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27423 #[tokio::test]
27424 async fn foundation_models_stream_refuses_images_inside_messages() {
27425 let tmp = TempDir::new().unwrap();
27426 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27427 let req = GenerateRequest {
27428 messages: Some(message_with_image()),
27429 ..foundation_models_request()
27430 };
27431 let Err(err) = engine.generate_tracked_stream(req).await else {
27432 panic!("a message-borne image must not be dropped from a stream");
27433 };
27434 assert_foundation_models_refused(err, "multimodal-content", "stream path takes text only");
27435 }
27436
27437 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27442 #[tokio::test]
27443 async fn foundation_models_refuses_images_with_tools() {
27444 let tmp = TempDir::new().unwrap();
27445 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27446 let req = GenerateRequest {
27447 images: Some(vec![inline_png()]),
27448 tools: Some(vec![lookup_tool()]),
27449 ..foundation_models_request()
27450 };
27451 let err = engine
27452 .generate_tracked(req)
27453 .await
27454 .expect_err("an image request must not lose its tools");
27455 assert_foundation_models_refused(err, "multimodal-content", "image path carries no tools");
27456
27457 let req = GenerateRequest {
27458 images: Some(vec![inline_png()]),
27459 tools: Some(vec![lookup_tool()]),
27460 ..foundation_models_request()
27461 };
27462 let Err(err) = engine.generate_tracked_stream(req).await else {
27463 panic!("a streamed image request must not lose its tools");
27464 };
27465 assert_foundation_models_refused(err, "multimodal-content", "stream path takes text only");
27466 }
27467
27468 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27472 #[tokio::test]
27473 async fn foundation_models_refuses_images_with_json_schema() {
27474 let tmp = TempDir::new().unwrap();
27475 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27476 let req = GenerateRequest {
27477 images: Some(vec![inline_png()]),
27478 response_format: Some(crate::tasks::generate::ResponseFormat::JsonSchema {
27479 name: Some("caption".into()),
27480 schema: serde_json::json!({
27481 "type": "object",
27482 "properties": {"caption": {"type": "string"}},
27483 "required": ["caption"],
27484 }),
27485 strict: true,
27486 }),
27487 ..foundation_models_request()
27488 };
27489 let err = engine
27490 .generate_tracked(req)
27491 .await
27492 .expect_err("an image request must not lose its JsonSchema");
27493 assert_foundation_models_refused(
27494 err,
27495 "structured-output",
27496 "no schema-constrained decoding",
27497 );
27498 }
27499
27500 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27505 #[tokio::test]
27506 async fn foundation_models_refuses_url_image_beside_inline_image() {
27507 let tmp = TempDir::new().unwrap();
27508 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27509 let req = GenerateRequest {
27510 images: Some(vec![inline_png(), url_image()]),
27511 ..foundation_models_request()
27512 };
27513 let err = engine
27514 .generate_tracked(req)
27515 .await
27516 .expect_err("a URL image must not be dropped beside an inline one");
27517 assert_foundation_models_refused(
27518 err,
27519 "multimodal-content",
27520 "image URLs must be fetched by the caller first",
27521 );
27522
27523 let req = GenerateRequest {
27524 images: Some(vec![inline_png(), url_image()]),
27525 ..foundation_models_request()
27526 };
27527 let Err(err) = engine.generate_tracked_stream(req).await else {
27528 panic!("a streamed URL image must not be dropped beside an inline one");
27529 };
27530 assert_foundation_models_refused(err, "multimodal-content", "stream path takes text only");
27531 }
27532
27533 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27540 #[tokio::test]
27541 async fn foundation_models_image_refusal_resolves_through_the_candidate_loop() {
27542 let tmp = TempDir::new().unwrap();
27543 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27544 let mut req = GenerateRequest {
27545 images: Some(vec![inline_png(), url_image()]),
27546 ..foundation_models_request()
27547 };
27548 req.params.strict_model = false;
27549 let err = engine
27550 .generate_tracked(req)
27551 .await
27552 .expect_err("the only candidate refuses");
27553 assert_foundation_models_refused(
27554 err,
27555 "multimodal-content",
27556 "image URLs must be fetched by the caller first",
27557 );
27558 let pending = engine.outcome_tracker().read().await.pending_trace_ids();
27559 assert!(
27560 pending.is_empty(),
27561 "the refused attempt must be resolved by the candidate loop, still pending: {pending:?}"
27562 );
27563 }
27564
27565 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27568 #[test]
27569 fn foundation_models_inline_images_names_the_block_it_cannot_carry() {
27570 let text = ContentBlock::Text {
27571 text: "caption".into(),
27572 };
27573 let err = foundation_models_inline_images(&[inline_png(), text]).unwrap_err();
27574 assert_foundation_models_refused(
27575 err,
27576 "multimodal-content",
27577 "only image blocks in `images`",
27578 );
27579 let ok = foundation_models_inline_images(&[inline_png(), inline_png()]).unwrap();
27580 assert_eq!(
27581 ok,
27582 vec!["iVBORw0KGgo=".to_string(), "iVBORw0KGgo=".to_string()]
27583 );
27584 }
27585
27586 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27590 #[test]
27591 fn foundation_models_instructions_inject_json_object_only() {
27592 use crate::tasks::generate::ResponseFormat;
27593 assert_eq!(
27594 foundation_models_instructions(Some("Be brief."), Some(&ResponseFormat::JsonObject))
27595 .as_deref(),
27596 Some("Be brief.\n\nRespond with a single valid JSON object and nothing else.")
27597 );
27598 assert_eq!(
27599 foundation_models_instructions(Some("Be brief."), None).as_deref(),
27600 Some("Be brief.")
27601 );
27602 assert_eq!(foundation_models_instructions(None, None), None);
27603 }
27604
27605 #[test]
27609 fn vision_stream_intent_excludes_foundation_models_only_for_vision() {
27610 let tmp = TempDir::new().unwrap();
27611 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27612 let registry = engine.unified_registry();
27613 let fm_ids: Vec<String> = registry
27614 .all()
27615 .filter(|schema| schema.is_foundation_models())
27616 .map(|schema| schema.id.clone())
27617 .collect();
27618 assert!(
27619 !fm_ids.is_empty(),
27620 "the built-in catalog carries the system model"
27621 );
27622 let caller = crate::intent::IntentHint {
27623 exclude_models: vec!["openai/gpt-5.6".into()],
27624 ..Default::default()
27625 };
27626 let hint = InferenceEngine::vision_stream_intent(Some(&caller), true, ®istry)
27627 .expect("a vision stream gets an intent");
27628 assert!(hint.exclude_models.contains(&"openai/gpt-5.6".to_string()));
27629 for id in &fm_ids {
27630 assert!(hint.exclude_models.contains(id), "{id} must be excluded");
27631 }
27632 assert!(InferenceEngine::vision_stream_intent(Some(&caller), false, ®istry).is_none());
27633 }
27634
27635 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
27639 #[tokio::test]
27640 #[ignore = "requires Apple FoundationModels (Apple Intelligence enabled, macOS 26+)"]
27641 async fn foundation_models_stream_honors_json_schema() {
27642 let tmp = TempDir::new().unwrap();
27643 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
27644 let req = GenerateRequest {
27645 prompt: "Name one primary color.".into(),
27646 response_format: Some(crate::tasks::generate::ResponseFormat::JsonSchema {
27647 name: Some("color".into()),
27648 schema: serde_json::json!({
27649 "type": "object",
27650 "properties": {"color": {"type": "string"}},
27651 "required": ["color"],
27652 }),
27653 strict: true,
27654 }),
27655 ..foundation_models_request()
27656 };
27657 let mut stream = engine
27658 .generate_tracked_stream(req)
27659 .await
27660 .expect("the system model serves a schema-constrained stream");
27661 let mut done_text = None;
27662 while let Some(event) = stream.events.recv().await {
27663 match event {
27664 StreamEvent::Done { text, .. } => done_text = Some(text),
27665 StreamEvent::Error(message) => panic!("stream failed: {message}"),
27666 _ => {}
27667 }
27668 }
27669 let text = done_text.expect("the stream completes");
27670 let value: serde_json::Value =
27671 serde_json::from_str(&text).unwrap_or_else(|e| panic!("not JSON ({e}): {text:?}"));
27672 assert!(value["color"].is_string(), "schema not honored: {text}");
27673 }
27674}
27675
27676#[cfg(test)]
27677mod response_format_support_tests {
27678 use super::*;
27679
27680 #[test]
27684 fn rejection_reason_tracks_the_protocol_handler() {
27685 let root = tempfile::tempdir().unwrap();
27688 let engine = InferenceEngine::new(InferenceConfig {
27689 state_root: root.path().join("state"),
27690 models_dir: root.path().join("models"),
27691 ..InferenceConfig::default()
27692 });
27693 let rf = crate::tasks::generate::ResponseFormat::JsonObject;
27694 let models = engine.list_models_unified();
27695 let anthropic = models
27696 .iter()
27697 .find(|m| m.provider.eq_ignore_ascii_case("anthropic"));
27698 if let Some(m) = anthropic {
27699 let reason = engine
27700 .response_format_rejection_reason(&m.id, &rf)
27701 .expect("the Anthropic protocol rejects response_format");
27702 assert!(reason.contains("protocol rejects"), "{reason}");
27703 }
27704 let openrouter = models
27705 .iter()
27706 .find(|m| m.provider.eq_ignore_ascii_case("openrouter"));
27707 if let Some(m) = openrouter {
27708 assert_eq!(
27709 engine.response_format_rejection_reason(&m.id, &rf),
27710 None,
27711 "OpenRouter forwards the format upstream"
27712 );
27713 }
27714 assert!(
27715 anthropic.is_some() || openrouter.is_some(),
27716 "the builtin catalog should list at least one of the two providers this pins"
27717 );
27718 assert_eq!(
27719 engine.response_format_rejection_reason("no/such-model", &rf),
27720 None,
27721 "an unknown model is not a rejection"
27722 );
27723 }
27724}