1pub mod action_ledger;
26pub mod adaptive_router;
27pub mod aws_sigv4;
28pub mod backend;
29pub mod backend_cache;
30pub mod billing;
31pub mod calibration;
32pub mod catalog;
33pub mod catalog_identity;
34pub mod concierge;
35pub mod concierge_answer;
36pub mod discovery;
37pub mod doctor;
38pub mod download;
39pub mod handle;
40pub mod hardware;
41pub mod hf_cache;
42pub mod hf_schema;
43pub mod intent;
44pub mod key_pool;
45pub mod lane_defaults;
46pub mod maintenance;
47pub mod managed_venv;
48pub mod media_tokens;
49pub mod model_management;
50pub mod models;
51pub mod nudge;
52pub mod offload;
53pub mod openrouter;
54pub mod outcome;
55pub mod parslee_credential;
56pub mod portfolio;
57pub mod protocol;
58pub mod recommend;
59pub mod registry;
60pub mod remote;
61pub mod resource_policy;
62pub mod retire;
63pub mod router;
64pub mod routing_ext;
65pub mod runner;
66pub mod schema;
67pub mod scoreboard;
68pub mod search;
69pub mod service;
70pub mod stream;
71pub mod tasks;
72pub(crate) mod tls_client;
76pub mod uninstall;
77pub mod update_prefs;
78pub mod upgrade;
79pub mod usage_profile;
80pub mod vllm_mlx;
81pub mod vllm_pool;
82pub mod vllm_runtime;
83
84use std::path::{Path, PathBuf};
85use std::sync::Arc;
86use std::time::Instant;
87use std::time::{SystemTime, UNIX_EPOCH};
88
89use reqwest::multipart::{Form, Part};
90use serde::Serialize;
91use thiserror::Error;
92use tokio::io::AsyncReadExt;
93use tokio::process::Command;
94use tokio::sync::Mutex;
95use tokio::sync::RwLock;
96use tracing::{debug, instrument};
97
98pub use action_ledger::{ConciergeActionEntry, ConciergeActionKind};
100pub use adaptive_router::{
101 AdaptiveRouter, AdaptiveRoutingDecision, RoutingConfig, RoutingStrategy,
102};
103pub use catalog_identity::{CatalogModelRow, CatalogSnapshot};
104pub use concierge::{
105 decide_concierge, evaluate_concierge, ConciergeDecision, ConciergeMode, ConciergeStatus,
106 ConciergeSuggestion, DismissReason, DismissalRecord, ModelHealth,
107 DEFAULT_CONCIERGE_THROTTLE_SECS, DEFAULT_WATCHED_USE_CASES,
108};
109pub use download::{DownloadEvent, DownloadProgress, ProgressSink};
110pub use handle::InferenceHandle;
111pub use intent::{IntentHint, Privacy, QualityTier, TaskHint, TierWeights, UseCase, UseCaseRole};
112pub use key_pool::{KeyPool, KeyStats};
113pub use lane_defaults::{LaneDefault, LaneDefaults};
114pub use nudge::{NudgeDecision, NudgeState, UpgradeNudge};
115pub use outcome::{
116 prune_ledger, read_ledger, CodeOutcome, InferenceOutcome, InferenceTask, InferredOutcome,
117 ModelProfile, OutcomeLedgerEntry, OutcomeTracker,
118};
119pub use recommend::{
120 model_fit, platform_compatible, recommend, recommend_with_policy, FitStatus, ModelFit,
121 ModelFitStatus, Recommendation, RecommendationSet,
122};
123pub use resource_policy::{
124 estimate_model_memory, estimate_model_memory_with_measured_weights, evaluate_resources,
125 AcceleratorResourceBudget, EffectiveResourceBudget, FileResourcePolicyRepository,
126 LocalAdmissionCoordinator, LocalLoadPreflight, LocalLoadVerdict, ModelMemoryEstimate,
127 ModelResourceEvidence, ResourceEvaluation, ResourcePolicy, ResourcePolicyError,
128 ResourcePolicyLoadEvidence, ResourcePolicyLoadSource, ResourcePolicyRepository,
129 ResourceProfile, RECOMMENDATION_CONTEXT_TOKENS,
130};
131pub use update_prefs::{UpdateChannel, UpdatePolicy, UpdatePreferences};
132pub use upgrade::{HuggingFaceProbe, UpgradeFinding, UpgradeSource, UpstreamProbe};
133pub use usage_profile::{use_case_for_task, LaneUsage, UsageProfile};
134
135fn now_unix() -> u64 {
137 std::time::SystemTime::now()
138 .duration_since(std::time::UNIX_EPOCH)
139 .map(|d| d.as_secs())
140 .unwrap_or(0)
141}
142pub use offload::{
143 clear_remote_deadline, current_controlled_termination_token, current_inference_control_id,
144 current_local_offload, current_remote_deadline, ensure_not_controlled_terminated,
145 is_offload_worker, scope_inference_control_id, set_local_offload, set_remote_deadline,
146 ControlledTerminationToken, InferenceTerminationAck, LocalGenerationOffload,
147 LocalOffloadResult, LocalOffloadStream, LocalWorkerAdmission, LocalWorkerResidency,
148 RemoteDeadline,
149};
150pub use registry::{
151 is_builtin_subscription_billed, model_unavailable_reason, ModelFilter, ModelInfo,
152 ModelRuntimeRequirement, ModelUpgrade, UnifiedRegistry,
153};
154pub use remote::RemoteBackend;
155pub use routing_ext::{
156 CircuitBreaker, CircuitBreakerRegistry, CircuitState, ImplicitSignal, ImplicitSignalType,
157 RoutingMode, SpendControl, SpendLimitExceeded, SpendLimits, SpendStatus,
158};
159pub use runner::{
160 current_inference_runner, set_inference_runner, EventEmitter, InferenceRunner, RunnerError,
161 RunnerResult,
162};
163pub use schema::{
164 ApiProtocol, ApproxCost, BenchmarkScore, CostModel, ModelCapability, ModelSchema, ModelSource,
165 PerformanceEnvelope, ProprietaryAuth, ProprietaryProtocol, ProprietaryWire, QuantScheme,
166 Quantization, TrustTier, OPENAI_CODEX_SIGN_IN_HINT,
167};
168
169pub use adaptive_router::{RouterExclusion, RouterExclusionReason, TaskComplexity};
171#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
172pub use backend::CandleBackend;
173#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
174pub use backend::EmbeddingBackend;
175#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
176pub use backend::SwiftLmBackend;
177pub use hardware::HardwareInfo;
178pub use models::{ModelRegistry, ModelRole};
179pub use router::{ModelRouter, RoutingDecision};
180pub use stream::{StreamAccumulator, StreamEvent};
181pub use tasks::{
182 parse_boxes, BoundingBox, ClassifyRequest, ClassifyResult, ContentBlock, EmbedRequest,
183 GenerateImageRequest, GenerateImageResult, GenerateParams, GenerateRequest,
184 GenerateVideoRequest, GenerateVideoResult, GroundRequest, GroundResult, Message,
185 OptionProbabilities, Provenance, RerankRequest, RerankResult, RerankedDocument, ResponseFormat,
186 RoutingWorkload, SynthesizeRequest, SynthesizeResult, ThinkingMode, ToolCall,
187 TranscribeRequest, TranscribeResult, VideoMode,
188};
189#[derive(Debug, Clone, PartialEq, Eq)]
197pub struct InferenceRetryProgress {
198 pub model: String,
200 pub attempt: u32,
202 pub reason: &'static str,
204 pub backoff_ms: u64,
206}
207
208#[derive(Error, Debug)]
209pub enum InferenceError {
210 #[error("model not found: {0}")]
211 ModelNotFound(String),
212
213 #[error("no eligible model remains after required exclusions: {excluded_models}")]
217 NoEligibleModel { excluded_models: String },
218
219 #[error("model download failed: {0}")]
220 DownloadFailed(String),
221
222 #[error("inference failed: {0}")]
223 InferenceFailed(String),
224
225 #[error("classify: the model's reply {reply:?} {reason}")]
229 ClassifyNoAnswer { reply: String, reason: String },
230
231 #[error("classify: invalid labels: {0}")]
235 InvalidClassifyLabels(String),
236
237 #[error("catalog precondition mismatch: {detail}")]
242 CatalogPreconditionMismatch { detail: String },
243
244 #[error("controlled inference termination confirmed")]
249 ControlledTermination,
250
251 #[error(transparent)]
252 ModelManagement(#[from] model_management::ModelManagementError),
253
254 #[error("{recovery}")]
257 LocalResourceBlocked {
258 preflight: resource_policy::LocalLoadPreflight,
259 recovery: String,
260 },
261
262 #[error("transient remote failure after retries (status={status:?}): {message}")]
271 Transient {
272 status: Option<u16>,
273 message: String,
274 },
275
276 #[error("deadline exceeded: the caller's {applied_ms} ms infer deadline elapsed after {elapsed_ms} ms; last attempt: {last_error}")]
284 DeadlineExceeded {
285 applied_ms: u64,
286 elapsed_ms: u64,
287 last_error: String,
288 },
289
290 #[error("mode {mode} not implemented on backend {backend}: {reason}")]
295 UnsupportedMode {
296 mode: &'static str,
297 backend: &'static str,
298 reason: &'static str,
299 },
300
301 #[error("{provider} account rejected the request (HTTP {status}): {message}")]
316 ProviderAccount {
317 provider: String,
318 status: u16,
319 message: String,
320 },
321
322 #[error("inference failed: {message}")]
331 ProviderKeyMissing {
332 provider: String,
333 model: String,
334 env_vars: Vec<String>,
337 message: String,
340 },
341
342 #[error("no credential for proprietary provider '{provider}' (model {model}): {detail}")]
359 CredentialUnavailable {
360 provider: String,
361 model: String,
362 reason: CredentialFailure,
364 detail: String,
366 },
367
368 #[error("{provider} account has no workspace yet: {detail}")]
385 WorkspaceRequired {
386 provider: String,
388 detail: String,
390 },
391
392 #[error("{provider} refused this request on content grounds{}: {message}",
412 content_refusal_detail(.kind.as_deref(), .code.as_deref()))]
413 ContentRefused {
414 provider: String,
415 kind: Option<String>,
417 code: Option<String>,
418 message: String,
419 },
420
421 #[error("{provider} gateway has no upstream configured for '{namespace}' (HTTP {status}): {message}")]
439 GatewayUnconfigured {
440 provider: String,
441 namespace: String,
442 status: u16,
443 message: String,
444 },
445
446 #[error("tokenization error: {0}")]
447 TokenizationError(String),
448
449 #[error("device error: {0}")]
450 DeviceError(String),
451
452 #[error("io error: {0}")]
453 Io(#[from] std::io::Error),
454}
455
456impl From<resource_policy::LocalAdmissionError> for InferenceError {
457 fn from(error: resource_policy::LocalAdmissionError) -> Self {
458 let recovery = match error.preflight.verdict {
459 resource_policy::LocalLoadVerdict::DisabledByPolicy => {
460 "Local model loading is disabled by the 0 GB allocation. Increase Local Models RAM in Settings, or choose a remote model. Downloads remain available.".to_string()
461 }
462 resource_policy::LocalLoadVerdict::ExceedsConfiguredCeiling => format!(
463 "This model needs about {} MB for this request, beyond the configured {} MB local-model allocation. Increase the allocation or choose a smaller model.",
464 error.preflight.estimated_incremental_mb,
465 error.preflight.configured_ceiling_mb
466 ),
467 resource_policy::LocalLoadVerdict::InsufficientLiveMemory => format!(
468 "Not enough memory is free to start this model while preserving CAR's {} MB emergency reserve. Close memory-heavy apps or choose a smaller model.",
469 error.preflight.emergency_reserve_mb
470 ),
471 resource_policy::LocalLoadVerdict::LiveMemoryUnknown => {
472 "CAR could not measure live memory. The static allocation fits, but current safety is unknown.".to_string()
473 }
474 resource_policy::LocalLoadVerdict::ModelMaintenance => {
475 "This local model is being removed or maintained. Wait for that operation to finish, then retry.".to_string()
476 }
477 resource_policy::LocalLoadVerdict::PendingTeardown => {
478 "CAR is still confirming that the previous local model process exited. Wait for teardown to finish, then retry.".to_string()
479 }
480 resource_policy::LocalLoadVerdict::Allowed => error.to_string(),
481 };
482 Self::LocalResourceBlocked {
483 preflight: error.preflight,
484 recovery,
485 }
486 }
487}
488
489fn error_counts_against_circuit_breaker(e: &InferenceError) -> bool {
526 failure_attribution(e).is_none()
527}
528
529fn content_refusal_detail(kind: Option<&str>, code: Option<&str>) -> String {
535 match (kind, code) {
536 (Some(k), Some(c)) => format!(" (type={k}, code={c})"),
537 (Some(k), None) => format!(" (type={k})"),
538 (None, Some(c)) => format!(" (code={c})"),
539 (None, None) => String::new(),
540 }
541}
542
543fn record_dispatch_failure(tracker: &mut OutcomeTracker, trace_id: &str, error: &InferenceError) {
549 match failure_attribution(error) {
550 Some(reason) => tracker.record_not_model_fault(trace_id, &error.to_string(), reason),
551 None => tracker.record_failure(trace_id, &error.to_string()),
552 }
553}
554
555fn failure_attribution(error: &InferenceError) -> Option<outcome::UnattributedReason> {
561 use outcome::UnattributedReason as R;
562 match error {
563 InferenceError::UnsupportedMode { .. }
564 | InferenceError::CatalogPreconditionMismatch { .. }
565 | InferenceError::InvalidClassifyLabels { .. } => Some(R::Capability),
567 InferenceError::ProviderAccount { .. }
568 | InferenceError::ProviderKeyMissing { .. }
569 | InferenceError::CredentialUnavailable { .. }
570 | InferenceError::WorkspaceRequired { .. }
571 | InferenceError::GatewayUnconfigured { .. } => Some(R::Account),
572 InferenceError::ContentRefused { .. } => Some(R::ContentRefused),
573 InferenceError::LocalResourceBlocked { .. } => Some(R::ResourceBlocked),
574 InferenceError::ControlledTermination => Some(R::Cancelled),
575 InferenceError::ModelNotFound { .. }
576 | InferenceError::NoEligibleModel { .. }
577 | InferenceError::DownloadFailed { .. }
578 | InferenceError::InferenceFailed { .. }
579 | InferenceError::ClassifyNoAnswer { .. }
580 | InferenceError::ModelManagement { .. }
581 | InferenceError::Transient { .. }
582 | InferenceError::DeadlineExceeded { .. }
583 | InferenceError::TokenizationError { .. }
584 | InferenceError::DeviceError { .. }
585 | InferenceError::Io { .. } => None,
586 }
587}
588
589#[cfg(any(
595 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
596 all(target_os = "ios", target_arch = "aarch64")
597))]
598fn send_stream_failure_blocking(
599 tx: &tokio::sync::mpsc::Sender<stream::StreamEvent>,
600 error: &InferenceError,
601) {
602 if let Some(reason) = failure_attribution(error) {
603 let _ = tx.blocking_send(stream::StreamEvent::Attribution(reason));
604 }
605 let _ = tx.blocking_send(stream::StreamEvent::Error(error.to_string()));
606}
607
608fn error_ends_fallback_chain(e: &InferenceError) -> bool {
630 if matches!(e, InferenceError::ContentRefused { provider, .. } if provider == "apple") {
637 return false;
638 }
639 matches!(
640 e,
641 InferenceError::ContentRefused { .. }
642 | InferenceError::CatalogPreconditionMismatch { .. }
643 | InferenceError::ControlledTermination
644 | InferenceError::WorkspaceRequired { .. }
650 )
651}
652
653fn apply_exhaustion_recovery_hint(underlying: InferenceError) -> InferenceError {
664 if matches!(
665 underlying,
666 InferenceError::ContentRefused { .. }
667 | InferenceError::CatalogPreconditionMismatch { .. }
668 | InferenceError::ControlledTermination
669 | InferenceError::WorkspaceRequired { .. }
674 ) {
675 return underlying;
676 }
677 let underlying_str = underlying.to_string();
678 match no_backend_recovery_hint(&underlying_str)
679 .or_else(|| auth_expired_recovery_hint(&underlying_str))
680 {
681 Some(msg) => InferenceError::InferenceFailed(msg),
682 None => underlying,
683 }
684}
685
686const AUTH_LOGIN_MARKER: &str = "auth login";
687const AUTH_STORE_UNREADABLE_MARKER: &str = "credential store unreadable";
688const AUTH_ENV_MISSING_MARKER: &str = "credential environment variable missing";
689
690const AUTH_FAILURE_MESSAGE_MARKERS: &[&str] = &[
695 "no credential for proprietary",
696 AUTH_LOGIN_MARKER,
697 "session has expired",
698 "cannot read parslee credentials",
699 AUTH_STORE_UNREADABLE_MARKER,
700 AUTH_ENV_MISSING_MARKER,
701 "credential was rejected",
707 "repair its provider login",
708];
709
710#[derive(Debug, Clone, PartialEq, Eq)]
720struct RouteCredentialFailure {
721 summary: String,
722 source_error: String,
723}
724
725fn parslee_signed_out_route_failure() -> RouteCredentialFailure {
726 let summary = "Parslee login is absent — run `car auth login` before retrying".to_string();
727 RouteCredentialFailure {
728 source_error: summary.clone(),
729 summary,
730 }
731}
732
733fn route_credential_failure(
734 candidate: &str,
735 error: &InferenceError,
736 promote_missing_credential: bool,
737) -> Option<String> {
738 match error {
739 InferenceError::CredentialUnavailable {
740 provider,
741 reason,
742 detail,
743 ..
744 } => {
745 let provider_name = if provider.eq_ignore_ascii_case("parslee") {
746 "Parslee".to_string()
747 } else {
748 provider.clone()
749 };
750 let summary = match reason {
751 CredentialFailure::Expired { .. } => format!(
752 "{provider_name} login expired for `{candidate}` — run `car auth login`"
753 ),
754 CredentialFailure::SignedOut => format!(
755 "{provider_name} login is absent for `{candidate}` — run `car auth login`"
756 ),
757 CredentialFailure::StoreUnreadable => format!(
758 "{provider_name} {AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
759 ),
760 CredentialFailure::EnvVarMissing { .. } if !promote_missing_credential => {
761 return None;
762 }
763 CredentialFailure::EnvVarMissing { env_var } => format!(
764 "{provider_name} {AUTH_ENV_MISSING_MARKER}: `{env_var}` for explicitly requested `{candidate}` — {detail}"
765 ),
766 CredentialFailure::RaceRetryable => return None,
769 };
770 Some(summary)
771 }
772 InferenceError::ProviderAccount {
773 provider, status, ..
774 } if matches!(*status, 401 | 403) => {
775 if provider.eq_ignore_ascii_case("parslee") {
776 Some(format!(
777 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
778 ))
779 } else {
780 Some(format!(
781 "{provider} credential was rejected for `{candidate}` (HTTP {status})"
782 ))
783 }
784 }
785 _ => {
786 let rendered = error.to_string();
787 let lower = rendered.to_ascii_lowercase();
788 if is_auth_rejection_message(&rendered) {
789 if candidate
790 .split_once('/')
791 .is_some_and(|(provider, _)| provider.eq_ignore_ascii_case("parslee"))
792 {
793 Some(format!(
794 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
795 ))
796 } else {
797 Some(format!(
798 "credential expired or was rejected for `{candidate}` — repair its provider login"
799 ))
800 }
801 } else if lower.contains(AUTH_STORE_UNREADABLE_MARKER)
802 || (promote_missing_credential && lower.contains("keychain lookup failed"))
803 {
804 Some(format!(
805 "{AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
806 ))
807 } else {
808 None
809 }
810 }
811 }
812}
813
814fn record_route_credential_failure(
815 slot: &mut Option<RouteCredentialFailure>,
816 candidate: &str,
817 error: &InferenceError,
818 promote_missing_credential: bool,
819) {
820 if let Some(summary) = route_credential_failure(candidate, error, promote_missing_credential) {
821 *slot = Some(RouteCredentialFailure {
826 summary,
827 source_error: error.to_string(),
828 });
829 }
830}
831
832fn chain_includes_parslee_route<'a>(
837 mut resolve: impl FnMut(&str) -> Option<&'a ModelSchema>,
838 chain: &[String],
839) -> bool {
840 chain.iter().any(|candidate| {
841 resolve(candidate).is_some_and(|schema| schema.provider.eq_ignore_ascii_case("parslee"))
842 })
843}
844
845fn apply_route_failure_context(
865 underlying: InferenceError,
866 credential: Option<&RouteCredentialFailure>,
867) -> InferenceError {
868 if matches!(
869 underlying,
870 InferenceError::ContentRefused { .. }
871 | InferenceError::CatalogPreconditionMismatch { .. }
872 | InferenceError::ControlledTermination
873 | InferenceError::WorkspaceRequired { .. }
881 ) {
882 return underlying;
883 }
884 let Some(credential) = credential else {
885 return apply_exhaustion_recovery_hint(underlying);
886 };
887 let rendered = underlying.to_string();
888 let summary = credential.summary.as_str();
889 let setup_hint = no_backend_recovery_hint(&rendered);
890 let connective = if rendered == credential.source_error {
891 "provider detail"
892 } else {
893 "fallback then failed"
894 };
895 let augment = |field: String| match &setup_hint {
896 Some(hint) => format!("{summary}; {hint}"),
898 None => format!("{summary}; {connective}: {field}"),
899 };
900 match underlying {
901 InferenceError::InferenceFailed(message) => {
902 InferenceError::InferenceFailed(augment(message))
903 }
904 InferenceError::ProviderKeyMissing {
905 provider,
906 model,
907 env_vars,
908 message,
909 } => InferenceError::ProviderKeyMissing {
910 provider,
911 model,
912 env_vars,
913 message: augment(message),
914 },
915 InferenceError::Transient { status, message } => InferenceError::Transient {
916 status,
917 message: augment(message),
918 },
919 InferenceError::ProviderAccount {
920 provider,
921 status,
922 message,
923 } => InferenceError::ProviderAccount {
924 provider,
925 status,
926 message: augment(message),
927 },
928 InferenceError::CredentialUnavailable {
929 provider,
930 model,
931 reason,
932 detail,
933 } => InferenceError::CredentialUnavailable {
934 provider,
935 model,
936 reason,
937 detail: augment(detail),
938 },
939 InferenceError::GatewayUnconfigured {
940 provider,
941 namespace,
942 status,
943 message,
944 } => InferenceError::GatewayUnconfigured {
945 provider,
946 namespace,
947 status,
948 message: augment(message),
949 },
950 InferenceError::LocalResourceBlocked {
951 preflight,
952 recovery,
953 } => InferenceError::LocalResourceBlocked {
954 preflight,
955 recovery: augment(recovery),
956 },
957 _ => InferenceError::InferenceFailed(augment(rendered)),
960 }
961}
962
963#[derive(Debug, Clone, PartialEq, Eq)]
972pub enum CredentialFailure {
973 Expired { expires_at: u64 },
980 SignedOut,
983 StoreUnreadable,
987 EnvVarMissing { env_var: String },
989 RaceRetryable,
992}
993
994#[derive(Debug, Clone, Copy, PartialEq, Eq)]
996pub enum Device {
997 Cpu,
998 Metal,
999 Cuda(usize), }
1001
1002impl Device {
1003 pub fn auto() -> Self {
1011 #[cfg(all(target_os = "macos", feature = "metal"))]
1012 {
1013 return Device::Metal;
1014 }
1015 #[cfg(all(
1016 any(target_os = "linux", target_os = "windows"),
1017 target_arch = "x86_64",
1018 not(car_skip_cuda)
1019 ))]
1020 {
1021 return Device::Cuda(0);
1022 }
1023 #[cfg(not(any(
1024 all(target_os = "macos", feature = "metal"),
1025 all(
1026 any(target_os = "linux", target_os = "windows"),
1027 target_arch = "x86_64",
1028 not(car_skip_cuda)
1029 )
1030 )))]
1031 {
1032 Device::Cpu
1033 }
1034 }
1035}
1036
1037#[derive(Debug, Clone)]
1039pub struct InferenceConfig {
1040 pub models_dir: std::path::PathBuf,
1046 pub state_root: std::path::PathBuf,
1055 pub device: Option<Device>,
1057 pub generation_model: String,
1059 pub preferred_generation_model: Option<String>,
1061 pub embedding_model: String,
1063 pub preferred_embedding_model: Option<String>,
1065 pub classification_model: String,
1067 pub preferred_classification_model: Option<String>,
1069}
1070
1071impl Default for InferenceConfig {
1072 fn default() -> Self {
1073 let models_dir = default_models_dir();
1090
1091 let hw = HardwareInfo::detect();
1092
1093 Self {
1094 models_dir,
1095 state_root: car_home::root_or_relative(),
1096 device: None,
1097 generation_model: hw.recommended_model,
1098 preferred_generation_model: None,
1099 embedding_model: "Qwen3-Embedding-0.6B".to_string(),
1100 preferred_embedding_model: None,
1101 classification_model: "Qwen3-0.6B".to_string(),
1102 preferred_classification_model: None,
1103 }
1104 }
1105}
1106
1107impl InferenceConfig {
1108 pub fn state_models_dir(&self) -> std::path::PathBuf {
1117 self.state_root.join("models")
1118 }
1119}
1120
1121pub fn default_models_dir() -> std::path::PathBuf {
1128 dirs_next()
1129 .unwrap_or_else(|| std::path::PathBuf::from("."))
1130 .join(".car")
1131 .join("models")
1132}
1133
1134fn dirs_next() -> Option<std::path::PathBuf> {
1135 std::env::var_os("HOME")
1139 .or_else(|| std::env::var_os("USERPROFILE"))
1140 .map(std::path::PathBuf::from)
1141}
1142
1143fn model_source_identity(schema: &ModelSchema) -> &str {
1144 match &schema.source {
1145 ModelSource::Local { hf_repo, .. }
1146 | ModelSource::Mlx { hf_repo, .. }
1147 | ModelSource::ManagedVllmMlx { hf_repo, .. } => hf_repo,
1148 ModelSource::WhisperCpp { model } | ModelSource::CodexCli { model } => model,
1149 _ => &schema.id,
1150 }
1151}
1152
1153#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1155pub struct TokenUsage {
1156 pub prompt_tokens: u64,
1166 pub completion_tokens: u64,
1168 pub total_tokens: u64,
1170 pub context_window: u64,
1172 #[serde(default)]
1177 pub cache_read_input_tokens: u64,
1178 #[serde(default)]
1183 pub cache_creation_input_tokens: u64,
1184}
1185
1186#[derive(
1188 Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq, schemars::JsonSchema,
1189)]
1190pub struct InferenceModelIdentity {
1191 #[serde(default)]
1194 pub requested_model_id: Option<String>,
1195 #[serde(default)]
1209 pub resolved_model_id: String,
1210 #[serde(default)]
1212 pub row_digest: String,
1213 #[serde(default)]
1215 pub catalog_revision: String,
1216}
1217
1218#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1220pub struct InferenceResult {
1221 pub text: String,
1223 pub tool_calls: Vec<crate::tasks::generate::ToolCall>,
1225 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1232 pub bounding_boxes: Vec<crate::tasks::grounding::BoundingBox>,
1233 pub trace_id: String,
1235 pub model_used: String,
1238 #[serde(flatten)]
1241 pub model_identity: InferenceModelIdentity,
1242 pub latency_ms: u64,
1244 #[serde(default)]
1256 pub time_to_first_token_ms: Option<u64>,
1257 pub usage: Option<TokenUsage>,
1276 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1289 pub provider_output_items: Vec<serde_json::Value>,
1290 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1298 pub thinking: Vec<crate::tasks::generate::ThinkingBlock>,
1299 #[serde(default)]
1315 pub stop_reason: Option<String>,
1316 #[serde(default, skip_serializing_if = "Option::is_none")]
1325 pub auth_fallback_from: Option<String>,
1326 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
1333 pub local_last_resort: bool,
1334 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1354 pub fallback_from: Vec<FallbackFrom>,
1355}
1356
1357#[derive(
1359 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1360)]
1361pub struct FallbackFrom {
1362 pub candidate: String,
1364 pub reason: FallbackReason,
1365}
1366
1367#[derive(
1375 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1376)]
1377#[serde(rename_all = "snake_case")]
1378pub enum FallbackReason {
1379 CredentialRejected,
1388 CredentialAbsent,
1391 RateLimited,
1394 QuotaExhausted,
1400 TimedOut,
1408 Failed,
1414}
1415
1416pub fn classify_fallback_reason(error: &InferenceError) -> FallbackReason {
1449 use FallbackReason as R;
1450 match error {
1451 InferenceError::ProviderAccount { status, .. } => match status {
1454 402 => R::QuotaExhausted,
1455 _ => R::CredentialRejected,
1456 },
1457 InferenceError::CredentialUnavailable { reason, .. } => match reason {
1459 CredentialFailure::Expired { .. } => R::CredentialRejected,
1461 CredentialFailure::SignedOut | CredentialFailure::EnvVarMissing { .. } => {
1463 R::CredentialAbsent
1464 }
1465 CredentialFailure::StoreUnreadable | CredentialFailure::RaceRetryable => R::Failed,
1468 },
1469 InferenceError::Transient {
1470 status: Some(429), ..
1471 } => R::RateLimited,
1472 InferenceError::Transient { status: None, .. } => R::Failed,
1479 InferenceError::Transient { .. } => R::Failed,
1480 InferenceError::DeadlineExceeded { .. } => R::TimedOut,
1483 InferenceError::InferenceFailed(msg)
1484 | InferenceError::ProviderKeyMissing { message: msg, .. } => classify_untyped_failure(msg),
1485 _ => R::Failed,
1486 }
1487}
1488
1489fn classify_untyped_failure(msg: &str) -> FallbackReason {
1497 use FallbackReason as R;
1498 if let Some(status) = parse_api_returned_status(msg) {
1501 return match status {
1502 401 | 403 => R::CredentialRejected,
1503 402 => R::QuotaExhausted,
1504 429 => R::RateLimited,
1505 408 | 504 => R::TimedOut,
1506 _ => R::Failed,
1507 };
1508 }
1509 let l = msg.to_ascii_lowercase();
1510 if l.contains("org lookup failed") {
1513 return if l.contains("http 401") || l.contains("http 403") {
1514 R::CredentialRejected
1515 } else if l.contains("http 429") {
1516 R::RateLimited
1517 } else {
1518 R::Failed
1519 };
1520 }
1521 if l.contains("authentication required")
1522 || l.contains("invalid_grant")
1523 || l.contains("token expired")
1524 {
1525 return R::CredentialRejected;
1526 }
1527 if l.contains("no credential for proprietary") || l.contains("no api key") {
1528 return R::CredentialAbsent;
1529 }
1530 if l.contains("too many requests") || l.contains("rate limit") {
1531 return R::RateLimited;
1532 }
1533 if l.contains("timed out") || l.contains("deadline exceeded") {
1534 return R::TimedOut;
1535 }
1536 R::Failed
1537}
1538
1539pub fn record_fallback_from(hops: &mut Vec<FallbackFrom>, candidate: &str, error: &InferenceError) {
1545 hops.push(FallbackFrom {
1546 candidate: candidate.to_string(),
1547 reason: classify_fallback_reason(error),
1548 });
1549}
1550
1551pub struct TrackedStream {
1558 pub model_used: String,
1560 pub trace_id: String,
1562 pub events: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
1564}
1565
1566struct AbortOnDropTask<T>(Option<tokio::task::JoinHandle<T>>);
1567
1568impl<T> AbortOnDropTask<T> {
1569 async fn join(mut self) -> Result<T, tokio::task::JoinError> {
1570 self.0.take().expect("owned task handle available").await
1571 }
1572}
1573
1574impl<T> Drop for AbortOnDropTask<T> {
1575 fn drop(&mut self) {
1576 if let Some(task) = self.0.take() {
1577 task.abort();
1578 }
1579 }
1580}
1581
1582tokio::task_local! {
1583 static PINNED_REGISTRY: (usize, Arc<UnifiedRegistry>);
1587}
1588
1589fn bound_model_identity(
1590 snapshot: &CatalogSnapshot,
1591 requested_model_id: Option<&str>,
1592 resolved_model_id: &str,
1593) -> Result<InferenceModelIdentity, InferenceError> {
1594 let row = snapshot.model_by_exact_id(resolved_model_id).ok_or_else(|| {
1595 InferenceError::InferenceFailed(format!(
1596 "resolved model `{resolved_model_id}` was absent from the catalog snapshot bound to this request"
1597 ))
1598 })?;
1599 Ok(InferenceModelIdentity {
1600 requested_model_id: requested_model_id.map(str::to_string),
1601 resolved_model_id: row.model.id.clone(),
1602 row_digest: row.row_digest.clone(),
1603 catalog_revision: snapshot.catalog_revision.clone(),
1604 })
1605}
1606
1607fn validate_expected_catalog_revision(
1608 req: &GenerateRequest,
1609 snapshot: &CatalogSnapshot,
1610) -> Result<(), InferenceError> {
1611 if let Some(expected) = req.expected_catalog_revision.as_deref() {
1612 if expected != snapshot.catalog_revision {
1613 return Err(InferenceError::CatalogPreconditionMismatch {
1614 detail: format!(
1615 "expected catalog revision {expected}, got {}",
1616 snapshot.catalog_revision
1617 ),
1618 });
1619 }
1620 }
1621 Ok(())
1622}
1623
1624fn validate_expected_catalog_row(
1625 req: &GenerateRequest,
1626 snapshot: &CatalogSnapshot,
1627 resolved_model_id: &str,
1628) -> Result<(), InferenceError> {
1629 let Some(expected) = req.expected_row_digest.as_deref() else {
1630 return Ok(());
1631 };
1632 let row = snapshot
1633 .model_by_exact_id(resolved_model_id)
1634 .ok_or_else(|| InferenceError::CatalogPreconditionMismatch {
1635 detail: format!(
1636 "resolved model `{resolved_model_id}` is absent from the bound catalog snapshot"
1637 ),
1638 })?;
1639 if expected != row.row_digest {
1640 return Err(InferenceError::CatalogPreconditionMismatch {
1641 detail: format!(
1642 "expected row digest {expected} for `{resolved_model_id}`, got {}",
1643 row.row_digest
1644 ),
1645 });
1646 }
1647 Ok(())
1648}
1649
1650const EXACT_MODEL_ID_PREFIX: &str = "\0car-exact-model-id:";
1651
1652pub fn pin_exact_model_id(req: &mut GenerateRequest, model_id: String) -> Result<(), String> {
1655 if req.model.is_some() {
1656 return Err("`model` and `model_id` are mutually exclusive".to_string());
1657 }
1658 if model_id.trim().is_empty() {
1659 return Err("`model_id` must be a non-empty immutable id".to_string());
1660 }
1661 req.model = Some(format!("{EXACT_MODEL_ID_PREFIX}{model_id}"));
1662 req.params.strict_model = true;
1663 Ok(())
1664}
1665
1666fn default_system_one_classifier<'a>(
1671 schemas: impl Iterator<Item = &'a ModelSchema>,
1672) -> Option<String> {
1673 schemas
1674 .filter(|schema| {
1675 schema.has_capability(ModelCapability::Classify)
1676 && schema.available_now()
1677 && matches!(
1678 &schema.source,
1679 ModelSource::Proprietary { protocol, .. }
1680 if protocol.wire == ProprietaryWire::SystemOne
1681 )
1682 })
1683 .map(|schema| schema.id.clone())
1684 .min()
1685}
1686
1687pub fn exact_pinned_model_id(req: &GenerateRequest) -> Option<&str> {
1688 req.model
1689 .as_deref()
1690 .and_then(|model| model.strip_prefix(EXACT_MODEL_ID_PREFIX))
1691}
1692
1693fn auto_thinking_budget(
1702 is_code_intent: bool,
1703 is_complex: bool,
1704 supports_thinking: bool,
1705) -> Option<usize> {
1706 if !supports_thinking {
1707 return None;
1708 }
1709 if is_code_intent {
1710 Some(24_000)
1711 } else if is_complex {
1712 Some(8_000)
1713 } else {
1714 None
1715 }
1716}
1717
1718fn is_explicit_code_intent(intent: Option<&intent::IntentHint>) -> bool {
1728 intent.and_then(|h| h.task) == Some(intent::TaskHint::Code)
1729}
1730
1731fn should_append_local_last_resort(chain_has_local: bool, strict_model: bool) -> bool {
1736 !chain_has_local && !strict_model
1737}
1738
1739fn resolved_max_tokens(requested: usize, schema: &ModelSchema) -> usize {
1763 if requested != crate::tasks::generate::DEFAULT_MAX_TOKENS
1764 || schema.decodes_in_process()
1765 || schema.is_codex_cli()
1766 {
1767 return requested;
1768 }
1769 schema.effective_max_output()
1770}
1771
1772const DEFAULT_LOCAL_DECODE_TIMEOUT_SECS: u64 = 300;
1785
1786pub const LOCAL_DECODE_TIMEOUT_STOP_REASON: &str = "local_decode_timeout";
1793
1794#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1802const LOCAL_DECODE_HEARTBEAT_SECS: u64 = 10;
1803
1804fn parse_decode_timeout(raw: Option<&str>) -> Option<std::time::Duration> {
1807 let secs = match raw {
1808 Some(v) => v
1809 .trim()
1810 .parse::<u64>()
1811 .unwrap_or(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS),
1812 None => DEFAULT_LOCAL_DECODE_TIMEOUT_SECS,
1813 };
1814 (secs > 0).then(|| std::time::Duration::from_secs(secs))
1815}
1816
1817#[cfg_attr(
1830 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
1831 allow(dead_code)
1832)]
1833fn deadline_exceeded(elapsed: std::time::Duration, timeout: Option<std::time::Duration>) -> bool {
1834 timeout.is_some_and(|limit| elapsed >= limit)
1835}
1836
1837#[cfg_attr(
1843 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
1844 allow(dead_code)
1845)]
1846fn heartbeat_due(
1847 elapsed: std::time::Duration,
1848 last: std::time::Duration,
1849 interval: std::time::Duration,
1850) -> bool {
1851 elapsed.saturating_sub(last) >= interval
1852}
1853
1854#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1858enum EmptyPassAction {
1859 RetryWithoutThinking,
1862 FailDecodeCeiling,
1866 Accept,
1868}
1869
1870fn classify_empty_pass(
1878 recover: bool,
1879 stop_reason: Option<&str>,
1880 text: &str,
1881 tool_calls_empty: bool,
1882) -> EmptyPassAction {
1883 if !text.trim().is_empty() || !tool_calls_empty {
1884 return EmptyPassAction::Accept;
1885 }
1886 if stop_reason == Some(LOCAL_DECODE_TIMEOUT_STOP_REASON) {
1887 return EmptyPassAction::FailDecodeCeiling;
1888 }
1889 if recover {
1890 return EmptyPassAction::RetryWithoutThinking;
1891 }
1892 EmptyPassAction::Accept
1893}
1894
1895fn local_decode_timeout() -> Option<std::time::Duration> {
1896 parse_decode_timeout(
1897 std::env::var("CAR_LOCAL_DECODE_TIMEOUT_SECS")
1898 .ok()
1899 .as_deref(),
1900 )
1901}
1902
1903impl InferenceResult {
1904 pub fn served_model_id(&self) -> &str {
1909 if self.model_identity.resolved_model_id.is_empty() {
1910 &self.model_used
1911 } else {
1912 &self.model_identity.resolved_model_id
1913 }
1914 }
1915
1916 pub fn has_tool_calls(&self) -> bool {
1918 !self.tool_calls.is_empty()
1919 }
1920
1921 pub fn was_truncated(&self) -> bool {
1933 matches!(
1934 self.stop_reason.as_deref(),
1935 Some(
1936 "length"
1937 | "max_tokens"
1938 | "max_output_tokens"
1939 | "MAX_TOKENS"
1940 | crate::LOCAL_DECODE_TIMEOUT_STOP_REASON
1941 )
1942 )
1943 }
1944
1945 pub fn append_assistant_history(
1954 &self,
1955 messages: &mut Vec<crate::tasks::generate::Message>,
1956 tool_calls: Vec<crate::tasks::generate::ToolCall>,
1957 ) {
1958 if !self.provider_output_items.is_empty() {
1959 messages.push(crate::tasks::generate::Message::ProviderOutputItems {
1960 protocol: crate::protocol::OPENAI_RESPONSES_PROTOCOL.to_string(),
1961 items: self.provider_output_items.clone(),
1962 });
1963 }
1964 messages.push(crate::tasks::generate::Message::Assistant {
1965 content: self.text.clone(),
1966 tool_calls,
1967 thinking: self.thinking.clone(),
1968 model_id: Some(self.served_model_id().to_string()),
1969 local_last_resort: self.local_last_resort,
1970 });
1971 }
1972}
1973
1974#[derive(Debug, Clone, Serialize)]
1975pub struct SpeechRuntimeHealth {
1976 pub root: PathBuf,
1977 pub installed: bool,
1978 pub python: PathBuf,
1979 pub stt_command: PathBuf,
1980 pub tts_command: PathBuf,
1981 pub configured_python: Option<String>,
1982 pub detected_python: Option<String>,
1983}
1984
1985#[derive(Debug, Clone, Serialize)]
1986pub struct SpeechModelHealth {
1987 pub id: String,
1988 pub name: String,
1989 pub provider: String,
1990 pub capability: ModelCapability,
1991 pub is_local: bool,
1992 pub available: bool,
1993 pub cached: bool,
1994 pub selected_by_default: bool,
1995 pub source: String,
1996}
1997
1998#[derive(Debug, Clone, Serialize)]
1999pub struct SpeechHealthReport {
2000 pub runtime: SpeechRuntimeHealth,
2001 pub local_models: Vec<SpeechModelHealth>,
2002 pub remote_models: Vec<SpeechModelHealth>,
2003 pub elevenlabs_configured: bool,
2004 pub prefer_local: bool,
2005 pub allow_remote_fallback: bool,
2006 pub preferred_local_stt: Option<String>,
2007 pub preferred_local_tts: Option<String>,
2008 pub preferred_remote_stt: Option<String>,
2009 pub preferred_remote_tts: Option<String>,
2010 pub local_stt_default: Option<String>,
2011 pub local_tts_default: Option<String>,
2012 pub remote_stt_default: Option<String>,
2013 pub remote_tts_default: Option<String>,
2014}
2015
2016#[derive(Debug, Clone, Serialize)]
2017pub struct ModelDefaultHealth {
2018 pub capability: ModelCapability,
2019 pub configured_model: String,
2020 pub available: bool,
2021 pub is_local: bool,
2022 pub provider: Option<String>,
2023}
2024
2025#[derive(Debug, Clone, Serialize)]
2026pub struct ModelProviderHealth {
2027 pub provider: String,
2028 pub configured: bool,
2029 pub local_models: usize,
2030 pub remote_models: usize,
2031 pub available_models: usize,
2032 pub capabilities: Vec<ModelCapability>,
2033}
2034
2035#[derive(Debug, Clone, Serialize)]
2036pub struct ModelCapabilityHealth {
2037 pub capability: ModelCapability,
2038 pub total_models: usize,
2039 pub available_models: usize,
2040 pub local_available_models: usize,
2041 pub remote_available_models: usize,
2042}
2043
2044#[derive(Debug, Clone, Serialize)]
2045pub struct RoutingScenarioHealth {
2046 pub name: String,
2047 pub workload: RoutingWorkload,
2048 pub task_family: String,
2049 pub has_tools: bool,
2050 pub has_vision: bool,
2051 pub prefer_local: bool,
2052 pub quality_first_cold_start: bool,
2053 pub bootstrap_min_task_observations: u64,
2054 pub bootstrap_quality_floor: f64,
2055 pub model_id: String,
2056 pub model_name: String,
2057 pub reason: String,
2058 pub strategy: RoutingStrategy,
2059}
2060
2061#[derive(Debug, Clone, Serialize)]
2062pub struct ModelBenchmarkPriorHealth {
2063 pub model_id: String,
2064 pub model_name: Option<String>,
2065 pub overall_score: f64,
2066 pub overall_latency_ms: Option<f64>,
2067 pub task_scores: std::collections::HashMap<String, f64>,
2068 pub task_latency_ms: std::collections::HashMap<String, f64>,
2069 pub source_path: PathBuf,
2070}
2071
2072#[derive(Debug, Clone, Serialize)]
2073pub struct ModelHealthReport {
2074 pub total_models: usize,
2075 pub available_models: usize,
2076 pub local_models: usize,
2077 pub remote_models: usize,
2078 pub defaults: Vec<ModelDefaultHealth>,
2079 pub providers: Vec<ModelProviderHealth>,
2080 pub capabilities: Vec<ModelCapabilityHealth>,
2081 pub routing_prefer_local: bool,
2082 pub routing_quality_first_cold_start: bool,
2083 pub routing_min_observations: u64,
2084 pub routing_bootstrap_min_task_observations: u64,
2085 pub routing_bootstrap_quality_floor: f64,
2086 pub routing_quality_weight: f64,
2087 pub routing_latency_weight: f64,
2088 pub routing_cost_weight: f64,
2089 pub routing_scenarios: Vec<RoutingScenarioHealth>,
2090 pub benchmark_priors: Vec<ModelBenchmarkPriorHealth>,
2091 pub speech: SpeechHealthReport,
2092}
2093
2094#[derive(Debug, Clone, Serialize)]
2095pub struct SpeechInstallReport {
2096 pub name: String,
2097 pub hf_repo: String,
2098 pub snapshot_path: PathBuf,
2099 pub files_downloaded: usize,
2100}
2101
2102#[derive(Debug, Clone, Serialize)]
2103pub struct SpeechSmokePathReport {
2104 pub path: String,
2105 pub tts_model: String,
2106 pub stt_model: String,
2107 pub audio_path: PathBuf,
2108 pub transcript: String,
2109}
2110
2111#[derive(Debug, Clone, Serialize, Default)]
2112pub struct SpeechSmokeReport {
2113 pub local: Option<SpeechSmokePathReport>,
2114 pub remote: Option<SpeechSmokePathReport>,
2115 pub skipped: Vec<String>,
2116}
2117
2118#[derive(Debug, Clone, Serialize, Default)]
2119pub struct SpeechPolicy {
2120 pub prefer_local: bool,
2121 pub allow_remote_fallback: bool,
2122 pub preferred_local_stt: Option<String>,
2123 pub preferred_local_tts: Option<String>,
2124 pub preferred_remote_stt: Option<String>,
2125 pub preferred_remote_tts: Option<String>,
2126}
2127
2128fn render_for_local_backend(mut req: GenerateRequest) -> GenerateRequest {
2137 let has_msgs = req.messages.as_ref().is_some_and(|m| !m.is_empty());
2138 let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
2139 if has_msgs || has_tools {
2140 req.prompt = tasks::generate::render_chat_prompt(&req);
2141 req.messages = None;
2142 req.tools = None;
2143 }
2144 req
2145}
2146
2147#[cfg(test)]
2148static UNRETIRED_RELEASE_WARNING_COUNT: std::sync::atomic::AtomicUsize =
2149 std::sync::atomic::AtomicUsize::new(0);
2150
2151#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2165fn foundation_models_usage(
2166 instructions: Option<&str>,
2167 prompt: &str,
2168 completion: &str,
2169 context_window: u64,
2170) -> Option<crate::TokenUsage> {
2171 let (prompt_tokens, completion_tokens) =
2172 crate::backend::foundation_models::count_tokens(instructions, prompt, completion)?;
2173 Some(crate::TokenUsage {
2174 prompt_tokens,
2175 completion_tokens,
2176 total_tokens: prompt_tokens + completion_tokens,
2177 context_window,
2178 ..Default::default()
2180 })
2181}
2182
2183pub struct InferenceEngine {
2184 pub config: InferenceConfig,
2185 unified: std::sync::RwLock<Arc<UnifiedRegistry>>,
2191 registry_write: std::sync::Mutex<()>,
2194 pub adaptive_router: AdaptiveRouter,
2196 pub outcome_tracker: Arc<RwLock<OutcomeTracker>>,
2198 last_outcome_flush: Arc<std::sync::Mutex<Option<Instant>>>,
2204 outcome_profiles_loaded: bool,
2208 ledger_io_lock: Arc<tokio::sync::Mutex<()>>,
2213 spend_limits: Arc<std::sync::RwLock<Option<SpendLimits>>>,
2219 lane_defaults_cache: Arc<std::sync::RwLock<crate::lane_defaults::LaneDefaults>>,
2224 external_pins: Arc<std::sync::RwLock<std::collections::BTreeMap<String, Vec<String>>>>,
2228 disk_pressured: Arc<std::sync::atomic::AtomicBool>,
2231 failed_upgrades: Arc<std::sync::Mutex<std::collections::HashMap<String, u64>>>,
2235 concierge_action_lock: Arc<tokio::sync::Mutex<()>>,
2241 concierge_action_seq: Arc<std::sync::atomic::AtomicU64>,
2245 remote_backend: RemoteBackend,
2247 model_management: model_management::ModelManagementStore,
2250 local_admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2252 _runtime_scope: Arc<ScopedInferenceRuntime>,
2257 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2258 resource_policy_evidence: Arc<std::sync::RwLock<resource_policy::ResourcePolicyLoadEvidence>>,
2262 model_budget: Arc<backend_cache::SharedModelBudget>,
2265 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2271 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2272 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2279 local_backends:
2280 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2281 pub registry: models::ModelRegistry,
2283 pub router: ModelRouter,
2284 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2285 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2286 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2287 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2288 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2289 speech_policy: SpeechPolicy,
2290 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2294}
2295
2296enum SpeechCandidateAdmission {
2297 Proceed(Option<resource_policy::LocalLoadReservation>),
2298 SkipBlocked(InferenceError),
2299 FailBlocked(InferenceError),
2300}
2301
2302struct ScopedInferenceRuntime {
2303 model_budget: Arc<backend_cache::SharedModelBudget>,
2304 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2305 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2306 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2307 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2308 local_backends:
2309 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2310 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2311 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2312 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2313 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2314 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2315 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2316 #[cfg(test)]
2317 load_probe: Arc<backend_cache::BackendCache<()>>,
2318}
2319
2320fn scoped_inference_runtime_registry() -> &'static std::sync::Mutex<
2321 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2322> {
2323 static REGISTRY: std::sync::OnceLock<
2324 std::sync::Mutex<
2325 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2326 >,
2327 > = std::sync::OnceLock::new();
2328 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
2329}
2330
2331fn configured_cache_budget_mb(default_mb: u64) -> u64 {
2332 std::env::var("CAR_INFERENCE_MODEL_CACHE_MB")
2333 .ok()
2334 .and_then(|value| value.parse::<u64>().ok())
2335 .unwrap_or(default_mb)
2336}
2337
2338fn scoped_inference_runtime(
2339 state_root: &Path,
2340 configured_ceiling_mb: u64,
2341 admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2342) -> Arc<ScopedInferenceRuntime> {
2343 let state_root = resource_policy::normalized_state_root_key(state_root);
2344 let mut registry = scoped_inference_runtime_registry()
2345 .lock()
2346 .unwrap_or_else(std::sync::PoisonError::into_inner);
2347 registry.retain(|_, runtime| runtime.strong_count() > 0);
2348 if let Some(runtime) = registry.get(&state_root).and_then(std::sync::Weak::upgrade) {
2349 runtime.model_budget.set_budget_bytes(
2350 configured_cache_budget_mb(configured_ceiling_mb).saturating_mul(1024 * 1024),
2351 );
2352 return runtime;
2353 }
2354
2355 let model_budget = backend_cache::SharedModelBudget::from_env_or(configured_ceiling_mb);
2356 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2357 let cache_idle = backend_cache::idle_ttl_from_env();
2358 let runtime = Arc::new(ScopedInferenceRuntime {
2359 model_budget: model_budget.clone(),
2360 resource_policy_generation: Arc::new(std::sync::atomic::AtomicU64::new(1)),
2361 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2362 text_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2363 model_budget.clone(),
2364 cache_idle,
2365 Some(admission.clone()),
2366 )),
2367 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2368 local_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2369 model_budget.clone(),
2370 cache_idle,
2371 Some(admission.clone()),
2372 )),
2373 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2374 backend: Arc::new(RwLock::new(std::collections::HashMap::new())),
2375 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2376 embedding_backend: Arc::new(RwLock::new(None)),
2377 speech_runtime: Arc::new(Mutex::new(None)),
2378 vllm_pool: Arc::new(
2382 vllm_pool::VllmServerPool::with_admission(
2383 std::time::Duration::from_secs(
2384 std::env::var("CAR_VLLM_IDLE_SECS")
2385 .ok()
2386 .and_then(|value| value.parse().ok())
2387 .unwrap_or(300),
2388 ),
2389 admission,
2390 )
2391 .with_process_records(state_root.join("run").join("vllm-mlx")),
2392 ),
2393 #[cfg(test)]
2394 load_probe: Arc::new(backend_cache::BackendCache::new(1024)),
2395 });
2396 registry.insert(state_root, Arc::downgrade(&runtime));
2397 runtime
2398}
2399
2400impl InferenceEngine {
2401 fn requires_local_admission(schema: &ModelSchema) -> bool {
2402 schema.is_car_managed_vllm_mlx() || Self::supports_worker_offload(schema)
2403 }
2404
2405 fn reserve_in_outer_dispatch(schema: &ModelSchema) -> bool {
2411 Self::requires_local_admission(schema) && !schema.is_car_managed_vllm_mlx()
2412 }
2413
2414 fn supports_worker_offload(schema: &ModelSchema) -> bool {
2415 matches!(
2416 schema.source,
2417 ModelSource::Local { .. } | ModelSource::Mlx { .. }
2418 )
2419 }
2420
2421 pub fn local_model_retention(&self, model_id: &str) -> backend_cache::BackendRetention {
2424 if self.local_admission.is_resident(model_id) {
2425 backend_cache::BackendRetention::Resident
2426 } else {
2427 backend_cache::BackendRetention::Transient
2428 }
2429 }
2430 pub fn local_model_preflight(
2432 &self,
2433 model_id: &str,
2434 context_tokens: usize,
2435 ) -> Result<resource_policy::LocalLoadPreflight, InferenceError> {
2436 let schema = self
2437 .resolve_schema(model_id)
2438 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
2439 self.ensure_model_enabled(&schema.id)?;
2440 Ok(self.local_admission.preflight(&schema, context_tokens))
2441 }
2442
2443 pub fn active_local_resource_policy(&self) -> resource_policy::ResourcePolicyLoadEvidence {
2450 let mut evidence = self
2451 .resource_policy_evidence
2452 .read()
2453 .unwrap_or_else(|poisoned| poisoned.into_inner())
2454 .clone();
2455 evidence.policy = self.local_admission.policy();
2456 evidence
2457 }
2458
2459 pub fn apply_local_resource_policy(&self, policy: resource_policy::ResourcePolicy) {
2463 let ceiling_mb = policy
2464 .effective_budget(HardwareInfo::detect().total_ram_mb)
2465 .configured_model_ceiling_mb;
2466 self.local_admission.set_policy(policy.clone());
2467 *self
2468 .resource_policy_evidence
2469 .write()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner()) =
2471 resource_policy::ResourcePolicyLoadEvidence {
2472 policy,
2473 source: resource_policy::ResourcePolicyLoadSource::Loaded,
2474 warning: None,
2475 };
2476 let cache_ceiling_mb = configured_cache_budget_mb(ceiling_mb);
2477 self.model_budget
2478 .set_budget_bytes(cache_ceiling_mb.saturating_mul(1024 * 1024));
2479 let generation = self
2480 .resource_policy_generation
2481 .fetch_add(1, std::sync::atomic::Ordering::AcqRel)
2482 .saturating_add(1);
2483 if let Some(offload) = crate::offload::current_local_offload() {
2484 offload.refresh_resource_policy(generation);
2485 }
2486 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2487 {
2488 self.text_backends.enforce_budget();
2489 self.local_backends.enforce_budget();
2490 }
2491 }
2492
2493 pub fn begin_local_model_maintenance(
2494 &self,
2495 model_id: &str,
2496 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2497 {
2498 self.local_admission.begin_model_maintenance(model_id)
2499 }
2500
2501 pub async fn prepare_local_model_removal(
2502 &self,
2503 model_id: &str,
2504 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2505 {
2506 let maintenance = self.local_admission.begin_model_maintenance(model_id)?;
2507 if let Some(offload) = crate::offload::current_local_offload() {
2508 if offload
2509 .resident_models()
2510 .await
2511 .iter()
2512 .any(|resident| resident == model_id)
2513 {
2514 let acknowledged = offload.release_model(model_id).await.map_err(|error| {
2515 resource_policy::ModelMaintenanceError::ReleaseFailed(error.to_string())
2516 })?;
2517 if !acknowledged
2518 || offload
2519 .resident_models()
2520 .await
2521 .iter()
2522 .any(|resident| resident == model_id)
2523 {
2524 return Err(
2525 resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
2526 model_id.to_string(),
2527 ),
2528 );
2529 }
2530 }
2531 }
2532 if self
2533 .vllm_pool
2534 .release_model_if_present(model_id)
2535 .await
2536 .is_err()
2537 {
2538 return Err(
2539 resource_policy::ModelMaintenanceError::ProcessReleaseUnacknowledged(
2540 model_id.to_string(),
2541 ),
2542 );
2543 }
2544 if !self.evict_local_model_if_idle(model_id) {
2545 return Err(resource_policy::ModelMaintenanceError::CacheReleaseBlocked(
2546 model_id.to_string(),
2547 ));
2548 }
2549 let allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2550 if !allocation_ids.is_empty() {
2551 return Err(resource_policy::ModelMaintenanceError::ResidualResidency {
2552 model_id: model_id.to_string(),
2553 allocation_ids,
2554 });
2555 }
2556 Ok(maintenance)
2557 }
2558
2559 pub fn evict_local_model_if_idle(&self, model_id: &str) -> bool {
2563 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2564 {
2565 let mut allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2566 if !allocation_ids.iter().any(|id| id == model_id) {
2567 allocation_ids.push(model_id.to_string());
2568 }
2569 allocation_ids.into_iter().all(|allocation_id| {
2570 [
2571 self.text_backends.evict_if_idle(&allocation_id),
2572 self.local_backends.evict_if_idle(&allocation_id),
2573 ]
2574 .into_iter()
2575 .all(|evicted| evicted)
2576 })
2577 }
2578 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2579 {
2580 let _ = model_id;
2581 let backend_released = self.backend.try_write().is_ok_and(|mut backends| {
2582 if backends.remove(model_id).is_some() {
2583 self.local_admission.mark_evicted(model_id);
2584 }
2585 true
2586 });
2587 backend_released
2591 && self
2592 .embedding_backend
2593 .try_write()
2594 .is_ok_and(|slot| slot.is_none())
2595 }
2596 }
2597
2598 fn reserve_local_request(
2599 &self,
2600 schema: &ModelSchema,
2601 context_tokens: usize,
2602 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2603 self.ensure_model_enabled(&schema.id)?;
2604 let activity_lease = self.model_management.acquire_lease(&schema.id)?;
2605 let mut reservation = self
2606 .local_admission
2607 .reserve(schema, context_tokens)
2608 .map_err(InferenceError::from)?;
2609 reservation.attach_activity_lease(activity_lease);
2610 Ok(reservation)
2611 }
2612
2613 async fn reserve_local_request_with_worker_retry(
2618 &self,
2619 schema: &ModelSchema,
2620 context_tokens: usize,
2621 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2622 let original_error = match self.reserve_local_request(schema, context_tokens) {
2623 Ok(reservation) => return Ok(reservation),
2624 Err(error) => error,
2625 };
2626 let first_verdict = match &original_error {
2627 InferenceError::LocalResourceBlocked { preflight, .. } => preflight.verdict.clone(),
2628 _ => return Err(original_error),
2629 };
2630 if first_verdict != resource_policy::LocalLoadVerdict::InsufficientLiveMemory {
2631 return Err(original_error);
2632 }
2633
2634 let Some(offload) = crate::offload::current_local_offload() else {
2635 tracing::info!(
2636 model = %schema.id,
2637 first_verdict = ?first_verdict,
2638 release_happened = false,
2639 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2640 "local admission worker release-and-retry decision"
2641 );
2642 return Err(original_error);
2643 };
2644 if !offload
2645 .resident_models()
2646 .await
2647 .iter()
2648 .any(|model_id| model_id == &schema.id)
2649 {
2650 tracing::info!(
2651 model = %schema.id,
2652 first_verdict = ?first_verdict,
2653 release_happened = false,
2654 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2655 "local admission worker release-and-retry decision"
2656 );
2657 return Err(original_error);
2658 }
2659 let Some(scoped_allocation_id) = offload.resident_allocation_id(&schema.id) else {
2660 tracing::debug!(
2661 model = %schema.id,
2662 "release-and-retry skipped because the offload exposes no allocation id"
2663 );
2664 return Err(original_error);
2665 };
2666 match offload.release_model(&schema.id).await {
2667 Ok(true) => {
2668 let offload_still_resident = offload
2672 .resident_models()
2673 .await
2674 .iter()
2675 .any(|model_id| model_id == &schema.id);
2676 let allocation_still_charged = self
2679 .local_admission
2680 .resident_allocation_ids(&schema.id)
2681 .iter()
2682 .any(|allocation_id| allocation_id == &scoped_allocation_id);
2683 if offload_still_resident || allocation_still_charged {
2684 tracing::warn!(
2688 model = %schema.id,
2689 allocation = %scoped_allocation_id,
2690 offload_still_resident,
2691 allocation_still_charged,
2692 "release acknowledged but allocation {} still charged/pending",
2693 scoped_allocation_id
2694 );
2695 #[cfg(test)]
2696 UNRETIRED_RELEASE_WARNING_COUNT
2697 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2698 return Err(original_error);
2699 }
2700 let second = self.reserve_local_request(schema, context_tokens);
2701 let second_verdict = match &second {
2702 Ok(_) => Some("admitted".to_string()),
2703 Err(InferenceError::LocalResourceBlocked { preflight, .. }) => {
2704 Some(format!("{:?}", preflight.verdict))
2705 }
2706 Err(_) => None,
2707 };
2708 tracing::info!(
2709 model = %schema.id,
2710 first_verdict = ?first_verdict,
2711 release_happened = true,
2712 second_verdict = ?second_verdict,
2713 "local admission worker release-and-retry decision"
2714 );
2715 second
2716 }
2717 Ok(false) | Err(_) => {
2718 tracing::info!(
2719 model = %schema.id,
2720 first_verdict = ?first_verdict,
2721 release_happened = false,
2722 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2723 "local admission worker release-and-retry decision"
2724 );
2725 Err(original_error)
2726 }
2727 }
2728 }
2729
2730 #[cfg(any(
2735 test,
2736 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))
2737 ))]
2738 fn reconcile_transient_local_allocation(
2739 reservation: &mut resource_policy::LocalLoadReservation,
2740 measured_weights_bytes: u64,
2741 ) -> Result<(), InferenceError> {
2742 reservation
2743 .reconcile_measured_weights(measured_weights_bytes)
2744 .map(|_| ())
2745 .map_err(InferenceError::from)
2746 }
2747
2748 fn prepare_worker_admission(
2749 &self,
2750 schema: &ModelSchema,
2751 reservation: &mut resource_policy::LocalLoadReservation,
2752 ) -> Result<crate::offload::LocalWorkerAdmission, InferenceError> {
2753 let installed = self.config.models_dir.join(&schema.name);
2754 let installed_weights_bytes = backend_cache::estimate_model_size(&installed);
2755 let measured_weights_bytes =
2762 installed_weights_bytes.max(reservation.reconciled_weights_bytes());
2763 reservation
2764 .reconcile_measured_weights(measured_weights_bytes)
2765 .map_err(InferenceError::from)?;
2766 Ok(crate::offload::LocalWorkerAdmission {
2767 policy: self.local_admission.policy(),
2768 policy_generation: self
2769 .resource_policy_generation
2770 .load(std::sync::atomic::Ordering::Acquire),
2771 state_root: resource_policy::normalized_state_root_key(&self.config.state_root),
2772 measured_weights_bytes,
2773 })
2774 }
2775
2776 async fn reconcile_worker_residency(
2777 offload: &dyn crate::offload::LocalGenerationOffload,
2778 expected_model_id: &str,
2779 residency: &crate::offload::LocalWorkerResidency,
2780 retention: backend_cache::BackendRetention,
2781 reservation: &mut resource_policy::LocalLoadReservation,
2782 ) -> Result<(), InferenceError> {
2783 if residency.model_id != expected_model_id {
2784 if retention == backend_cache::BackendRetention::Resident {
2785 let released = offload
2786 .release_model(&residency.model_id)
2787 .await
2788 .unwrap_or(false);
2789 if !released {
2790 let allocation_id = offload
2791 .resident_allocation_id(&residency.model_id)
2792 .unwrap_or_else(|| {
2793 resource_policy::worker_process_allocation_id(&residency.model_id)
2794 });
2795 reservation.publish_resident_weights_as(
2796 &allocation_id,
2797 residency.measured_weights_bytes,
2798 );
2799 }
2800 }
2801 return Err(InferenceError::InferenceFailed(format!(
2802 "local worker acknowledged model '{}' for requested '{}'",
2803 residency.model_id, expected_model_id
2804 )));
2805 }
2806 if retention == backend_cache::BackendRetention::Resident {
2807 let allocation_id = offload
2808 .resident_allocation_id(expected_model_id)
2809 .unwrap_or_else(|| {
2810 resource_policy::worker_process_allocation_id(expected_model_id)
2811 });
2812 reservation
2813 .publish_resident_weights_as(&allocation_id, residency.measured_weights_bytes);
2814 }
2815 Ok(())
2816 }
2817
2818 fn admit_speech_candidate(
2819 &self,
2820 schema: &ModelSchema,
2821 explicit: bool,
2822 ) -> SpeechCandidateAdmission {
2823 if !schema.is_local()
2824 || matches!(
2825 schema.source,
2826 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. }
2827 )
2828 {
2829 return SpeechCandidateAdmission::Proceed(None);
2830 }
2831 match self.reserve_local_request(schema, 0) {
2832 Ok(reservation) => SpeechCandidateAdmission::Proceed(Some(reservation)),
2833 Err(error) if explicit => SpeechCandidateAdmission::FailBlocked(error),
2834 Err(error) => SpeechCandidateAdmission::SkipBlocked(error),
2835 }
2836 }
2837
2838 fn hold_local_reservation_for_stream(
2839 mut source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
2840 reservation: resource_policy::LocalLoadReservation,
2841 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
2842 let (tx, rx) = tokio::sync::mpsc::channel(64);
2843 tokio::spawn(async move {
2844 let _reservation = reservation;
2845 while let Some(event) = source.recv().await {
2846 if tx.send(event).await.is_err() {
2847 break;
2848 }
2849 }
2850 });
2851 rx
2852 }
2853
2854 fn hold_optional_reservation_for_stream(
2855 source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
2856 reservation: Option<resource_policy::LocalLoadReservation>,
2857 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
2858 match reservation {
2859 Some(reservation) => Self::hold_local_reservation_for_stream(source, reservation),
2860 None => source,
2861 }
2862 }
2863
2864 pub fn set_spend_limits(&self, limits: Option<SpendLimits>) {
2870 *self.spend_limits.write().unwrap() = limits;
2871 }
2872
2873 fn preferred_model_for_capability(&self, capability: ModelCapability) -> Option<&str> {
2874 match capability {
2875 ModelCapability::Generate => self.config.preferred_generation_model.as_deref(),
2876 ModelCapability::Embed => self.config.preferred_embedding_model.as_deref(),
2877 ModelCapability::Classify => self.config.preferred_classification_model.as_deref(),
2878 _ => None,
2879 }
2880 }
2881
2882 fn request_has_tools(req: &GenerateRequest) -> bool {
2888 req.tools.as_ref().is_some_and(|t| !t.is_empty())
2889 }
2890
2891 fn request_needs_vision(req: &GenerateRequest) -> bool {
2892 req.images.as_ref().is_some_and(|images| !images.is_empty())
2893 || req.messages.as_ref().is_some_and(|messages| {
2894 messages
2895 .iter()
2896 .any(|msg| matches!(msg, Message::UserMultimodal { .. }))
2897 })
2898 }
2899
2900 #[allow(dead_code)] fn request_has_video(req: &GenerateRequest) -> bool {
2906 let images_have_video = req
2907 .images
2908 .as_ref()
2909 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_video));
2910 let messages_have_video = req.messages.as_ref().is_some_and(|messages| {
2911 messages.iter().any(|msg| match msg {
2912 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_video),
2913 _ => false,
2914 })
2915 });
2916 images_have_video || messages_have_video
2917 }
2918
2919 #[allow(dead_code)] fn request_has_audio(req: &GenerateRequest) -> bool {
2924 let images_have_audio = req
2925 .images
2926 .as_ref()
2927 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_audio));
2928 let messages_have_audio = req.messages.as_ref().is_some_and(|messages| {
2929 messages.iter().any(|msg| match msg {
2930 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_audio),
2931 _ => false,
2932 })
2933 });
2934 images_have_audio || messages_have_audio
2935 }
2936
2937 pub fn new(config: InferenceConfig) -> Self {
2938 let registry = models::ModelRegistry::new(config.models_dir.clone());
2939 let hw = HardwareInfo::detect();
2940 let policy_evidence =
2941 resource_policy::FileResourcePolicyRepository::new(config.state_root.clone())
2942 .load_with_evidence()
2943 .unwrap_or_else(|error| {
2944 tracing::warn!(%error, "failed to read local model resource policy; using Everyday");
2945 resource_policy::ResourcePolicyLoadEvidence {
2946 policy: resource_policy::ResourcePolicy::everyday(),
2947 source: resource_policy::ResourcePolicyLoadSource::CorruptDefault,
2948 warning: Some(format!(
2949 "The local-model resource policy could not be read ({error}); CAR used Everyday."
2950 )),
2951 }
2952 });
2953 let policy = policy_evidence.policy.clone();
2954 let effective_budget = policy.effective_budget(hw.total_ram_mb);
2955 let local_admission = resource_policy::scoped_local_admission_with_models_dir(
2956 &config.state_root,
2957 policy,
2958 hw.clone(),
2959 Some(config.models_dir.clone()),
2960 );
2961 let runtime_scope = scoped_inference_runtime(
2962 &config.state_root,
2963 effective_budget.configured_model_ceiling_mb,
2964 local_admission.clone(),
2965 );
2966 let router = ModelRouter::new(hw.clone());
2967 let unified_registry = UnifiedRegistry::new_with_state_root(
2968 config.state_root.clone(),
2969 config.models_dir.clone(),
2970 );
2971 let adaptive_router = AdaptiveRouter::with_default_config(hw);
2972 let mut tracker = OutcomeTracker::new();
2973 let profiles_path = config.state_models_dir().join("outcome_profiles.json");
2975 let outcome_profiles_loaded = match tracker.load_from_file(&profiles_path) {
2978 Ok(n) => {
2979 if n > 0 {
2980 tracing::info!(loaded = n, "loaded persisted model profiles");
2981 }
2982 true
2983 }
2984 Err(error) => error.kind() == std::io::ErrorKind::NotFound,
2985 };
2986 let mut benchmark_models_loaded = 0usize;
2987 for path in benchmark_priors_paths(&config.state_models_dir()) {
2988 match routing_ext::load_benchmark_priors(&path) {
2989 Ok(priors) if !priors.is_empty() => {
2990 benchmark_models_loaded += priors.len();
2991 routing_ext::apply_benchmark_priors(&mut tracker, &priors);
2992 tracing::info!(
2993 path = %path.display(),
2994 loaded = priors.len(),
2995 "loaded benchmark quality priors"
2996 );
2997 }
2998 Ok(_) => {}
2999 Err(error) => {
3000 tracing::warn!(path = %path.display(), %error, "failed to load benchmark priors");
3001 }
3002 }
3003 }
3004 if benchmark_models_loaded > 0 {
3005 tracing::info!(
3006 loaded = benchmark_models_loaded,
3007 "applied benchmark priors to cold-start routing"
3008 );
3009 }
3010 let outcome_tracker = Arc::new(RwLock::new(tracker));
3011
3012 let remote_backend = RemoteBackend::new();
3013 let model_management = model_management::ModelManagementStore::new(
3014 config.state_root.clone(),
3015 config.models_dir.clone(),
3016 );
3017
3018 let lane_defaults_path = config
3019 .state_root
3020 .join(crate::lane_defaults::LANE_DEFAULTS_FILE);
3021 let next_action_seq = crate::action_ledger::read_actions(
3022 &config
3023 .state_root
3024 .join(crate::action_ledger::ACTION_LEDGER_FILE),
3025 0,
3026 )
3027 .iter()
3028 .map(|a| a.seq)
3029 .max()
3030 .map(|m| m + 1)
3031 .unwrap_or(1);
3032 Self {
3033 config,
3034 unified: std::sync::RwLock::new(Arc::new(unified_registry)),
3035 registry_write: std::sync::Mutex::new(()),
3036 adaptive_router,
3037 outcome_tracker,
3038 last_outcome_flush: Arc::new(std::sync::Mutex::new(None)),
3039 outcome_profiles_loaded,
3040 ledger_io_lock: Arc::new(tokio::sync::Mutex::new(())),
3041 spend_limits: Arc::new(std::sync::RwLock::new(None)),
3042 lane_defaults_cache: Arc::new(std::sync::RwLock::new(crate::lane_defaults::load_from(
3043 &lane_defaults_path,
3044 ))),
3045 concierge_action_lock: Arc::new(tokio::sync::Mutex::new(())),
3046 external_pins: Default::default(),
3047 disk_pressured: Default::default(),
3048 failed_upgrades: Default::default(),
3049 concierge_action_seq: Arc::new(std::sync::atomic::AtomicU64::new(next_action_seq)),
3050 remote_backend,
3051 model_management,
3052 local_admission: local_admission.clone(),
3053 _runtime_scope: runtime_scope.clone(),
3054 resource_policy_generation: runtime_scope.resource_policy_generation.clone(),
3055 resource_policy_evidence: Arc::new(std::sync::RwLock::new(policy_evidence)),
3056 model_budget: runtime_scope.model_budget.clone(),
3057 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3058 text_backends: runtime_scope.text_backends.clone(),
3059 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3060 local_backends: runtime_scope.local_backends.clone(),
3061 registry,
3062 router,
3063 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3064 backend: runtime_scope.backend.clone(),
3065 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3066 embedding_backend: runtime_scope.embedding_backend.clone(),
3067 speech_runtime: runtime_scope.speech_runtime.clone(),
3068 speech_policy: SpeechPolicy {
3069 prefer_local: cfg!(all(
3070 target_os = "macos",
3071 target_arch = "aarch64",
3072 not(car_skip_mlx)
3073 )),
3074 allow_remote_fallback: true,
3075 preferred_local_stt: None,
3076 preferred_local_tts: None,
3077 preferred_remote_stt: None,
3078 preferred_remote_tts: None,
3079 },
3080 vllm_pool: runtime_scope.vllm_pool.clone(),
3081 }
3082 }
3083
3084 pub async fn init_key_pool(&self) {
3087 for schema in self.unified_registry().list() {
3089 if schema.is_remote() {
3090 self.remote_backend.register_model_keys(schema).await;
3091 }
3092 }
3093
3094 let stats_path = self.config.state_models_dir().join("key_pool_stats.json");
3096 if let Ok(n) = self.remote_backend.key_pool.load_stats(&stats_path).await {
3097 if n > 0 {
3098 tracing::info!(loaded = n, "loaded persisted key pool stats");
3099 }
3100 }
3101
3102 let total = self.remote_backend.key_pool.total_keys().await;
3103 if total > 0 {
3104 tracing::info!(keys = total, "key pool initialized");
3105 }
3106 }
3107
3108 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3111 async fn ensure_backend(
3112 &self,
3113 schema: &ModelSchema,
3114 reservation: &mut resource_policy::LocalLoadReservation,
3115 ) -> Result<(), InferenceError> {
3116 let read = self.backend.read().await;
3117 if read.contains_key(&schema.id) {
3118 return Ok(());
3119 }
3120 drop(read);
3121
3122 let mut write = self.backend.write().await;
3123 if write.contains_key(&schema.id) {
3124 return Ok(());
3125 }
3126
3127 let model_path = self.registry.ensure_model(&schema.name).await?;
3128 let mut measured = backend_cache::estimate_model_size(&model_path);
3129 reservation
3130 .reconcile_measured_weights(measured)
3131 .map_err(InferenceError::from)?;
3132 let device = self.config.device.unwrap_or_else(Device::auto);
3133 let backend = match CandleBackend::load(&model_path, device) {
3135 Ok(b) => b,
3136 Err(load_err) => {
3137 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3141 return Err(load_err);
3142 }
3143 tracing::warn!(
3144 model = %schema.id,
3145 error = %load_err,
3146 "candle backend load failed; purged corrupt cache files and re-pulling once"
3147 );
3148 let model_path = self.registry.ensure_model(&schema.name).await?;
3149 measured = backend_cache::estimate_model_size(&model_path);
3150 reservation
3151 .reconcile_measured_weights(measured)
3152 .map_err(InferenceError::from)?;
3153 CandleBackend::load(&model_path, device)?
3154 }
3155 };
3156 write.insert(schema.id.clone(), backend);
3157 reservation.publish_resident_weights(measured);
3158 Ok(())
3159 }
3160
3161 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3164 async fn ensure_embedding_backend(
3165 &self,
3166 reservation: &mut resource_policy::LocalLoadReservation,
3167 ) -> Result<(), InferenceError> {
3168 let read = self.embedding_backend.read().await;
3169 if read.is_some() {
3170 return Ok(());
3171 }
3172 drop(read);
3173
3174 let mut write = self.embedding_backend.write().await;
3175 if write.is_some() {
3176 return Ok(());
3177 }
3178
3179 let embedding_model = self
3180 .preferred_model_for_capability(ModelCapability::Embed)
3181 .unwrap_or(&self.config.embedding_model);
3182 let model_path = self.registry.ensure_model(embedding_model).await?;
3183 let mut measured = backend_cache::estimate_model_size(&model_path);
3184 reservation
3185 .reconcile_measured_weights(measured)
3186 .map_err(InferenceError::from)?;
3187 let device = self.config.device.unwrap_or_else(Device::auto);
3188 let backend = match EmbeddingBackend::load(&model_path, device) {
3190 Ok(b) => b,
3191 Err(load_err) => {
3192 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3193 return Err(load_err);
3194 }
3195 tracing::warn!(
3196 model = embedding_model,
3197 error = %load_err,
3198 "embedding backend load failed; purged corrupt cache files and re-pulling once"
3199 );
3200 let model_path = self.registry.ensure_model(embedding_model).await?;
3201 measured = backend_cache::estimate_model_size(&model_path);
3202 reservation
3203 .reconcile_measured_weights(measured)
3204 .map_err(InferenceError::from)?;
3205 EmbeddingBackend::load(&model_path, device)?
3206 }
3207 };
3208 *write = Some(backend);
3209 reservation.publish_resident_weights(measured);
3210 Ok(())
3211 }
3212
3213 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3216 async fn ensure_mlx_embedding_backend(&self) -> Result<String, InferenceError> {
3217 let embedding_model_name = self
3218 .preferred_model_for_capability(ModelCapability::Embed)
3219 .unwrap_or(&self.config.embedding_model)
3220 .to_string();
3221 let schema = self
3222 .resolve_schema(&embedding_model_name)
3223 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model_name.clone()))?
3224 .clone();
3225 Ok(schema.id)
3226 }
3227
3228 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3244 async fn load_backend_healing<T, F, RF, R>(
3245 schema_id: &str,
3246 model_dir: std::path::PathBuf,
3247 cache: &backend_cache::BackendCache<T>,
3248 size: u64,
3249 reservation: &mut resource_policy::LocalLoadReservation,
3250 loader: F,
3251 repull: RF,
3252 ) -> Result<
3253 (
3254 backend_cache::CachedBackend<T>,
3255 backend_cache::BackendRetention,
3256 ),
3257 InferenceError,
3258 >
3259 where
3260 T: Send + 'static,
3261 F: Fn(&Path) -> Result<T, InferenceError>,
3262 RF: FnOnce() -> R,
3263 R: std::future::Future<Output = Result<std::path::PathBuf, InferenceError>>,
3264 {
3265 match cache.get_or_load_admitted(schema_id, size, reservation, || loader(&model_dir)) {
3266 Ok(admitted) => Ok(admitted),
3267 Err(load_err) => {
3268 let purged = crate::download::purge_corrupt_cache_files(&model_dir);
3276 if purged == 0 {
3277 return Err(load_err);
3279 }
3280 tracing::warn!(
3281 model = schema_id,
3282 purged,
3283 error = %load_err,
3284 "backend load failed; purged corrupt cache files and re-pulling once"
3285 );
3286 let fresh_dir = repull().await?;
3287 let fresh_size = backend_cache::estimate_model_size(&fresh_dir);
3288 cache
3289 .get_or_load_admitted(schema_id, fresh_size, reservation, || loader(&fresh_dir))
3290 }
3291 }
3292 }
3293
3294 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3298 async fn ensure_text_backend(
3299 &self,
3300 schema: &ModelSchema,
3301 reservation: &mut resource_policy::LocalLoadReservation,
3302 ) -> Result<
3303 (
3304 backend_cache::CachedBackend<backend::SwiftLmBackend>,
3305 backend_cache::BackendRetention,
3306 ),
3307 InferenceError,
3308 > {
3309 if !Self::supports_native_mlx(schema) {
3310 return Err(InferenceError::InferenceFailed(format!(
3311 "native MLX backend does not support {} ({}) yet; use vLLM-MLX or add a family-specific MLX backend",
3312 schema.name, schema.family
3313 )));
3314 }
3315
3316 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3317 let size = backend_cache::estimate_model_size(&model_dir);
3318 if !reservation.authorizes_model(&schema.id) {
3319 return Err(InferenceError::InferenceFailed(format!(
3320 "local admission reservation does not authorize {}",
3321 schema.id
3322 )));
3323 }
3324 let role = if schema.has_capability(ModelCapability::Embed)
3334 && !schema.has_capability(ModelCapability::Generate)
3335 {
3336 backend::swift_lm::BackendRole::Embedding
3337 } else {
3338 backend::swift_lm::BackendRole::Text
3339 };
3340 let loader = move |dir: &Path| {
3341 let dir = dir.to_path_buf();
3342 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3343 backend::SwiftLmBackend::load_with_role(&dir, role)
3345 }))
3346 .map_err(|e| {
3347 InferenceError::InferenceFailed(format!(
3348 "MLX backend loading panicked (possible Metal/accelerate exception): {:?}",
3349 e
3350 ))
3351 })?
3352 };
3353 let registry = self.unified_registry();
3354 Self::load_backend_healing(
3355 &schema.id,
3356 model_dir,
3357 &self.text_backends,
3358 size,
3359 reservation,
3360 loader,
3361 || registry.redownload_local(&schema.id),
3362 )
3363 .await
3364 }
3365
3366 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3371 async fn ensure_local_backend(
3372 &self,
3373 schema: &ModelSchema,
3374 reservation: &mut resource_policy::LocalLoadReservation,
3375 ) -> Result<
3376 (
3377 backend_cache::CachedBackend<Box<dyn backend::local::LocalInferenceBackend>>,
3378 backend_cache::BackendRetention,
3379 ),
3380 InferenceError,
3381 > {
3382 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3383 let size = backend_cache::estimate_model_size(&model_dir);
3384 if !reservation.authorizes_model(&schema.id) {
3385 return Err(InferenceError::InferenceFailed(format!(
3386 "local admission reservation does not authorize {}",
3387 schema.id
3388 )));
3389 }
3390 let loader = |dir: &Path| {
3391 let dir = dir.to_path_buf();
3392 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3393 backend::SwiftLmBackend::load(&dir)
3403 .map(|b| Box::new(b) as Box<dyn backend::local::LocalInferenceBackend>)
3404 .map_err(|e| {
3405 InferenceError::InferenceFailed(format!(
3406 "mlx-swift-lm could not load model_type '{}' ({}): {e}",
3407 backend::local::describe_model_type(&dir),
3408 dir.display()
3409 ))
3410 })
3411 }))
3412 .map_err(|e| {
3413 InferenceError::InferenceFailed(format!(
3414 "local backend loading panicked (possible Metal/accelerate exception): {:?}",
3415 e
3416 ))
3417 })?
3418 };
3419 let registry = self.unified_registry();
3420 Self::load_backend_healing(
3421 &schema.id,
3422 model_dir,
3423 &self.local_backends,
3424 size,
3425 reservation,
3426 loader,
3427 || registry.redownload_local(&schema.id),
3428 )
3429 .await
3430 }
3431
3432 pub async fn reset_local_kv_cache(&self, model_id: &str) {
3443 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3447 {
3448 let Some(schema) = self.unified_registry().get(model_id).cloned() else {
3449 return;
3450 };
3451 if !matches!(
3455 schema.source,
3456 ModelSource::Mlx { .. } | ModelSource::Local { .. }
3457 ) {
3458 return;
3459 }
3460 if !self.local_backends.contains(&schema.id) {
3463 return;
3464 }
3465 let Ok(mut reservation) = self.reserve_local_request(&schema, 0) else {
3466 return;
3467 };
3468 if let Ok((handle, _retention)) =
3469 self.ensure_local_backend(&schema, &mut reservation).await
3470 {
3471 if let Ok(mut guard) = handle.lock() {
3472 guard.clear_kv_cache();
3473 }
3474 }
3475 }
3476 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3477 let _ = model_id;
3478 }
3479
3480 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3487 pub async fn warm_up<S: AsRef<str>>(
3488 &self,
3489 schema_ids: &[S],
3490 ) -> Vec<Result<(), InferenceError>> {
3491 let mut results = Vec::with_capacity(schema_ids.len());
3492 for id in schema_ids {
3493 let id = id.as_ref();
3494 let outcome: Result<(), InferenceError> = async {
3495 let schema = self.unified_registry().get(id).cloned().ok_or_else(|| {
3496 InferenceError::InferenceFailed(format!("warm_up: unknown schema id {id}"))
3497 })?;
3498 let mut reservation = self.reserve_local_request(&schema, 0)?;
3500 match schema.capabilities.first().copied() {
3501 Some(ModelCapability::ImageGeneration) => {
3502 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3507 }
3508 Some(ModelCapability::VideoGeneration) => {
3509 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3514 }
3515 Some(ModelCapability::TextToSpeech) => {
3516 }
3520 _ => {
3521 let _ = self.ensure_text_backend(&schema, &mut reservation).await?;
3522 }
3523 }
3524 Ok(())
3525 }
3526 .await;
3527 results.push(outcome);
3528 }
3529 results
3530 }
3531
3532 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3534 pub async fn warm_up<S: AsRef<str>>(
3535 &self,
3536 _schema_ids: &[S],
3537 ) -> Vec<Result<(), InferenceError>> {
3538 Vec::new()
3539 }
3540
3541 async fn vllm_live_schema(
3547 &self,
3548 schema: ModelSchema,
3549 reservation: Option<resource_policy::LocalLoadReservation>,
3550 context_tokens: usize,
3551 ) -> Result<(ModelSchema, Option<resource_policy::LocalLoadReservation>), InferenceError> {
3552 match &schema.source {
3553 ModelSource::ManagedVllmMlx { .. } => {}
3554 ModelSource::VllmMlx { .. } => return Ok((schema, None)),
3555 _ => return Ok((schema, None)),
3556 }
3557 let _dispatch = self.vllm_pool.acquire_dispatch(&schema.id).await;
3562 self.vllm_pool
3563 .wait_for_teardown(&schema.id, std::time::Duration::from_secs(5))
3564 .await
3565 .map_err(InferenceError::InferenceFailed)?;
3566 self.vllm_pool
3571 .reap_dead(&schema.id)
3572 .await
3573 .map_err(InferenceError::InferenceFailed)?;
3574 let mut reservation = match reservation {
3575 Some(reservation) if reservation.model_id() == schema.id => reservation,
3576 _ => self.reserve_local_request(&schema, context_tokens)?,
3577 };
3578 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3579 let model_name = model_dir.display().to_string();
3580 let measured = backend_cache::estimate_model_size(&model_dir)
3581 .max(reservation.reconciled_weights_bytes());
3582 reservation
3583 .reconcile_measured_weights(measured)
3584 .map_err(InferenceError::from)?;
3585 let endpoint = self
3587 .vllm_pool
3588 .ensure(&schema.id, &model_name, &reservation, &schema.family)
3591 .await
3592 .map_err(InferenceError::InferenceFailed)?;
3593 let allocation_id = resource_policy::vllm_process_allocation_id(&schema.id);
3594 reservation.publish_resident_weights_as(&allocation_id, measured);
3595 let mut schema = schema;
3596 schema.source = ModelSource::VllmMlx {
3597 endpoint,
3598 model_name,
3599 };
3600 Ok((schema, Some(reservation)))
3601 }
3602
3603 pub async fn evict_idle_vllm_servers(&self) -> usize {
3606 self.vllm_pool.evict_idle().await
3607 }
3608
3609 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3620 pub fn evict_idle_backends(&self) -> (usize, u64) {
3621 let mut entries = 0usize;
3622 let mut bytes = 0u64;
3623 for (n, b) in [
3624 self.text_backends.evict_idle(),
3625 self.local_backends.evict_idle(),
3626 ] {
3627 entries += n;
3628 bytes = bytes.saturating_add(b);
3629 }
3630 (entries, bytes)
3631 }
3632
3633 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3635 pub fn evict_idle_backends(&self) -> (usize, u64) {
3636 (0, 0)
3637 }
3638
3639 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3640 fn supports_native_mlx(schema: &ModelSchema) -> bool {
3641 matches!(schema.family.as_str(), "qwen3" | "qwen2.5-vl" | "qwen2-vl")
3642 }
3643
3644 fn resolve_schema(&self, name: &str) -> Option<ModelSchema> {
3648 let registry = self.unified_registry();
3649 registry
3650 .get(name)
3651 .or_else(|| registry.find_by_name(name))
3652 .cloned()
3653 }
3654
3655 fn resolve_schema_name_first(&self, name: &str) -> Option<ModelSchema> {
3658 let registry = self.unified_registry();
3659 registry
3660 .find_by_name(name)
3661 .or_else(|| registry.get(name))
3662 .cloned()
3663 }
3664
3665 pub fn unified_registry(&self) -> Arc<UnifiedRegistry> {
3670 let me = self as *const Self as usize;
3671 PINNED_REGISTRY
3672 .try_with(|(engine, pinned)| (*engine == me).then(|| pinned.clone()))
3673 .ok()
3674 .flatten()
3675 .unwrap_or_else(|| self.current_registry())
3676 }
3677
3678 fn current_registry(&self) -> Arc<UnifiedRegistry> {
3680 self.unified
3681 .read()
3682 .unwrap_or_else(std::sync::PoisonError::into_inner)
3683 .clone()
3684 }
3685
3686 async fn with_pinned_registry<F: std::future::Future>(&self, operation: F) -> F::Output {
3695 let me = self as *const Self as usize;
3696 let pinned_here = PINNED_REGISTRY
3697 .try_with(|(engine, _)| *engine == me)
3698 .unwrap_or(false);
3699 if pinned_here {
3700 operation.await
3701 } else {
3702 PINNED_REGISTRY
3703 .scope((me, self.current_registry()), operation)
3704 .await
3705 }
3706 }
3707
3708 pub fn unified_registry_mut(&mut self) -> &mut UnifiedRegistry {
3712 Arc::make_mut(
3713 self.unified
3714 .get_mut()
3715 .unwrap_or_else(std::sync::PoisonError::into_inner),
3716 )
3717 }
3718
3719 fn update_registry<R>(&self, edit: impl FnOnce(&mut UnifiedRegistry) -> R) -> R {
3725 let _writer = self
3726 .registry_write
3727 .lock()
3728 .unwrap_or_else(std::sync::PoisonError::into_inner);
3729 let mut next = (*self.current_registry()).clone();
3730 let out = edit(&mut next);
3731 *self
3732 .unified
3733 .write()
3734 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(next);
3735 out
3736 }
3737
3738 pub fn swap_signed_catalog(&self, catalog: catalog::VerifiedCatalog) -> usize {
3747 let origin = Arc::new(catalog.envelope());
3748 let revoked: std::collections::HashSet<String> =
3749 catalog.revoked().iter().cloned().collect();
3750 self.update_registry(|registry| {
3751 registry.refresh_availability();
3754 let mut keep: std::collections::HashSet<String> =
3756 self.protected_models(registry, true).into_keys().collect();
3757 keep.extend(
3758 registry
3759 .list()
3760 .into_iter()
3761 .filter(|m| {
3762 (m.is_local() && m.downloads_weights() && m.weights_ready)
3765 || matches!(self.model_management.load_receipt(&m.id), Ok(Some(_)))
3766 })
3767 .map(|m| m.id.clone()),
3768 );
3769 let loaded = registry.replace_signed_catalog(
3770 catalog.into_models(),
3771 Some(origin),
3772 &keep,
3773 &revoked,
3774 );
3775 registry.refresh_availability();
3776 self.persist_retained(registry);
3777 loaded
3778 })
3779 }
3780
3781 fn persist_retained(&self, registry: &UnifiedRegistry) {
3784 let path = catalog::retained_path(&self.config.state_root);
3785 if let Err(error) = catalog::save_retained(&path, ®istry.retained_rows()) {
3786 tracing::warn!(path = %path.display(), %error, "could not record retained catalog rows");
3787 }
3788 }
3789
3790 fn keep_pulled_row_if_withdrawn(&self, name: &str) {
3794 let Some(schema) = self.resolve_schema_name_first(name) else {
3795 return;
3796 };
3797 if self.current_registry().get(&schema.id).is_some() {
3798 return;
3799 }
3800 let Some(origin) = self.unified_registry().signed_origin_of(&schema.id) else {
3801 return;
3802 };
3803 self.update_registry(|registry| {
3804 registry.restore_retained(vec![(schema, origin)]);
3805 registry.refresh_availability();
3806 self.persist_retained(registry);
3807 });
3808 }
3809
3810 fn catalog_registry_snapshot(&self) -> UnifiedRegistry {
3811 self.catalog_registry_snapshot_with(false)
3812 }
3813
3814 fn catalog_registry_snapshot_with(&self, authoritative_credentials: bool) -> UnifiedRegistry {
3823 let mut registry = (*self.unified_registry()).clone();
3824 registry.prune_missing_on_disk_models();
3825 if authoritative_credentials {
3830 registry.refresh_availability_for_explicit_status();
3831 } else {
3832 registry.refresh_availability();
3833 }
3834 self.filter_disabled_local_models(&mut registry);
3835 registry
3836 }
3837
3838 fn filter_disabled_local_models(&self, registry: &mut UnifiedRegistry) {
3839 let disabled = registry
3840 .all()
3841 .filter(|schema| {
3842 schema.downloads_weights()
3843 && !self
3844 .model_management
3845 .car_enabled(&schema.id)
3846 .unwrap_or(false)
3847 })
3848 .map(|schema| schema.id.clone())
3849 .collect::<Vec<_>>();
3850 for model_id in disabled {
3851 registry.unregister(&model_id);
3852 }
3853 }
3854
3855 async fn routing_registry_snapshot_with_credential_failure(
3856 &self,
3857 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
3858 let parslee = car_auth::resolve_credential(car_auth::CredentialReadMode::Use).await;
3864 let (parslee_api_base, parslee_signed_out, credential_failure) = match &parslee {
3865 Ok(Some(credential)) => (Some(credential.api_base.as_str()), false, None),
3866 Ok(None) => (None, true, Some(parslee_signed_out_route_failure())),
3867 Err(error) => {
3868 let source_error = error.to_string();
3869 (
3870 None,
3871 false,
3872 Some(RouteCredentialFailure {
3873 summary: format!(
3874 "Parslee {AUTH_STORE_UNREADABLE_MARKER} — unlock or grant access to the credential store, then retry"
3875 ),
3876 source_error,
3877 }),
3878 )
3879 }
3880 };
3881 let mut registry = (*self.unified_registry()).clone();
3882 registry.prune_missing_on_disk_models();
3883 registry.refresh_routing_availability(parslee_api_base, parslee_signed_out);
3884 self.filter_disabled_local_models(&mut registry);
3885 (registry, credential_failure)
3886 }
3887
3888 async fn work_context_routing_snapshot(
3889 &self,
3890 context: &car_auth::context::CredentialContext,
3891 requested_model: Option<&str>,
3892 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
3893 let credential = if requested_model.is_some() {
3896 None
3897 } else {
3898 Some(car_auth::context::resolve_context_credential(context).await)
3899 };
3900 let available = context.organization_id.is_some()
3901 && credential.as_ref().is_none_or(|result| result.is_ok());
3902 let failure =
3903 credential
3904 .and_then(|result| result.err())
3905 .map(|error| RouteCredentialFailure {
3906 summary: format!("Work-context credentials unavailable: {error}"),
3907 source_error: error.to_string(),
3908 });
3909 let mut registry = (*self.unified_registry()).clone();
3910 registry.prune_missing_on_disk_models();
3911 registry.refresh_work_context_availability(&context.api_base, available);
3912 self.filter_disabled_local_models(&mut registry);
3913 (registry, failure)
3914 }
3915
3916 async fn routing_registry_snapshot(&self) -> UnifiedRegistry {
3917 self.routing_registry_snapshot_with_credential_failure()
3918 .await
3919 .0
3920 }
3921
3922 async fn request_routing_registry_snapshot(
3927 &self,
3928 requested_model: Option<&str>,
3929 ) -> UnifiedRegistry {
3930 if requested_model.is_some() {
3931 self.catalog_registry_snapshot()
3932 } else {
3933 self.routing_registry_snapshot().await
3934 }
3935 }
3936
3937 pub async fn route_adaptive(&self, prompt: &str) -> AdaptiveRoutingDecision {
3939 self.route_adaptive_with_intent(prompt, None).await
3940 }
3941
3942 pub async fn route_adaptive_with_intent(
3952 &self,
3953 prompt: &str,
3954 intent: Option<crate::intent::IntentHint>,
3955 ) -> AdaptiveRoutingDecision {
3956 if let Some(model) = self.preferred_model_for_capability(ModelCapability::Generate) {
3957 let exclude_set = self
3958 .adaptive_router
3959 .build_exclude_set(intent.as_ref(), &self.unified_registry());
3960 let overridden_is_excluded =
3961 Self::model_is_excluded(&exclude_set, &self.unified_registry(), model);
3962 if !overridden_is_excluded {
3963 let ctx_len = self
3969 .resolve_schema(model)
3970 .map(|s| s.context_length)
3971 .unwrap_or(0);
3972 return AdaptiveRoutingDecision {
3973 model_id: model.to_string(),
3974 model_name: model.to_string(),
3975 task: InferenceTask::Generate,
3976 complexity: TaskComplexity::assess(prompt),
3977 reason: "preferred generation model override".into(),
3978 strategy: RoutingStrategy::Explicit,
3979 predicted_quality: 0.5,
3980 fallbacks: vec![],
3981 context_length: ctx_len,
3982 needs_compaction: false,
3983 candidates: vec![],
3984 };
3985 }
3986 }
3987 let routing_registry = self.routing_registry_snapshot().await;
3988 let tracker = self.outcome_tracker.read().await;
3989 match intent {
3990 Some(hint) => self
3991 .adaptive_router
3992 .route_with(crate::adaptive_router::RouteRequest {
3993 intent: Some(&hint),
3994 ..crate::adaptive_router::RouteRequest::new(prompt, &routing_registry, &tracker)
3995 }),
3996 None => self
3997 .adaptive_router
3998 .route(prompt, &routing_registry, &tracker),
3999 }
4000 }
4001
4002 pub async fn route_adaptive_with_work_context(
4005 &self,
4006 prompt: &str,
4007 intent: Option<crate::intent::IntentHint>,
4008 context: &car_auth::context::CredentialContext,
4009 ) -> Result<AdaptiveRoutingDecision, String> {
4010 context.validate()?;
4011 let (registry, failure) = self.work_context_routing_snapshot(context, None).await;
4012 if let Some(failure) = failure {
4013 return Err(failure.summary);
4014 }
4015 let tracker = self.outcome_tracker.read().await;
4016 Ok(self
4017 .adaptive_router
4018 .route_with(crate::adaptive_router::RouteRequest {
4019 intent: intent.as_ref(),
4020 ..crate::adaptive_router::RouteRequest::new(prompt, ®istry, &tracker)
4021 }))
4022 }
4023
4024 pub fn route(&self, prompt: &str) -> RoutingDecision {
4026 self.router.route_generate(prompt, &self.registry)
4027 }
4028
4029 pub fn estimated_tokens(
4040 &self,
4041 req: &GenerateRequest,
4042 model_id: Option<&str>,
4043 ) -> (usize, usize, bool) {
4044 let prompt_tokens = remote::estimate_tokens(&req.prompt);
4045 let context_tokens = req
4046 .context
4047 .as_ref()
4048 .map(|c| remote::estimate_tokens(c))
4049 .unwrap_or(0);
4050 let tools_tokens = req
4051 .tools
4052 .as_ref()
4053 .map(|t| remote::estimate_tokens(&serde_json::to_string(t).unwrap_or_default()))
4054 .unwrap_or(0);
4055 let media_tokens = media_tokens::request_media_and_history_tokens(
4056 req.images.as_deref(),
4057 req.messages.as_deref(),
4058 );
4059 let total_input = prompt_tokens + context_tokens + tools_tokens + media_tokens;
4060
4061 let context_window = match model_id {
4070 Some(id) => {
4071 let routing_registry = self.catalog_registry_snapshot();
4072 routing_registry
4073 .get(id)
4074 .or_else(|| routing_registry.find_by_name(id))
4075 .map(|s| s.context_length)
4076 .unwrap_or(0)
4077 }
4078 None => 0,
4079 };
4080
4081 let fits = context_window == 0 || (total_input + req.params.max_tokens) <= context_window;
4082 (total_input, context_window, fits)
4083 }
4084
4085 fn routing_cache_estimates(req: &GenerateRequest, estimated_input: usize) -> (usize, usize) {
4091 let read = req
4092 .params
4093 .estimated_cache_read_input_tokens
4094 .min(estimated_input);
4095 let write = req
4096 .params
4097 .estimated_cache_write_input_tokens
4098 .min(estimated_input.saturating_sub(read));
4099 (read, write)
4100 }
4101
4102 pub fn model_context_window(&self, model_id: &str) -> usize {
4108 let routing_registry = self.catalog_registry_snapshot();
4109 routing_registry
4110 .get(model_id)
4111 .or_else(|| routing_registry.find_by_name(model_id))
4112 .map(|s| s.context_length)
4113 .unwrap_or(0)
4114 }
4115
4116 pub async fn generate_tracked(
4131 &self,
4132 req: GenerateRequest,
4133 ) -> Result<InferenceResult, InferenceError> {
4134 let mut ignore_retry = |_: InferenceRetryProgress| {};
4135 self.generate_tracked_with_retry_observer(req, &mut ignore_retry)
4136 .await
4137 }
4138
4139 pub async fn generate_tracked_with_retry_observer(
4143 &self,
4144 req: GenerateRequest,
4145 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4146 ) -> Result<InferenceResult, InferenceError> {
4147 let mut ignore_exclusion = |_: RouterExclusion| {};
4148 self.generate_tracked_with_observers(req, retry_observer, &mut ignore_exclusion)
4149 .await
4150 }
4151
4152 pub async fn generate_tracked_with_observers(
4156 &self,
4157 req: GenerateRequest,
4158 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4159 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4160 ) -> Result<InferenceResult, InferenceError> {
4161 self.with_pinned_registry(self.generate_tracked_with_observers_unpinned(
4163 req,
4164 retry_observer,
4165 exclusion_observer,
4166 ))
4167 .await
4168 }
4169
4170 async fn generate_tracked_with_observers_unpinned(
4171 &self,
4172 req: GenerateRequest,
4173 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4174 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4175 ) -> Result<InferenceResult, InferenceError> {
4176 crate::offload::ensure_not_controlled_terminated()?;
4177 let catalog_snapshot = self
4178 .catalog_snapshot()
4179 .map_err(InferenceError::InferenceFailed)?;
4180 let recover = matches!(req.params.thinking, ThinkingMode::Auto);
4181 let mut result = self
4182 .generate_tracked_inner(
4183 req.clone(),
4184 &catalog_snapshot,
4185 retry_observer,
4186 exclusion_observer,
4187 )
4188 .await?;
4189
4190 let action = classify_empty_pass(
4191 recover,
4192 result.stop_reason.as_deref(),
4193 &result.text,
4194 result.tool_calls.is_empty(),
4195 );
4196 let hit_decode_ceiling = action == EmptyPassAction::FailDecodeCeiling;
4197
4198 if action == EmptyPassAction::RetryWithoutThinking {
4199 result.stop_reason = Some("thinking_truncated".to_string());
4200 let mut retry = req;
4201 retry.params.thinking = ThinkingMode::Off;
4202 crate::offload::ensure_not_controlled_terminated()?;
4203 match self
4204 .generate_tracked_inner(
4205 retry,
4206 &catalog_snapshot,
4207 retry_observer,
4208 exclusion_observer,
4209 )
4210 .await
4211 {
4212 Ok(mut recovered) => {
4213 if !recovered.text.trim().is_empty() || !recovered.tool_calls.is_empty() {
4214 recovered.stop_reason = Some("thinking_recovered".to_string());
4215 return Ok(recovered);
4216 }
4217 }
4218 Err(error @ InferenceError::ControlledTermination) => return Err(error),
4219 Err(_) => {}
4220 }
4221 }
4222
4223 if hit_decode_ceiling && result.text.trim().is_empty() && result.tool_calls.is_empty() {
4229 return Err(InferenceError::InferenceFailed(format!(
4230 "local generation hit its {}s wall-clock ceiling before producing any output. \
4231 The `local prefill starting` / `local decode in progress` log lines show where \
4232 the time went — a large prompt can spend most of it on prefill. Try a smaller \
4233 model, shorten the prompt, check for another process contending for the GPU, \
4234 or raise the ceiling with CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables it).",
4235 local_decode_timeout().map_or(0, |t| t.as_secs())
4236 )));
4237 }
4238
4239 Ok(result)
4240 }
4241
4242 #[instrument(
4257 name = "inference.generate",
4258 skip_all,
4259 fields(
4260 model = tracing::field::Empty,
4261 max_tokens = req.params.max_tokens,
4262 prompt_tokens = tracing::field::Empty,
4263 completion_tokens = tracing::field::Empty,
4264 latency_ms = tracing::field::Empty,
4265 ttft_ms = tracing::field::Empty,
4269 gen_ai.operation.name = "chat",
4270 gen_ai.request.model = tracing::field::Empty,
4271 gen_ai.response.model = tracing::field::Empty,
4272 gen_ai.request.max_tokens = req.params.max_tokens,
4273 gen_ai.request.temperature = req.params.temperature,
4274 gen_ai.usage.input_tokens = tracing::field::Empty,
4275 gen_ai.usage.output_tokens = tracing::field::Empty,
4276 )
4277 )]
4278 async fn generate_tracked_inner(
4279 &self,
4280 mut req: GenerateRequest,
4281 catalog_snapshot: &CatalogSnapshot,
4282 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4283 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4284 ) -> Result<InferenceResult, InferenceError> {
4285 self.refuse_system_one_generate(&req)?;
4286 crate::offload::ensure_not_controlled_terminated()?;
4287 validate_expected_catalog_revision(&req, catalog_snapshot)?;
4288 if let Some(requested) = req.model.as_deref() {
4292 tracing::Span::current().record("gen_ai.request.model", requested);
4293 }
4294 let requested_model_id = exact_pinned_model_id(&req).map(str::to_string);
4295 if let Some(model_id) = requested_model_id.as_ref() {
4296 req.model = Some(model_id.clone());
4297 req.params.strict_model = true;
4298 }
4299 let start = Instant::now();
4300 let has_requested_route = req.model.is_some();
4301 let (routing_registry, initial_route_credential_failure) =
4302 if let Some(context) = &req.work_context {
4303 self.work_context_routing_snapshot(context, req.model.as_deref())
4304 .await
4305 } else if has_requested_route {
4306 (self.catalog_registry_snapshot(), None)
4307 } else {
4308 self.routing_registry_snapshot_with_credential_failure()
4309 .await
4310 };
4311 if let Some(requested) = requested_model_id.as_deref() {
4312 if routing_registry.get(requested).is_none() {
4313 return Err(InferenceError::ModelNotFound(requested.to_string()));
4314 }
4315 } else if let Some(requested) = req.model.as_deref() {
4316 if routing_registry
4317 .get(requested)
4318 .or_else(|| routing_registry.find_by_name(requested))
4319 .is_none()
4320 {
4321 return Err(InferenceError::ModelNotFound(requested.to_string()));
4322 }
4323 }
4324
4325 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
4327 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
4335 let (estimated_cache_read, estimated_cache_write) =
4336 Self::routing_cache_estimates(&req, estimated_input);
4337 let tracker_read = self.outcome_tracker.read().await;
4338 let has_tools = Self::request_has_tools(&req);
4339 let has_vision = Self::request_needs_vision(&req);
4340 let preferred_model = self
4341 .preferred_model_for_capability(ModelCapability::Generate)
4342 .map(str::to_string);
4343 let exclude_set = self
4344 .adaptive_router
4345 .build_exclude_set(req.intent.as_ref(), &routing_registry);
4346 let unpinned_override = self
4347 .lane_pin_for(&req, &routing_registry)
4348 .or(preferred_model)
4349 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model));
4350 let decision = match req.model.clone().or(unpinned_override) {
4351 Some(m) => {
4352 let ctx_len = routing_registry
4353 .get(&m)
4354 .or_else(|| routing_registry.find_by_name(&m))
4355 .map(|s| s.context_length)
4356 .unwrap_or(0);
4357 AdaptiveRoutingDecision {
4358 model_id: m.clone(),
4359 model_name: m.clone(),
4360 task: InferenceTask::Generate,
4361 complexity: TaskComplexity::assess(&req.prompt),
4362 reason: "explicit model".into(),
4363 strategy: RoutingStrategy::Explicit,
4364 predicted_quality: 0.5,
4365 fallbacks: vec![],
4366 context_length: ctx_len,
4367 needs_compaction: ctx_len > 0 && estimated_footprint > ctx_len,
4368 candidates: vec![],
4369 }
4370 }
4371 None => self.adaptive_router.route_with_exclusion_observer(
4372 crate::adaptive_router::RouteRequest {
4373 estimated_total_tokens: estimated_footprint,
4374 estimated_input_tokens: estimated_input,
4375 estimated_output_tokens: req.params.max_tokens,
4376 estimated_cache_read_tokens: estimated_cache_read,
4377 estimated_cache_write_tokens: estimated_cache_write,
4378 has_tools,
4379 has_vision,
4380 workload: req.params.workload,
4381 intent: req.intent.as_ref(),
4382 ..crate::adaptive_router::RouteRequest::new(
4383 &req.prompt,
4384 &routing_registry,
4385 &tracker_read,
4386 )
4387 },
4388 exclusion_observer,
4389 ),
4390 };
4391 drop(tracker_read);
4392
4393 if decision.model_id.is_empty() {
4394 let excluded_models = req
4395 .intent
4396 .as_ref()
4397 .map(|hint| hint.exclude_models.join(", "))
4398 .unwrap_or_default();
4399 return Err(InferenceError::NoEligibleModel { excluded_models });
4400 }
4401
4402 if decision.needs_compaction {
4403 tracing::info!(
4404 model = %decision.model_name,
4405 prompt_tokens = estimated_input,
4406 context_window = decision.context_length,
4407 "prompt exceeds model context window — compaction or truncation needed"
4408 );
4409 }
4410
4411 debug!(
4416 model = %decision.model_name,
4417 strategy = ?decision.strategy,
4418 reason = %decision.reason,
4419 "adaptive-routed generate request"
4420 );
4421
4422 let mut req = req;
4425
4426 if let Some(schema) = routing_registry
4432 .get(&decision.model_id)
4433 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4434 {
4435 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
4436 }
4437
4438 let is_code_intent = is_explicit_code_intent(req.intent.as_ref());
4447 let is_complex = matches!(decision.complexity, TaskComplexity::Complex);
4448 if req.params.budget_tokens == 0 && (is_code_intent || is_complex) {
4449 let supports_thinking = routing_registry
4453 .get(&decision.model_id)
4454 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4455 .map(|s| {
4456 s.supported_params
4457 .contains(&schema::GenerateParam::ExtendedThinking)
4458 })
4459 .unwrap_or(false);
4460 if let Some(budget) =
4461 auto_thinking_budget(is_code_intent, is_complex, supports_thinking)
4462 {
4463 req.params.budget_tokens = budget;
4464 tracing::info!(
4465 model = %decision.model_name,
4466 budget,
4467 code_intent = is_code_intent,
4468 "auto-enabled extended thinking"
4469 );
4470 }
4471 }
4472
4473 let mut models_to_try = vec![decision.model_id.clone()];
4475 models_to_try.extend(decision.fallbacks.iter().cloned());
4476
4477 let chain_has_local = models_to_try.iter().any(|m| {
4492 routing_registry
4493 .get(m)
4494 .or_else(|| routing_registry.find_by_name(m))
4495 .map(|s| s.is_local())
4496 .unwrap_or(false)
4497 });
4498 let mut local_last_resort_id = None;
4499 if should_append_local_last_resort(chain_has_local, req.params.strict_model) {
4500 if let Some(local) = self.first_installed_local_model(has_tools) {
4504 tracing::info!(
4505 local_model = %local,
4506 needs_tools = has_tools,
4507 "appended on-device model as last-resort fallback (chain was remote-only)"
4508 );
4509 models_to_try.push(local.clone());
4510 local_last_resort_id = Some(local);
4511 }
4512 } else if !chain_has_local && req.params.strict_model {
4513 tracing::info!(
4514 model = %decision.model_id,
4515 "strict_model set — not degrading to on-device; a remote failure will surface as an error"
4516 );
4517 }
4518
4519 let mut last_error = None;
4520 let mut auth_dead_lane: Option<String> = None;
4526 let mut route_credential_failure = if chain_includes_parslee_route(
4536 |m| {
4537 routing_registry
4538 .get(m)
4539 .or_else(|| routing_registry.find_by_name(m))
4540 },
4541 &models_to_try,
4542 ) {
4543 initial_route_credential_failure
4544 } else {
4545 None
4546 };
4547 let mut fallback_hops: Vec<FallbackFrom> = Vec::new();
4551
4552 let mut candidate_queue: std::collections::VecDeque<String> =
4559 models_to_try.iter().cloned().collect();
4560 let mut is_primary_attempt = true;
4561 while let Some(candidate_owned) = candidate_queue.pop_front() {
4562 crate::offload::ensure_not_controlled_terminated()?;
4563 let was_primary = is_primary_attempt;
4564 is_primary_attempt = false;
4565 let candidate_id = &candidate_owned;
4566 #[allow(unused_mut)]
4569 let mut schema = routing_registry
4570 .get(candidate_id)
4571 .or_else(|| routing_registry.find_by_name(candidate_id))
4572 .cloned();
4573
4574 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4582 if requested_model_id.is_none() {
4583 if let Some(ref s) = schema {
4584 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
4585 tracing::info!(
4586 from = %s.id, to = %mlx_equiv.id,
4587 "redirecting GGUF model to MLX equivalent on Apple Silicon"
4588 );
4589 schema = Some(mlx_equiv.clone());
4590 }
4591 }
4592 }
4593
4594 let candidate_name = schema
4599 .as_ref()
4600 .map(|s| s.name.clone())
4601 .unwrap_or_else(|| candidate_id.clone());
4602
4603 if has_tools
4614 && schema
4615 .as_ref()
4616 .map(|s| !s.has_capability(ModelCapability::ToolUse))
4617 .unwrap_or(false)
4618 {
4619 let backend = schema
4620 .as_ref()
4621 .map(|s| if s.is_local() { "local" } else { "remote" })
4622 .unwrap_or("unknown");
4623 tracing::warn!(
4624 model = %candidate_id,
4625 backend,
4626 "tools requested but resolved model lacks ToolUse capability — skipping candidate"
4627 );
4628 let unsupported = InferenceError::UnsupportedMode {
4629 mode: "tool_use",
4630 backend,
4631 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
4632 };
4633 record_fallback_from(&mut fallback_hops, &candidate_name, &unsupported);
4639 last_error = Some(unsupported);
4640 continue;
4641 }
4642
4643 let resolved_id = schema
4654 .as_ref()
4655 .map(|s| s.id.clone())
4656 .unwrap_or_else(|| candidate_id.clone());
4657 let reported_model_used = if requested_model_id.is_some() {
4658 resolved_id.clone()
4659 } else {
4660 candidate_name.clone()
4661 };
4662 validate_expected_catalog_row(&req, catalog_snapshot, &resolved_id)?;
4663
4664 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4670 if requested_model_id.is_some()
4671 && schema
4672 .as_ref()
4673 .is_some_and(|schema| matches!(schema.source, ModelSource::Local { .. }))
4674 {
4675 let pinned = InferenceError::InferenceFailed(format!(
4676 "exact model_id `{resolved_id}` names a GGUF/Candle row, but Candle is disabled on Apple Silicon; MLX-equivalent substitution is disabled for exact pins"
4677 ));
4678 record_fallback_from(&mut fallback_hops, &candidate_name, &pinned);
4679 last_error = Some(pinned);
4680 continue;
4681 }
4682
4683 let is_remote = schema
4684 .as_ref()
4685 .map(|s| s.is_remote() || s.is_vllm_mlx())
4686 .unwrap_or(false);
4687 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
4688 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
4689
4690 let attempt_trace = {
4695 let mut tracker = self.outcome_tracker.write().await;
4696 tracker.record_start(&resolved_id, decision.task, &decision.reason)
4697 };
4698
4699 if is_delegated {
4705 let runner = match runner::current_inference_runner() {
4706 Some(r) => r,
4707 None => {
4708 let msg = "model declares ModelSource::Delegated but no inference runner is registered";
4709 self.outcome_tracker.write().await.record_not_model_fault(
4712 &attempt_trace,
4713 msg,
4714 outcome::UnattributedReason::Capability,
4715 );
4716 let unregistered = InferenceError::InferenceFailed(msg.into());
4717 record_fallback_from(&mut fallback_hops, &candidate_name, &unregistered);
4718 last_error = Some(unregistered);
4719 continue;
4720 }
4721 };
4722 let (tx, mut rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
4723 let emitter = runner::EventEmitter::new(tx);
4724 let runner_req = req.clone();
4725 let runner_handle = AbortOnDropTask(Some(tokio::spawn(async move {
4726 runner.run(runner_req, emitter).await
4727 })));
4728 let mut accumulator = stream::StreamAccumulator::default();
4729 while let Some(evt) = rx.recv().await {
4730 accumulator.push(&evt);
4731 }
4732 let (acc_text, acc_tool_calls, acc_usage, acc_stop_reason) =
4761 accumulator.finish_with_usage();
4762 match runner_handle.join().await {
4763 Ok(Ok(runner_result)) => {
4764 let elapsed = start.elapsed().as_millis() as u64;
4765 let acc_text = if acc_text.trim().is_empty() {
4766 runner_result.text
4767 } else {
4768 acc_text
4769 };
4770 let acc_tool_calls = if acc_tool_calls.is_empty() {
4771 runner_result.tool_calls
4772 } else {
4773 acc_tool_calls
4774 };
4775 let est_out = acc_text.split_whitespace().count();
4782 {
4783 let mut tracker = self.outcome_tracker.write().await;
4784 tracker.record_complete(
4785 &attempt_trace,
4786 elapsed,
4787 estimated_input,
4788 est_out,
4789 );
4790 }
4791 let local_last_resort = report_local_last_resort_served(
4792 local_last_resort_id.as_deref(),
4793 candidate_id,
4794 &resolved_id,
4795 );
4796 record_generation_span(candidate_name.as_str(), elapsed, None, None);
4803 return Ok(InferenceResult {
4804 text: acc_text,
4805 tool_calls: acc_tool_calls,
4806 bounding_boxes: vec![],
4807 trace_id: attempt_trace,
4808 model_used: reported_model_used,
4809 model_identity: bound_model_identity(
4810 catalog_snapshot,
4811 requested_model_id.as_deref(),
4812 &resolved_id,
4813 )?,
4814 latency_ms: elapsed,
4815 time_to_first_token_ms: None,
4816 usage: acc_usage,
4819 provider_output_items: vec![],
4820 thinking: vec![],
4823 stop_reason: acc_stop_reason,
4824 auth_fallback_from: auth_dead_lane.clone(),
4825 local_last_resort,
4826 fallback_from: fallback_hops.clone(),
4827 });
4828 }
4829 Ok(Err(e)) => {
4830 {
4831 let mut tracker = self.outcome_tracker.write().await;
4832 match &e {
4833 runner::RunnerError::Declined(_) => tracker.record_not_model_fault(
4836 &attempt_trace,
4837 &e.to_string(),
4838 outcome::UnattributedReason::Capability,
4839 ),
4840 runner::RunnerError::Failed(_) => {
4841 tracker.record_failure(&attempt_trace, &e.to_string())
4842 }
4843 }
4844 }
4845 let failed = InferenceError::InferenceFailed(e.to_string());
4852 record_auth_dead_lane(
4853 &mut auth_dead_lane,
4854 &candidate_name,
4855 &failed.to_string(),
4856 );
4857 record_route_credential_failure(
4858 &mut route_credential_failure,
4859 &candidate_name,
4860 &failed,
4861 was_primary && has_requested_route,
4862 );
4863 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
4864 last_error = Some(failed);
4865 continue;
4866 }
4867 Err(join_err) => {
4868 let msg = format!("runner task panicked: {join_err}");
4869 self.outcome_tracker
4870 .write()
4871 .await
4872 .record_failure(&attempt_trace, &msg);
4873 let failed = InferenceError::InferenceFailed(msg);
4879 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
4880 last_error = Some(failed);
4881 continue;
4882 }
4883 }
4884 }
4885
4886 let has_tools = Self::request_has_tools(&req);
4887
4888 let context = if has_tools
4890 && req.tools.as_ref().is_some_and(|t| {
4891 t.iter().any(|tool| {
4892 tool.get("function")
4893 .and_then(|f| f.get("name"))
4894 .and_then(|n| n.as_str())
4895 == Some("done")
4896 })
4897 }) {
4898 let base = req.context.as_deref().unwrap_or("");
4899 Some(format!(
4900 "{base}\n\nIMPORTANT: When calling the `done` tool, the `result` field MUST contain a DETAILED summary of everything you found and did. This is the ONLY output the user sees. Do NOT just say 'completed' — include specific findings, data, and conclusions."
4901 ))
4902 } else {
4903 req.context.clone()
4904 };
4905
4906 let mut captured_thinking: Vec<crate::tasks::generate::ThinkingBlock> = Vec::new();
4910 let mut captured_provider_output_items: Vec<serde_json::Value> = Vec::new();
4911 let mut local_reservation = if !is_delegated {
4917 match schema
4918 .as_ref()
4919 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
4920 {
4921 Some(local_schema) => {
4922 match self
4923 .reserve_local_request_with_worker_retry(
4924 local_schema,
4925 estimated_footprint,
4926 )
4927 .await
4928 {
4929 Ok(reservation) => Some(reservation),
4930 Err(error) if req.model.is_some() || req.params.strict_model => {
4931 self.outcome_tracker
4932 .write()
4933 .await
4934 .record_capability_rejection(
4935 &attempt_trace,
4936 &error.to_string(),
4937 );
4938 return Err(error);
4939 }
4940 Err(error) => {
4941 tracing::warn!(
4942 model = %local_schema.id,
4943 error = %error,
4944 "adaptive local candidate blocked by resource policy; trying next route"
4945 );
4946 self.outcome_tracker
4947 .write()
4948 .await
4949 .record_capability_rejection(
4950 &attempt_trace,
4951 &error.to_string(),
4952 );
4953 record_fallback_from(&mut fallback_hops, &candidate_name, &error);
4958 last_error = Some(error);
4959 continue;
4960 }
4961 }
4962 }
4963 None => None,
4964 }
4965 } else {
4966 None
4967 };
4968 let result = if is_codex_cli {
4969 let schema_ref = schema
4970 .as_ref()
4971 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
4972 if Self::request_needs_vision(&req)
4973 || Self::request_has_video(&req)
4974 || Self::request_has_audio(&req)
4975 {
4976 Err(InferenceError::UnsupportedMode {
4977 mode: "multimodal-content",
4978 backend: "codex-cli",
4979 reason: "the subscription-backed Codex source is text-only",
4980 })
4981 } else if req.response_format.is_some() {
4982 Err(InferenceError::UnsupportedMode {
4983 mode: "response-format",
4984 backend: "codex-cli",
4985 reason: "the subscription-backed Codex source returns plain text and does not expose provider-enforced schemas",
4986 })
4987 } else {
4988 let model = match &schema_ref.source {
4989 ModelSource::CodexCli { model } => model,
4990 _ => unreachable!("is_codex_cli matched a different source"),
4991 };
4992 crate::backend::codex_cli::generate_request(
4993 model,
4994 &req,
4995 context.as_deref(),
4996 schema_ref.context_length,
4997 )
4998 .await
4999 .map(|(output, calls)| (output.text, calls, Some(output.usage), None, None))
5000 }
5001 } else if is_remote {
5002 let (schema_val, _remote_request_reservation) = match self
5006 .vllm_live_schema(
5007 schema.unwrap(),
5008 local_reservation.take(),
5009 estimated_footprint,
5010 )
5011 .await
5012 {
5013 Ok(pair) => pair,
5014 Err(e) => {
5015 record_dispatch_failure(
5019 &mut *self.outcome_tracker.write().await,
5020 &attempt_trace,
5021 &e,
5022 );
5023 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5024 last_error = Some(e);
5025 continue;
5026 }
5027 };
5028 let _ctx_len = schema_val.context_length;
5029 let temperature = if !schema_val.supported_params.is_empty()
5032 && !schema_val
5033 .supported_params
5034 .contains(&crate::schema::GenerateParam::Temperature)
5035 {
5036 -1.0
5037 } else {
5038 req.params.temperature
5039 };
5040
5041 self.remote_backend
5047 .with_work_context(req.work_context.clone())
5048 .generate_with_tools_multi_observed(
5049 &schema_val,
5050 &req.prompt,
5051 context.as_deref(),
5052 temperature,
5053 req.params.max_tokens,
5054 req.tools.as_deref(),
5055 req.images.as_deref(),
5056 req.messages.as_deref(),
5057 req.params.tool_choice.as_deref(),
5058 req.params.parallel_tool_calls,
5059 req.params.budget_tokens,
5060 req.cache_control,
5061 req.params.cache_ttl,
5062 req.context_stable_prefix.as_deref(),
5063 req.response_format.as_ref(),
5064 retry_observer,
5065 )
5066 .await
5067 .map(|(t, c, thinking, provider_items, u, stop)| {
5073 captured_thinking = thinking;
5074 captured_provider_output_items = provider_items;
5075 (t, c, u, None::<u64>, stop)
5076 })
5077 } else if let Some(offload) = schema
5078 .as_ref()
5079 .filter(|schema| Self::supports_worker_offload(schema))
5080 .and_then(|_| crate::offload::current_local_offload())
5081 {
5082 let mut offload_req = req.clone();
5093 offload_req.model = Some(resolved_id.clone());
5094 let schema_ref = schema
5095 .as_ref()
5096 .ok_or_else(|| InferenceError::ModelNotFound(resolved_id.clone()))?;
5097 let reservation = local_reservation.as_mut().ok_or_else(|| {
5098 InferenceError::InferenceFailed(
5099 "local worker dispatch missing admission reservation".into(),
5100 )
5101 })?;
5102 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_id) {
5103 reservation.bind_allocation_id(&allocation_id);
5109 }
5110 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
5111 match offload.generate_admitted(offload_req, admission).await {
5112 Ok(outcome) => {
5113 Self::reconcile_worker_residency(
5114 offload.as_ref(),
5115 &resolved_id,
5116 &outcome.residency,
5117 outcome.retention,
5118 reservation,
5119 )
5120 .await?;
5121 let ir = outcome.result;
5122 captured_thinking = ir.thinking;
5129 captured_provider_output_items = ir.provider_output_items;
5130 Ok((
5131 ir.text,
5132 ir.tool_calls,
5133 ir.usage,
5134 ir.time_to_first_token_ms,
5135 ir.stop_reason,
5136 ))
5137 }
5138 Err(e) => Err(e),
5139 }
5140 } else {
5141 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5142 {
5143 let schema_ref = schema
5147 .as_ref()
5148 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5149
5150 if schema_ref.is_foundation_models() {
5158 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5159 if Self::request_has_video(&req) || Self::request_has_audio(&req) {
5160 Err(InferenceError::UnsupportedMode {
5161 mode: "multimodal-content",
5162 backend: "foundation-models",
5163 reason: "the FoundationModels bridge exposes text and image \
5164 input — route audio/video to a remote VL model",
5165 })
5166 } else if has_images {
5167 let images: Vec<String> = req
5173 .images
5174 .as_ref()
5175 .map(|blocks| {
5176 blocks
5177 .iter()
5178 .filter_map(|b| match b {
5179 crate::ContentBlock::ImageBase64 { data, .. } => {
5180 Some(data.clone())
5181 }
5182 _ => None,
5186 })
5187 .collect()
5188 })
5189 .unwrap_or_default();
5190 if images.is_empty() {
5191 return Err(InferenceError::UnsupportedMode {
5192 mode: "multimodal-content",
5193 backend: "foundation-models",
5194 reason: "FoundationModels takes inline image bytes; \
5195 image URLs must be fetched by the caller first",
5196 });
5197 }
5198 let prompt = req.prompt.clone();
5199 let instructions = context.clone();
5200 let fm_prompt = prompt.clone();
5201 let fm_instructions = instructions.clone();
5202 let fm_ctx_window = schema_ref.context_length as u64;
5203 let max_tokens = req.params.max_tokens as u32;
5204 let temperature = req.params.temperature;
5205 tokio::task::spawn_blocking(move || {
5206 crate::backend::foundation_models::generate_with_images(
5207 &prompt,
5208 instructions.as_deref(),
5209 &images,
5210 max_tokens,
5211 temperature as f32,
5212 )
5213 })
5214 .await
5215 .map_err(|e| {
5216 InferenceError::InferenceFailed(format!(
5217 "FoundationModels task panicked: {e}"
5218 ))
5219 })
5220 .and_then(|r| r)
5221 .map(|text| {
5222 let usage = foundation_models_usage(
5223 fm_instructions.as_deref(),
5224 &fm_prompt,
5225 &text,
5226 fm_ctx_window,
5227 );
5228 (text, vec![], usage, None, None)
5229 })
5230 } else if has_tools {
5231 if req.response_format.is_some() {
5237 tracing::warn!(
5238 "FoundationModels: response_format is ignored when tools \
5239 are present — one turn is either tool-enabled or \
5240 schema-constrained"
5241 );
5242 }
5243 let prompt = req.prompt.clone();
5244 let instructions = context.clone();
5245 let fm_prompt = prompt.clone();
5248 let fm_instructions = instructions.clone();
5249 let fm_ctx_window = schema_ref.context_length as u64;
5250 let tools_defs = req.tools.clone().unwrap_or_default();
5251 let fm_tool_choice = req.params.tool_choice.clone();
5252 let max_tokens = req.params.max_tokens as u32;
5253 let temperature = req.params.temperature;
5254 tokio::task::spawn_blocking(move || {
5255 crate::backend::foundation_models::generate_with_tools(
5256 &prompt,
5257 instructions.as_deref(),
5258 &tools_defs,
5259 fm_tool_choice.as_deref(),
5260 max_tokens,
5261 temperature as f32,
5262 )
5263 })
5264 .await
5265 .map_err(|e| {
5266 InferenceError::InferenceFailed(format!(
5267 "FoundationModels task panicked: {e}"
5268 ))
5269 })
5270 .and_then(|r| r)
5271 .map(|(text, calls)| {
5272 let usage = foundation_models_usage(
5273 fm_instructions.as_deref(),
5274 &fm_prompt,
5275 &text,
5276 fm_ctx_window,
5277 );
5278 (text, calls, usage, None, None)
5279 })
5280 } else if let Some(crate::tasks::generate::ResponseFormat::JsonSchema {
5281 schema,
5282 ..
5283 }) = &req.response_format
5284 {
5285 let prompt = req.prompt.clone();
5289 let instructions = context.clone();
5290 let fm_prompt = prompt.clone();
5293 let fm_instructions = instructions.clone();
5294 let fm_ctx_window = schema_ref.context_length as u64;
5295 let schema_val = schema.clone();
5296 let max_tokens = req.params.max_tokens as u32;
5297 let temperature = req.params.temperature;
5298 tokio::task::spawn_blocking(move || {
5299 crate::backend::foundation_models::generate_structured(
5300 &prompt,
5301 instructions.as_deref(),
5302 &schema_val,
5303 max_tokens,
5304 temperature as f32,
5305 )
5306 })
5307 .await
5308 .map_err(|e| {
5309 InferenceError::InferenceFailed(format!(
5310 "FoundationModels task panicked: {e}"
5311 ))
5312 })
5313 .and_then(|r| r)
5314 .map(|text| {
5315 let usage = foundation_models_usage(
5316 fm_instructions.as_deref(),
5317 &fm_prompt,
5318 &text,
5319 fm_ctx_window,
5320 );
5321 (text, vec![], usage, None, None)
5322 })
5323 } else {
5324 let instructions = if matches!(
5328 req.response_format,
5329 Some(crate::tasks::generate::ResponseFormat::JsonObject)
5330 ) {
5331 tracing::warn!(
5332 "FoundationModels: JsonObject response_format has no native \
5333 constrained mode — enforcing via instruction injection"
5334 );
5335 let base = context.clone().unwrap_or_default();
5336 Some(format!(
5337 "{base}\n\nRespond with a single valid JSON object and \
5338 nothing else."
5339 ))
5340 } else {
5341 context.clone()
5342 };
5343 let prompt = req.prompt.clone();
5344 let fm_prompt = prompt.clone();
5347 let fm_instructions = instructions.clone();
5348 let fm_ctx_window = schema_ref.context_length as u64;
5349 let max_tokens = req.params.max_tokens as u32;
5350 let temperature = req.params.temperature;
5351 tokio::task::spawn_blocking(move || {
5352 crate::backend::foundation_models::generate(
5353 &prompt,
5354 instructions.as_deref(),
5355 max_tokens,
5356 temperature as f32,
5357 )
5358 })
5359 .await
5360 .map_err(|e| {
5361 InferenceError::InferenceFailed(format!(
5362 "FoundationModels task panicked: {e}"
5363 ))
5364 })
5365 .and_then(|r| r)
5366 .map(|text| {
5367 let usage = foundation_models_usage(
5368 fm_instructions.as_deref(),
5369 &fm_prompt,
5370 &text,
5371 fm_ctx_window,
5372 );
5373 (text, vec![], usage, None, None)
5374 })
5375 }
5376 } else if !schema_ref.is_mlx() {
5377 Err(InferenceError::InferenceFailed(format!(
5378 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
5379 schema_ref.id
5380 )))
5381 } else if schema_ref.tags.iter().any(|t| t == "mlx-vlm-cli") {
5382 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5394 if !has_images {
5395 return Err(InferenceError::UnsupportedMode {
5396 mode: "text-only-on-mlx-vlm-id",
5397 backend: "mlx-vlm-cli",
5398 reason: "the `mlx-vlm/...` model IDs route exclusively \
5399 through the mlx-vlm CLI for image inference. \
5400 For text-only generation, route to a Qwen3 \
5401 text model (`mlx/qwen3-4b:4bit` etc.) — the \
5402 CLI shell-out has higher latency than the \
5403 in-process MLX text tower.",
5404 });
5405 }
5406 let vlm_status = crate::backend::mlx_vlm_cli::runtime_status();
5407 if !vlm_status.is_available() {
5408 return Err(InferenceError::InferenceFailed(vlm_status.user_message()));
5409 }
5410 let model_dir =
5411 self.unified_registry().ensure_local(&schema_ref.id).await?;
5412 let reservation = local_reservation
5413 .as_mut()
5414 .expect("local VLM branch has admission reservation");
5415 Self::reconcile_transient_local_allocation(
5416 reservation,
5417 backend_cache::estimate_model_size(&model_dir),
5418 )?;
5419 let detached_lease = reservation.detached_lease();
5420 let repo = match &schema_ref.source {
5421 crate::schema::ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
5422 _ => {
5423 return Err(InferenceError::InferenceFailed(format!(
5424 "model '{}' is tagged mlx-vlm-cli but its \
5425 source isn't ModelSource::Mlx — registry bug",
5426 schema_ref.id
5427 )));
5428 }
5429 };
5430 let imgs = req.images.clone().unwrap_or_default();
5431 let temp = req.params.temperature;
5432 let max_t = req.params.max_tokens;
5433 let prompt = req.prompt.clone();
5434 let (text, cli_usage) = run_admitted_blocking(detached_lease, move || {
5435 crate::backend::mlx_vlm_cli::generate(
5436 &repo, &prompt, &imgs, temp, max_t,
5437 )
5438 })
5439 .await
5440 .map_err(|e| {
5441 InferenceError::InferenceFailed(format!(
5442 "mlx_vlm CLI task panicked: {e}"
5443 ))
5444 })??;
5445 let bounding_boxes = parse_boxes(&text);
5446 let latency_ms = start.elapsed().as_millis() as u64;
5447 let usage = cli_usage.map(|u| TokenUsage {
5457 prompt_tokens: u.prompt_tokens,
5458 completion_tokens: u.completion_tokens,
5459 total_tokens: u.prompt_tokens + u.completion_tokens,
5460 context_window: schema_ref.context_length as u64,
5461 ..Default::default()
5463 });
5464 {
5465 let (in_tokens, out_tokens) = match &usage {
5471 Some(u) => (u.prompt_tokens as usize, u.completion_tokens as usize),
5472 None => (estimated_input, text.split_whitespace().count()),
5473 };
5474 let mut tracker = self.outcome_tracker.write().await;
5475 tracker.record_complete(
5476 &attempt_trace,
5477 latency_ms,
5478 in_tokens,
5479 out_tokens,
5480 );
5481 }
5482 let local_last_resort = report_local_last_resort_served(
5483 local_last_resort_id.as_deref(),
5484 candidate_id,
5485 &resolved_id,
5486 );
5487 record_generation_span(
5492 candidate_name.as_str(),
5493 latency_ms,
5494 None,
5495 usage.as_ref(),
5496 );
5497 return Ok(InferenceResult {
5498 text,
5499 tool_calls: vec![],
5500 bounding_boxes,
5501 trace_id: attempt_trace,
5502 model_used: schema_ref.id.clone(),
5503 model_identity: bound_model_identity(
5504 catalog_snapshot,
5505 requested_model_id.as_deref(),
5506 &resolved_id,
5507 )?,
5508 latency_ms,
5509 time_to_first_token_ms: None,
5510 usage,
5511 provider_output_items: Vec::new(),
5512 thinking: Vec::new(), stop_reason: None,
5514 auth_fallback_from: auth_dead_lane.clone(),
5515 local_last_resort,
5516 fallback_from: fallback_hops.clone(),
5517 });
5518 } else if !Self::supports_native_mlx(schema_ref) {
5519 if req.images.as_ref().is_some_and(|i| !i.is_empty())
5529 || Self::request_has_video(&req)
5530 || Self::request_has_audio(&req)
5531 {
5532 return Err(InferenceError::UnsupportedMode {
5533 mode: "multimodal-content-block",
5534 backend: "native-mlx-local",
5535 reason: "this in-process MLX backend is text-only; route \
5536 image/video/audio understanding to a vLLM-MLX or remote \
5537 multimodal model",
5538 });
5539 }
5540 self.generate_local(
5551 req.clone(),
5552 &schema_ref.id,
5553 local_reservation
5554 .as_mut()
5555 .expect("local branch has admission reservation"),
5556 )
5557 .await
5558 } else {
5559 if Self::request_has_video(&req) {
5565 return Err(InferenceError::UnsupportedMode {
5566 mode: "video-content-block",
5567 backend: "native-mlx-qwen25vl",
5568 reason: "Qwen2.5-VL video understanding is on the request surface \
5569 but the video-tokenization path (frame sampling + merger) \
5570 is not yet wired; route to a remote VL provider for now",
5571 });
5572 }
5573 if Self::request_has_audio(&req) {
5574 return Err(InferenceError::UnsupportedMode {
5575 mode: "audio-content-block",
5576 backend: "native-mlx-qwen25vl",
5577 reason: "audio understanding is on the request surface (Gemma 4 \
5578 E2B/E4B and Gemini accept it) but the native MLX path \
5579 for this model does not — route to Gemini or Gemma-4",
5580 });
5581 }
5582 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5583 if has_images {
5584 let (handle, _retention) = self
5604 .ensure_text_backend(
5605 schema_ref,
5606 local_reservation
5607 .as_mut()
5608 .expect("local branch has admission reservation"),
5609 )
5610 .await?;
5611 let can_do_vision = {
5612 let guard = handle.lock().map_err(|_| {
5613 InferenceError::InferenceFailed(
5614 "MLX backend mutex poisoned".into(),
5615 )
5616 })?;
5617 guard.supports_capability(crate::schema::ModelCapability::Vision)
5618 };
5619 if !can_do_vision {
5620 return Err(InferenceError::UnsupportedMode {
5630 mode: "image-content-block",
5631 backend: "native-mlx-text",
5632 reason: "this MLX backend is a plain Qwen3 text tower. \
5633 For local image inference, route to \
5634 `mlx-vlm/qwen3-vl-2b:bf16` or another `mlx-vlm/...` \
5635 catalog ID so CAR shells out to `mlx_vlm.generate`. \
5636 Alternatives: a local vLLM-MLX VLM server, or a \
5637 remote VL model. (#115)",
5638 });
5639 }
5640 }
5641 self.generate_mlx(
5642 render_for_local_backend(req.clone()),
5643 &schema_ref.id,
5644 local_reservation
5645 .as_mut()
5646 .expect("local branch has admission reservation"),
5647 )
5648 .await
5649 .map(|(text, usage, ttft, stop)| (text, vec![], usage, ttft, stop))
5650 }
5651 }
5652
5653 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
5654 {
5655 let schema_ref = schema
5656 .as_ref()
5657 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5658 match self
5659 .ensure_backend(
5660 schema_ref,
5661 local_reservation
5662 .as_mut()
5663 .expect("local branch has admission reservation"),
5664 )
5665 .await
5666 {
5667 Ok(()) => {
5668 let mut write = self.backend.write().await;
5669 let backend = write.get_mut(&schema_ref.id).ok_or_else(|| {
5670 InferenceError::InferenceFailed(format!(
5671 "candle backend missing after ensure_backend for {}",
5672 schema_ref.id
5673 ))
5674 })?;
5675 let ctx_window = backend.context_length().unwrap_or(0) as u64;
5681 tasks::generate::generate(
5682 backend,
5683 render_for_local_backend(req.clone()),
5684 )
5685 .await
5686 .map(
5687 |(text, ttft, prompt_tokens, completion_tokens)| {
5688 let usage = TokenUsage {
5689 prompt_tokens: prompt_tokens as u64,
5690 completion_tokens: completion_tokens as u64,
5691 total_tokens: (prompt_tokens + completion_tokens) as u64,
5692 context_window: ctx_window,
5693 ..Default::default()
5696 };
5697 (text, vec![], Some(usage), ttft, None)
5698 },
5699 )
5700 }
5701 Err(e) => Err(e),
5702 }
5703 }
5704 };
5705
5706 match result {
5707 Ok((text, mut tool_calls, usage, time_to_first_token_ms, stop_reason)) => {
5708 let text = if !is_remote
5715 && !is_codex_cli
5716 && !is_delegated
5717 && req.tools.is_some()
5718 && tool_calls.is_empty()
5719 {
5720 let (clean, parsed) = tasks::generate::parse_tool_calls(&text);
5721 tool_calls = parsed;
5722 clean
5723 } else {
5724 text
5725 };
5726 let latency_ms = start.elapsed().as_millis() as u64;
5727 let estimated_tokens = usage
5728 .as_ref()
5729 .map(|u| u.completion_tokens as usize)
5730 .unwrap_or_else(|| text.split_whitespace().count());
5731 let input_tokens = usage
5736 .as_ref()
5737 .map(|u| u.prompt_tokens as usize)
5738 .unwrap_or(estimated_input);
5739 let (cache_read, cache_creation) = usage
5744 .as_ref()
5745 .map(|u| {
5746 (
5747 u.cache_read_input_tokens as usize,
5748 u.cache_creation_input_tokens as usize,
5749 )
5750 })
5751 .unwrap_or((0, 0));
5752 {
5753 let mut tracker = self.outcome_tracker.write().await;
5754 tracker.record_complete_cached(
5755 &attempt_trace,
5756 latency_ms,
5757 input_tokens,
5758 estimated_tokens,
5759 cache_read,
5760 cache_creation,
5761 );
5762 }
5763 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
5769 cb.record_success(&resolved_id);
5770 }
5771 self.auto_save_outcomes().await;
5773
5774 record_generation_span(
5776 candidate_name.as_str(),
5777 latency_ms,
5778 time_to_first_token_ms,
5779 usage.as_ref(),
5780 );
5781
5782 let bounding_boxes = tasks::grounding::parse_boxes(&text);
5785 let local_last_resort = report_local_last_resort_served(
5786 local_last_resort_id.as_deref(),
5787 candidate_id,
5788 &resolved_id,
5789 );
5790 return Ok(InferenceResult {
5791 text,
5792 tool_calls,
5793 bounding_boxes,
5794 trace_id: attempt_trace,
5795 model_used: reported_model_used,
5796 model_identity: bound_model_identity(
5797 catalog_snapshot,
5798 requested_model_id.as_deref(),
5799 &resolved_id,
5800 )?,
5801 latency_ms,
5802 time_to_first_token_ms,
5803 usage,
5804 provider_output_items: captured_provider_output_items,
5805 thinking: captured_thinking,
5806 stop_reason,
5807 auth_fallback_from: auth_dead_lane.clone(),
5808 local_last_resort,
5809 fallback_from: fallback_hops.clone(),
5810 });
5811 }
5812 Err(e) => {
5813 if matches!(e, InferenceError::ControlledTermination) {
5814 self.outcome_tracker.write().await.record_not_model_fault(
5815 &attempt_trace,
5816 &e.to_string(),
5817 outcome::UnattributedReason::Cancelled,
5818 );
5819 return Err(e);
5820 }
5821 tracing::warn!(
5822 model = %candidate_name,
5823 error = %e,
5824 remaining = candidate_queue.len(),
5825 "model failed, trying next fallback immediately"
5826 );
5827 record_auth_dead_lane(&mut auth_dead_lane, &candidate_name, &e.to_string());
5834 record_route_credential_failure(
5835 &mut route_credential_failure,
5836 &candidate_name,
5837 &e,
5838 was_primary && has_requested_route,
5839 );
5840 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5841 {
5846 let mut tracker = self.outcome_tracker.write().await;
5847 record_dispatch_failure(&mut tracker, &attempt_trace, &e);
5848 }
5849 if error_counts_against_circuit_breaker(&e) {
5859 let err_str = e.to_string();
5860 let is_client_error =
5861 err_str.contains("API returned 4") && !err_str.contains("429");
5862 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
5863 cb.record_failure(&resolved_id);
5865 if is_client_error {
5868 cb.record_failure(&resolved_id);
5869 }
5870 }
5871 }
5872 #[cfg(not(all(
5874 target_os = "macos",
5875 target_arch = "aarch64",
5876 not(car_skip_mlx)
5877 )))]
5878 {
5879 let mut write = self.backend.write().await;
5880 if write.remove(&resolved_id).is_some() {
5881 self.local_admission.mark_evicted(&resolved_id);
5882 }
5883 }
5884 if let InferenceError::GatewayUnconfigured { namespace, .. } = &e {
5912 let before = candidate_queue.len();
5913 candidate_queue.retain(|id| !id.starts_with(namespace.as_str()));
5914 let dropped = before - candidate_queue.len();
5915 if dropped > 0 {
5916 tracing::info!(
5917 %namespace,
5918 dropped,
5919 remaining = candidate_queue.len(),
5920 "gateway has no upstream for this namespace; dropping its \
5921 remaining candidates instead of replaying the same rejection"
5922 );
5923 }
5924 }
5925 if error_ends_fallback_chain(&e) {
5926 let dropped = candidate_queue.len();
5927 candidate_queue.clear();
5928 if dropped > 0 {
5929 tracing::info!(
5930 dropped,
5931 "content refused for this request; ending the fallback chain \
5932 rather than answering with a different model"
5933 );
5934 }
5935 }
5936 if let InferenceError::ProviderAccount { provider, .. } = &e {
5937 let mut rest: Vec<String> = candidate_queue.iter().cloned().collect();
5941 let demoted = routing_ext::demote_provider(provider, &mut rest, |id| {
5942 routing_registry
5943 .get(id)
5944 .or_else(|| routing_registry.find_by_name(id))
5945 .map(|s| s.provider.clone())
5946 });
5947 candidate_queue = rest.into();
5948 if demoted > 0 {
5949 tracing::info!(
5950 %provider,
5951 demoted,
5952 remaining = candidate_queue.len(),
5953 "account-level rejection; deferring this provider's \
5954 remaining candidates to the end of the chain"
5955 );
5956 }
5957 }
5958 if was_primary && !candidate_queue.is_empty() {
5959 let err_str = e.to_string();
5960 let status = parse_api_returned_status(&err_str);
5967 if routing_ext::is_provider_transient(status, &err_str) {
5968 let provider_of = |id: &str| {
5973 routing_registry
5974 .get(id)
5975 .or_else(|| routing_registry.find_by_name(id))
5976 .map(|s| s.provider.clone())
5977 };
5978 let primary = provider_of(candidate_id).unwrap_or_default();
5979 let queue_vec: Vec<String> = candidate_queue.iter().cloned().collect();
5980 if let Some(cross) =
5981 routing_ext::first_cross_provider(&primary, &queue_vec, provider_of)
5982 {
5983 let cross = cross.to_string();
5984 if let Some(pos) = candidate_queue.iter().position(|m| *m == cross)
5985 {
5986 if pos > 0 {
5987 if let Some(m) = candidate_queue.remove(pos) {
5988 tracing::info!(
5989 promoted = %m,
5990 "transient provider error on primary; promoting cross-provider fallback"
5991 );
5992 candidate_queue.push_front(m);
5993 }
5994 }
5995 }
5996 }
5997 }
5998 }
5999 last_error = Some(e);
6000 }
6001 }
6002 }
6003
6004 let underlying = last_error.unwrap_or(InferenceError::InferenceFailed(
6006 "no models available".into(),
6007 ));
6008
6009 let e = apply_route_failure_context(underlying, route_credential_failure.as_ref());
6023 self.auto_save_outcomes().await;
6027 Err(e)
6028 }
6029
6030 async fn generate_stream_raw(
6035 &self,
6036 req: GenerateRequest,
6037 ) -> Result<(String, tokio::sync::mpsc::Receiver<stream::StreamEvent>), InferenceError> {
6038 let routing_registry = if let Some(context) = &req.work_context {
6039 self.work_context_routing_snapshot(context, req.model.as_deref())
6040 .await
6041 .0
6042 } else {
6043 self.request_routing_registry_snapshot(req.model.as_deref())
6044 .await
6045 };
6046 if let Some(requested) = req.model.as_deref() {
6047 if routing_registry
6048 .get(requested)
6049 .or_else(|| routing_registry.find_by_name(requested))
6050 .is_none()
6051 {
6052 return Err(InferenceError::ModelNotFound(requested.to_string()));
6053 }
6054 }
6055 let has_tools = Self::request_has_tools(&req);
6056 let has_vision = Self::request_needs_vision(&req);
6057 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6058 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
6059 let (estimated_cache_read, estimated_cache_write) =
6060 Self::routing_cache_estimates(&req, estimated_input);
6061 let preferred_model = self
6062 .preferred_model_for_capability(ModelCapability::Generate)
6063 .map(str::to_string);
6064 let exclude_set = self
6065 .adaptive_router
6066 .build_exclude_set(req.intent.as_ref(), &routing_registry);
6067 let unpinned_override = self
6068 .lane_pin_for(&req, &routing_registry)
6069 .or(preferred_model)
6070 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model));
6071 let decision = match req.model.clone().or(unpinned_override) {
6072 Some(m) => {
6073 let ctx_len = routing_registry
6074 .get(&m)
6075 .or_else(|| routing_registry.find_by_name(&m))
6076 .map(|s| s.context_length)
6077 .unwrap_or(0);
6078 AdaptiveRoutingDecision {
6079 model_id: m.clone(),
6080 model_name: m,
6081 task: InferenceTask::Generate,
6082 complexity: TaskComplexity::assess(&req.prompt),
6083 reason: "explicit model".into(),
6084 strategy: RoutingStrategy::Explicit,
6085 predicted_quality: 0.5,
6086 fallbacks: vec![],
6087 context_length: ctx_len,
6088 needs_compaction: false,
6089 candidates: vec![],
6090 }
6091 }
6092 None => {
6093 let tracker_read = self.outcome_tracker.read().await;
6094 self.adaptive_router
6095 .route_with(crate::adaptive_router::RouteRequest {
6096 estimated_total_tokens: estimated_footprint,
6097 estimated_input_tokens: estimated_input,
6098 estimated_output_tokens: req.params.max_tokens,
6099 estimated_cache_read_tokens: estimated_cache_read,
6100 estimated_cache_write_tokens: estimated_cache_write,
6101 has_tools,
6102 has_vision,
6103 workload: req.params.workload,
6104 intent: req.intent.as_ref(),
6105 ..crate::adaptive_router::RouteRequest::new(
6106 &req.prompt,
6107 &routing_registry,
6108 &tracker_read,
6109 )
6110 })
6111 }
6112 };
6113
6114 if decision.model_id.is_empty() {
6115 let excluded_models = req
6116 .intent
6117 .as_ref()
6118 .map(|hint| hint.exclude_models.join(", "))
6119 .unwrap_or_default();
6120 return Err(InferenceError::NoEligibleModel { excluded_models });
6121 }
6122
6123 #[allow(unused_mut)]
6126 let mut schema = routing_registry
6127 .get(&decision.model_id)
6128 .or_else(|| routing_registry.find_by_name(&decision.model_id))
6129 .cloned();
6130
6131 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6133 if let Some(ref s) = schema {
6134 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
6135 tracing::info!(
6136 from = %s.id, to = %mlx_equiv.id,
6137 "redirecting GGUF model to MLX equivalent on Apple Silicon (stream)"
6138 );
6139 schema = Some(mlx_equiv.clone());
6140 }
6141 }
6142
6143 if has_tools
6149 && schema
6150 .as_ref()
6151 .map(|s| !s.has_capability(ModelCapability::ToolUse))
6152 .unwrap_or(false)
6153 {
6154 let backend = schema
6155 .as_ref()
6156 .map(|s| if s.is_local() { "local" } else { "remote" })
6157 .unwrap_or("unknown");
6158 return Err(InferenceError::UnsupportedMode {
6159 mode: "tool_use",
6160 backend,
6161 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
6162 });
6163 }
6164
6165 let resolved_model_id = schema
6167 .as_ref()
6168 .map(|s| s.id.clone())
6169 .unwrap_or_else(|| decision.model_id.clone());
6170
6171 let mut req = req;
6178 if let Some(schema) = schema.as_ref() {
6179 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
6180 }
6181
6182 let is_remote = schema
6183 .as_ref()
6184 .map(|s| s.is_remote() || s.is_vllm_mlx())
6185 .unwrap_or(false);
6186
6187 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
6188 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
6189
6190 if is_codex_cli {
6191 return Err(InferenceError::UnsupportedMode {
6192 mode: "streaming",
6193 backend: "codex-cli",
6194 reason: "codex exec reports final answer items rather than token deltas; use non-streaming infer/generate_tracked",
6195 });
6196 }
6197
6198 if is_delegated {
6199 let runner = runner::current_inference_runner().ok_or_else(|| {
6203 InferenceError::InferenceFailed(
6204 "model declares ModelSource::Delegated but no inference runner is registered \
6205 (call set_inference_runner / registerInferenceRunner / register_inference_runner)"
6206 .into(),
6207 )
6208 })?;
6209 let (tx, rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6210 let emitter = runner::EventEmitter::new(tx);
6211 let request = req.clone();
6212 tokio::spawn(async move {
6213 if let Err(e) = runner.run(request, emitter).await {
6214 tracing::warn!(error = %e, "delegated inference runner failed");
6215 }
6216 });
6217 return Ok((resolved_model_id, rx));
6218 }
6219
6220 let mut local_reservation = if !is_remote
6222 || schema.as_ref().is_some_and(ModelSchema::is_vllm_mlx)
6223 {
6224 if let Some(local_schema) = schema
6225 .as_ref()
6226 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
6227 {
6228 Some(
6229 self.reserve_local_request_with_worker_retry(local_schema, estimated_footprint)
6230 .await?,
6231 )
6232 } else {
6233 None
6234 }
6235 } else {
6236 None
6237 };
6238
6239 if !is_remote && schema.as_ref().is_some_and(Self::supports_worker_offload) {
6246 if let Some(offload) = crate::offload::current_local_offload() {
6247 let mut offload_req = req.clone();
6248 offload_req.model = Some(resolved_model_id.clone());
6249 let schema_ref = schema
6250 .as_ref()
6251 .ok_or_else(|| InferenceError::ModelNotFound(resolved_model_id.clone()))?;
6252 let reservation = local_reservation.as_mut().ok_or_else(|| {
6253 InferenceError::InferenceFailed(
6254 "local worker stream missing admission reservation".into(),
6255 )
6256 })?;
6257 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_model_id) {
6258 reservation.bind_allocation_id(&allocation_id);
6259 }
6260 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
6261 let offload_stream = offload.stream_admitted(offload_req, admission).await?;
6262 Self::reconcile_worker_residency(
6263 offload.as_ref(),
6264 &resolved_model_id,
6265 &offload_stream.residency,
6266 offload_stream.retention,
6267 reservation,
6268 )
6269 .await?;
6270 let rx = offload_stream.events;
6271 let rx = match local_reservation {
6272 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6273 None => rx,
6274 };
6275 return Ok((resolved_model_id, rx));
6276 }
6277 }
6278
6279 if is_remote {
6280 let mut candidates = vec![schema.unwrap()];
6281 let mut local_fallback_ids = Vec::new();
6282 for fallback_id in &decision.fallbacks {
6283 if let Some(fallback) = routing_registry
6284 .get(fallback_id)
6285 .or_else(|| routing_registry.find_by_name(fallback_id))
6286 {
6287 if (fallback.is_remote() || fallback.is_vllm_mlx())
6288 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6289 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6290 && !candidates
6291 .iter()
6292 .any(|candidate| candidate.id == fallback.id)
6293 {
6294 candidates.push(fallback.clone());
6295 } else if fallback.is_local()
6296 && !fallback.is_vllm_mlx()
6297 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6298 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6299 {
6300 local_fallback_ids.push(fallback.id.clone());
6301 }
6302 }
6303 }
6304 let mut last_error = None;
6305 for candidate in candidates {
6306 let (candidate, mut candidate_reservation) = match self
6309 .vllm_live_schema(candidate, local_reservation.take(), estimated_footprint)
6310 .await
6311 {
6312 Ok(candidate) => candidate,
6313 Err(error) => {
6314 last_error = Some(error);
6315 continue;
6316 }
6317 };
6318 self.remote_backend.register_model_keys(&candidate).await;
6319
6320 let spend_guard = self
6321 .spend_limits
6322 .read()
6323 .unwrap()
6324 .as_ref()
6325 .and_then(|limits| limits.per_request_usd)
6326 .map(|budget| {
6327 let mut prompt_tokens =
6328 routing_ext::MidStreamSpendGuard::estimate_tokens(&req.prompt)
6329 + req
6330 .context
6331 .as_deref()
6332 .map(routing_ext::MidStreamSpendGuard::estimate_tokens)
6333 .unwrap_or(0);
6334 prompt_tokens += media_tokens::request_media_and_history_tokens(
6335 req.images.as_deref(),
6336 req.messages.as_deref(),
6337 ) as u64;
6338 if let Some(tools) = &req.tools {
6339 prompt_tokens += routing_ext::MidStreamSpendGuard::estimate_tokens(
6340 &serde_json::to_string(tools).unwrap_or_default(),
6341 );
6342 }
6343 let prices = candidate.cost.prices_for(prompt_tokens as usize);
6344 let input_price = prices
6345 .input_per_mtok
6346 .map(|c| c / 1_000_000.0)
6347 .unwrap_or(0.0);
6348 let output_price = prices
6349 .output_per_mtok
6350 .map(|c| c / 1_000_000.0)
6351 .unwrap_or(0.0);
6352 routing_ext::MidStreamSpendGuard::new(
6353 Some(budget),
6354 prompt_tokens as f64 * input_price,
6355 input_price,
6356 output_price,
6357 )
6358 });
6359
6360 match self
6361 .remote_backend
6362 .with_work_context(req.work_context.clone())
6363 .generate_stream(
6364 &candidate,
6365 &req.prompt,
6366 req.messages.as_deref(),
6367 req.context.as_deref(),
6368 req.params.temperature,
6369 req.params.max_tokens,
6370 req.tools.as_deref(),
6371 req.images.as_deref(),
6372 req.params.tool_choice.as_deref(),
6373 req.params.parallel_tool_calls,
6374 req.response_format.as_ref(),
6375 spend_guard,
6376 )
6377 .await
6378 {
6379 Ok(receiver) => {
6380 let receiver = Self::hold_optional_reservation_for_stream(
6381 receiver,
6382 candidate_reservation.take(),
6383 );
6384 return Ok((candidate.id, receiver));
6385 }
6386 Err(error) => {
6387 tracing::warn!(
6388 model = %candidate.id,
6389 %error,
6390 "remote stream setup failed; trying routed fallback"
6391 );
6392 last_error = Some(error);
6393 }
6394 }
6395 }
6396
6397 if !req.params.strict_model {
6402 if let Some(local) = self.first_installed_local_model(has_tools) {
6403 let schema = routing_registry
6404 .get(&local)
6405 .or_else(|| routing_registry.find_by_name(&local));
6406 let supports_request = schema.is_some_and(|schema| {
6407 !has_vision || schema.has_capability(ModelCapability::Vision)
6408 });
6409 if supports_request
6410 && !local_fallback_ids.iter().any(|candidate| {
6411 routing_registry
6412 .get(candidate)
6413 .or_else(|| routing_registry.find_by_name(candidate))
6414 .is_some_and(|schema| schema.id == local || schema.name == local)
6415 })
6416 {
6417 local_fallback_ids.push(local);
6418 }
6419 }
6420 for local_id in local_fallback_ids {
6421 let mut fallback_req = req.clone();
6422 fallback_req.model = Some(local_id);
6423 fallback_req.params.strict_model = true;
6424 match Box::pin(self.generate_stream_raw(fallback_req)).await {
6425 Ok(stream) => return Ok(stream),
6426 Err(error) => {
6427 tracing::warn!(%error, "local streaming fallback failed");
6428 last_error = Some(error);
6429 }
6430 }
6431 }
6432 }
6433
6434 Err(last_error.unwrap_or_else(|| {
6435 InferenceError::InferenceFailed(
6436 "no compatible remote streaming model available".to_string(),
6437 )
6438 }))
6439 } else {
6440 let schema =
6441 schema.ok_or_else(|| InferenceError::ModelNotFound(decision.model_id.clone()))?;
6442 let (tx, rx) = tokio::sync::mpsc::channel(64);
6443
6444 #[cfg(any(
6450 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
6451 all(target_os = "ios", target_arch = "aarch64")
6452 ))]
6453 {
6454 if schema.is_foundation_models() {
6455 if Self::request_has_video(&req)
6459 || Self::request_has_audio(&req)
6460 || req.images.as_ref().is_some_and(|imgs| !imgs.is_empty())
6461 {
6462 return Err(InferenceError::UnsupportedMode {
6463 mode: "multimodal-content",
6464 backend: "foundation-models",
6465 reason: "the FoundationModels bridge currently exposes text-only \
6466 generation — route image/audio/video to a remote VL model",
6467 });
6468 }
6469 if let Some(tools_defs) = req.tools.clone().filter(|t| !t.is_empty()) {
6475 let prompt = req.prompt.clone();
6476 let instructions = req.context.clone();
6477 let fm_tool_choice = req.params.tool_choice.clone();
6478 let max_tokens = req.params.max_tokens as u32;
6479 let temperature = req.params.temperature;
6480 tokio::task::spawn_blocking(move || {
6481 match crate::backend::foundation_models::generate_with_tools(
6482 &prompt,
6483 instructions.as_deref(),
6484 &tools_defs,
6485 fm_tool_choice.as_deref(),
6486 max_tokens,
6487 temperature as f32,
6488 ) {
6489 Ok((text, tool_calls)) => {
6490 if !text.is_empty() {
6491 let _ = tx.blocking_send(stream::StreamEvent::TextDelta(
6492 text.clone(),
6493 ));
6494 }
6495 let _ = tx.blocking_send(stream::StreamEvent::Done {
6496 text,
6497 tool_calls,
6498 });
6499 }
6500 Err(e) => {
6501 tracing::warn!(
6502 error = %e,
6503 "FoundationModels tool-enabled stream failed"
6504 );
6505 send_stream_failure_blocking(&tx, &e);
6511 }
6512 }
6513 });
6514 return Ok((resolved_model_id, rx));
6515 }
6516 let prompt = req.prompt.clone();
6517 let instructions = req.context.clone();
6518 let max_tokens = req.params.max_tokens as u32;
6519 let temperature = req.params.temperature;
6520 let tx_clone = tx.clone();
6521 tokio::task::spawn_blocking(move || {
6522 let accum = std::sync::Arc::new(std::sync::Mutex::new(String::new()));
6529 let accum_cb = accum.clone();
6530 let cb = crate::backend::foundation_models::StreamCallback::new(
6531 move |delta: &str| {
6532 if let Ok(mut g) = accum_cb.lock() {
6533 g.push_str(delta);
6534 }
6535 tx_clone
6536 .blocking_send(stream::StreamEvent::TextDelta(
6537 delta.to_string(),
6538 ))
6539 .is_ok()
6540 },
6541 );
6542 let result = crate::backend::foundation_models::stream(
6543 &prompt,
6544 instructions.as_deref(),
6545 max_tokens,
6546 temperature as f32,
6547 cb,
6548 );
6549 let final_text = accum.lock().map(|g| g.clone()).unwrap_or_default();
6550 match &result {
6554 Ok(()) => {
6555 let _ = tx.blocking_send(stream::StreamEvent::Done {
6556 text: final_text,
6557 tool_calls: vec![],
6558 });
6559 }
6560 Err(e) => send_stream_failure_blocking(&tx, e),
6561 }
6562 result
6563 });
6564 return Ok((resolved_model_id, rx));
6565 }
6566 }
6567
6568 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6570 {
6571 if !schema.is_mlx() {
6574 return Err(InferenceError::InferenceFailed(format!(
6575 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
6576 schema.id
6577 )));
6578 }
6579 let (backend, _retention) = self
6580 .ensure_text_backend(
6581 &schema,
6582 local_reservation
6583 .as_mut()
6584 .expect("local stream has admission reservation"),
6585 )
6586 .await?;
6587 let model_id = schema.id.clone();
6588 let cache = Arc::clone(&self.text_backends);
6589 let device_guard = Self::mlx_device_lock().lock_owned().await;
6596 tokio::task::spawn_blocking(move || {
6602 let _device_guard = device_guard;
6603 let _ = Self::stream_local_mlx(backend, cache, model_id, req, tx);
6604 });
6605 let rx = match local_reservation {
6606 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6607 None => rx,
6608 };
6609 Ok((resolved_model_id, rx))
6610 }
6611
6612 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6613 {
6614 self.ensure_backend(
6615 &schema,
6616 local_reservation
6617 .as_mut()
6618 .expect("local stream has admission reservation"),
6619 )
6620 .await?;
6621 let backend = self.backend.clone();
6622 let model_id = schema.id.clone();
6623 tokio::spawn(async move {
6624 let _ = Self::stream_local_candle(backend, model_id, req, tx).await;
6625 });
6626 let rx = match local_reservation {
6627 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6628 None => rx,
6629 };
6630 Ok((resolved_model_id, rx))
6631 }
6632 }
6633 }
6634
6635 pub async fn generate_tracked_stream(
6680 &self,
6681 req: GenerateRequest,
6682 ) -> Result<TrackedStream, InferenceError> {
6683 self.refuse_system_one_generate(&req)?;
6684 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6688 let start = Instant::now();
6689
6690 let (model_id, mut producer_rx) = self.generate_stream_raw(req).await?;
6694
6695 let trace = {
6696 let mut t = self.outcome_tracker.write().await;
6697 t.record_start(&model_id, InferenceTask::Generate, "stream")
6698 };
6699 let trace_for_return = trace.clone();
6703
6704 let (out_tx, out_rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6705 let tracker = Arc::clone(&self.outcome_tracker);
6706 tokio::spawn(async move {
6707 let mut acc = stream::StreamAccumulator::default();
6708 let mut stream_error: Option<String> = None;
6709 let mut attribution: Option<outcome::UnattributedReason> = None;
6710 let mut saw_done = false;
6711 let mut receiver_abandoned = false;
6712 while let Some(evt) = producer_rx.recv().await {
6713 if let stream::StreamEvent::Attribution(reason) = &evt {
6714 attribution = Some(*reason);
6716 continue;
6717 }
6718 if let stream::StreamEvent::Error(message) = &evt {
6719 stream_error = Some(message.clone());
6720 }
6721 if matches!(evt, stream::StreamEvent::Done { .. }) {
6722 saw_done = true;
6723 }
6724 acc.push(&evt);
6725 if out_tx.send(evt).await.is_err() {
6726 receiver_abandoned = true;
6727 break;
6728 }
6729 }
6730 if stream_error.is_none() && !saw_done && !receiver_abandoned {
6731 let error = "stream ended without positive provider completion".to_string();
6732 stream_error = Some(error.clone());
6733 let _ = out_tx.send(stream::StreamEvent::Error(error)).await;
6734 } else if stream_error.is_none() && receiver_abandoned {
6735 stream_error = Some("stream receiver was abandoned before completion".to_string());
6739 attribution = Some(outcome::UnattributedReason::Cancelled);
6740 }
6741 let (text, tool_calls, usage, _stop) = acc.finish_with_usage();
6742 let latency_ms = start.elapsed().as_millis() as u64;
6743 let input_tokens = usage
6744 .as_ref()
6745 .map(|u| u.prompt_tokens as usize)
6746 .unwrap_or(estimated_input);
6747 let mut output_tokens = usage
6752 .as_ref()
6753 .map(|u| u.completion_tokens as usize)
6754 .unwrap_or_else(|| text.split_whitespace().count());
6755 if output_tokens == 0 && !tool_calls.is_empty() {
6756 output_tokens = 1;
6757 }
6758 let (cache_read, cache_creation) = usage
6763 .as_ref()
6764 .map(|u| {
6765 (
6766 u.cache_read_input_tokens as usize,
6767 u.cache_creation_input_tokens as usize,
6768 )
6769 })
6770 .unwrap_or((0, 0));
6771 let mut t = tracker.write().await;
6772 if let Some(error) = stream_error {
6773 let reason = attribution.or_else(|| {
6781 stream::content_refusal_tags(&error)
6782 .map(|_| outcome::UnattributedReason::ContentRefused)
6783 });
6784 t.note_elapsed(&trace, latency_ms);
6785 match reason {
6786 Some(reason) => t.record_not_model_fault(&trace, &error, reason),
6787 None => t.record_failure(&trace, &error),
6788 }
6789 } else {
6790 t.record_complete_cached(
6791 &trace,
6792 latency_ms,
6793 input_tokens,
6794 output_tokens,
6795 cache_read,
6796 cache_creation,
6797 );
6798 }
6799 });
6800
6801 Ok(TrackedStream {
6802 model_used: model_id,
6803 trace_id: trace_for_return,
6804 events: out_rx,
6805 })
6806 }
6807
6808 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6809 async fn stream_local_candle(
6810 backend_lock: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
6811 model_id: String,
6812 req: GenerateRequest,
6813 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
6814 ) -> Result<(), InferenceError> {
6815 let mut write = backend_lock.write().await;
6816 let backend = write.get_mut(&model_id).ok_or_else(|| {
6817 InferenceError::InferenceFailed(format!("backend not initialized for {model_id}"))
6818 })?;
6819 backend.clear_kv_cache();
6820
6821 let formatted = tasks::generate::render_chat_prompt(&req);
6822 let tokens = backend.encode(&formatted)?;
6823 let eos = backend.eos_token_id();
6824 let eos_alt = backend.token_id("<|im_end|>");
6825 let params = &req.params;
6826
6827 if tokens.is_empty() {
6828 let _ = tx
6829 .send(stream::StreamEvent::Done {
6830 text: String::new(),
6831 tool_calls: vec![],
6832 })
6833 .await;
6834 return Ok(());
6835 }
6836
6837 let max_ctx = backend.context_length().unwrap_or(32768);
6838 let headroom = params.max_tokens.min(max_ctx / 4);
6839 let max_prompt = max_ctx.saturating_sub(headroom);
6840 let tokens = if tokens.len() > max_prompt {
6841 tokens[tokens.len() - max_prompt..].to_vec()
6842 } else {
6843 tokens
6844 };
6845
6846 let mut generated = Vec::new();
6847 let logits = backend.forward(&tokens, 0)?;
6848 let mut next_token = tasks::generate::sample_token(&logits, params)?;
6849
6850 for _ in 0..params.max_tokens {
6851 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
6852 break;
6853 }
6854
6855 generated.push(next_token);
6856 let delta = backend.decode(&[next_token])?;
6857 if !delta.is_empty()
6858 && tx
6859 .send(stream::StreamEvent::TextDelta(delta))
6860 .await
6861 .is_err()
6862 {
6863 return Ok(());
6864 }
6865
6866 if !params.stop.is_empty() {
6867 let text_so_far = backend.decode(&generated)?;
6868 if params.stop.iter().any(|s| text_so_far.contains(s)) {
6869 break;
6870 }
6871 }
6872
6873 let pos = tokens.len() + generated.len() - 1;
6874 let logits = backend.forward(&[next_token], pos)?;
6875 next_token = tasks::generate::sample_token(&logits, params)?;
6876 }
6877
6878 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
6879 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
6880 let _ = tx
6884 .send(stream::StreamEvent::Usage {
6885 input_tokens: tokens.len() as u64,
6886 output_tokens: generated.len() as u64,
6887 cache_read_input_tokens: 0,
6888 cache_creation_input_tokens: 0,
6889 })
6890 .await;
6891 let _ = tx
6892 .send(stream::StreamEvent::Done {
6893 text,
6894 tool_calls: vec![],
6895 })
6896 .await;
6897 Ok(())
6898 }
6899
6900 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6904 fn stream_local_mlx(
6905 handle: backend_cache::CachedBackend<backend::SwiftLmBackend>,
6906 cache: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
6907 model_id: String,
6908 req: GenerateRequest,
6909 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
6910 ) -> Result<(), InferenceError> {
6911 let mut guard = handle.lock().map_err(|_| {
6912 InferenceError::InferenceFailed(format!("MLX backend mutex poisoned for {model_id}"))
6913 })?;
6914 let backend: &mut backend::SwiftLmBackend = &mut guard;
6915 backend.clear_kv_cache();
6916
6917 let formatted = tasks::generate::render_chat_prompt(&req);
6918 let tokens = backend.encode(&formatted)?;
6919 let eos = backend.eos_token_id();
6920 let eos_alt = backend.token_id("<|im_end|>");
6921 let params = &req.params;
6922
6923 if tokens.is_empty() {
6924 let _ = tx.blocking_send(stream::StreamEvent::Done {
6925 text: String::new(),
6926 tool_calls: vec![],
6927 });
6928 return Ok(());
6929 }
6930
6931 let max_ctx = backend.context_length();
6932 let headroom = params.max_tokens.min(max_ctx / 4);
6933 let max_prompt = max_ctx.saturating_sub(headroom);
6934 let tokens = if tokens.len() > max_prompt {
6935 tokens[tokens.len() - max_prompt..].to_vec()
6936 } else {
6937 tokens
6938 };
6939
6940 let mut generated = Vec::new();
6941
6942 let started = std::time::Instant::now();
6948 let timeout = local_decode_timeout();
6949 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
6950 let mut last_heartbeat = std::time::Duration::ZERO;
6951 tracing::info!(
6952 prompt_tokens = tokens.len(),
6953 max_tokens = params.max_tokens,
6954 timeout_secs = timeout.map(|t| t.as_secs()),
6955 "local stream prefill starting"
6956 );
6957
6958 let logits = match Self::catch_mlx("stream prefill", || backend.forward(&tokens, 0)) {
6963 Ok(v) => v,
6964 Err(e) => {
6965 cache.invalidate(&model_id);
6966 return Err(e);
6967 }
6968 };
6969 let mut next_token = Self::sample_from_logits(&logits, params)?;
6970
6971 for _ in 0..params.max_tokens {
6972 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
6973 break;
6974 }
6975
6976 generated.push(next_token);
6977 let delta = backend.decode(&[next_token])?;
6978 if !delta.is_empty()
6979 && tx
6980 .blocking_send(stream::StreamEvent::TextDelta(delta))
6981 .is_err()
6982 {
6983 return Ok(());
6984 }
6985
6986 if !params.stop.is_empty() {
6987 let text_so_far = backend.decode(&generated)?;
6988 if params.stop.iter().any(|s| text_so_far.contains(s)) {
6989 break;
6990 }
6991 }
6992
6993 let elapsed = started.elapsed();
6998 if deadline_exceeded(elapsed, timeout) {
6999 tracing::warn!(
7000 elapsed_secs = elapsed.as_secs_f64(),
7001 timeout_secs = timeout.map(|t| t.as_secs()),
7002 completion_tokens = generated.len(),
7003 max_tokens = params.max_tokens,
7004 "local stream hit its wall-clock ceiling and was cut short; \
7005 returning what was streamed so far. Raise or disable it with \
7006 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
7007 );
7008 let _ = tx.blocking_send(stream::StreamEvent::StopReason(
7013 LOCAL_DECODE_TIMEOUT_STOP_REASON.to_string(),
7014 ));
7015 break;
7016 }
7017 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7018 last_heartbeat = elapsed;
7019 tracing::info!(
7020 completion_tokens = generated.len(),
7021 max_tokens = params.max_tokens,
7022 elapsed_secs = elapsed.as_secs_f64(),
7023 tokens_per_sec =
7024 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7025 "local stream in progress"
7026 );
7027 }
7028
7029 let pos = tokens.len() + generated.len() - 1;
7030 let logits =
7031 match Self::catch_mlx("stream forward", || backend.forward(&[next_token], pos)) {
7032 Ok(v) => v,
7033 Err(e) => {
7034 cache.invalidate(&model_id);
7035 return Err(e);
7036 }
7037 };
7038 next_token = Self::sample_from_logits(&logits, params)?;
7039 }
7040
7041 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
7042 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
7043 let _ = tx.blocking_send(stream::StreamEvent::Usage {
7051 input_tokens: tokens.len() as u64,
7052 output_tokens: generated.len() as u64,
7053 cache_read_input_tokens: 0,
7055 cache_creation_input_tokens: 0,
7056 });
7057 let _ = tx.blocking_send(stream::StreamEvent::Done {
7058 text,
7059 tool_calls: vec![],
7060 });
7061 Ok(())
7062 }
7063
7064 pub async fn route_context_snapshot(
7066 &self,
7067 prompt: &str,
7068 workload: RoutingWorkload,
7069 has_tools: bool,
7070 has_vision: bool,
7071 ) -> AdaptiveRoutingDecision {
7072 let routing_registry = self.catalog_registry_snapshot();
7073 let tracker = self.outcome_tracker.read().await;
7074 self.adaptive_router.route_context_aware(
7075 prompt,
7076 0,
7077 &routing_registry,
7078 &tracker,
7079 has_tools,
7080 has_vision,
7081 workload,
7082 )
7083 }
7084
7085 pub async fn generate(&self, req: GenerateRequest) -> Result<String, InferenceError> {
7088 Ok(self.generate_tracked(req).await?.text)
7089 }
7090
7091 pub async fn tokenize(&self, model: &str, text: &str) -> Result<Vec<u32>, InferenceError> {
7103 self.assert_local_for_tokenize(model)?;
7104 let admission_schema = self
7105 .resolve_schema(model)
7106 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7107 .clone();
7108 let mut reservation =
7110 self.reserve_local_request(&admission_schema, text.len().div_ceil(4))?;
7111
7112 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7113 {
7114 let schema = admission_schema;
7115 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7116 let guard = handle.lock().map_err(|_| {
7117 InferenceError::InferenceFailed(format!(
7118 "MLX backend mutex poisoned for {}",
7119 schema.id
7120 ))
7121 })?;
7122 let tokens = guard.tokenize_raw(text)?;
7123 Ok(tokens)
7124 }
7125
7126 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7127 {
7128 self.ensure_backend(&admission_schema, &mut reservation)
7129 .await?;
7130 let read = self.backend.read().await;
7131 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7132 InferenceError::InferenceFailed(
7133 "candle backend missing after ensure_backend".to_string(),
7134 )
7135 })?;
7136 let tokens = backend.tokenize_raw(text)?;
7137 Ok(tokens)
7138 }
7139 }
7140
7141 pub async fn detokenize(&self, model: &str, tokens: &[u32]) -> Result<String, InferenceError> {
7143 self.assert_local_for_tokenize(model)?;
7144 let admission_schema = self
7145 .resolve_schema(model)
7146 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7147 .clone();
7148 let mut reservation = self.reserve_local_request(&admission_schema, tokens.len())?;
7150
7151 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7152 {
7153 let schema = admission_schema;
7154 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7155 let guard = handle.lock().map_err(|_| {
7156 InferenceError::InferenceFailed(format!(
7157 "MLX backend mutex poisoned for {}",
7158 schema.id
7159 ))
7160 })?;
7161 let text = guard.detokenize_raw(tokens)?;
7162 Ok(text)
7163 }
7164
7165 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7166 {
7167 self.ensure_backend(&admission_schema, &mut reservation)
7168 .await?;
7169 let read = self.backend.read().await;
7170 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7171 InferenceError::InferenceFailed(
7172 "candle backend missing after ensure_backend".to_string(),
7173 )
7174 })?;
7175 let text = backend.detokenize_raw(tokens)?;
7176 Ok(text)
7177 }
7178 }
7179
7180 fn assert_local_for_tokenize(&self, model: &str) -> Result<(), InferenceError> {
7184 if let Some(schema) = self.resolve_schema(model) {
7185 if !schema.is_local() {
7186 return Err(InferenceError::UnsupportedMode {
7187 mode: "tokenize/detokenize",
7188 backend: "remote",
7189 reason: "remote provider tokenizer is not exposed by the runtime; \
7190 use a local model (Qwen3 GGUF / MLX) for tokenizer-correctness checks",
7191 });
7192 }
7193 }
7194 Ok(())
7196 }
7197
7198 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7204 fn catch_mlx<F, T>(context: &str, f: F) -> Result<T, InferenceError>
7205 where
7206 F: FnOnce() -> Result<T, InferenceError>,
7207 {
7208 std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|e| {
7209 InferenceError::InferenceFailed(format!("MLX panicked during {context}: {e:?}"))
7210 })?
7211 }
7212
7213 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7223 fn drive_generation(
7224 backend: &mut dyn backend::local::TextDecoder,
7225 prompt: &str,
7226 params: &GenerateParams,
7227 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7228 Self::drive_generation_with_timeout(backend, prompt, params, local_decode_timeout())
7229 }
7230
7231 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7235 fn drive_generation_with_timeout(
7236 backend: &mut dyn backend::local::TextDecoder,
7237 prompt: &str,
7238 params: &GenerateParams,
7239 timeout: Option<std::time::Duration>,
7240 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7241 use backend::local::{DriveError, LocalGeneration};
7242
7243 let start = std::time::Instant::now();
7244
7245 let tokens = backend.encode(prompt).map_err(DriveError::Recoverable)?;
7246 let eos_ids = backend.eos_ids();
7247
7248 if tokens.is_empty() {
7249 backend.clear_kv_cache();
7250 return Ok(LocalGeneration {
7251 text: String::new(),
7252 ttft_ms: None,
7253 stop_reason: None,
7254 prompt_tokens: 0,
7255 completion_tokens: 0,
7256 });
7257 }
7258
7259 let max_ctx = backend.context_length();
7261 let headroom = params.max_tokens.min(max_ctx / 4);
7262 let max_prompt = max_ctx.saturating_sub(headroom);
7263 let tokens = if tokens.len() > max_prompt {
7264 tokens[tokens.len() - max_prompt..].to_vec()
7265 } else {
7266 tokens
7267 };
7268
7269 let mut generated = Vec::new();
7270
7271 tracing::info!(
7278 prompt_tokens = tokens.len(),
7279 max_tokens = params.max_tokens,
7280 timeout_secs = timeout.map(|t| t.as_secs()),
7281 "local prefill starting"
7282 );
7283
7284 let offset = backend.begin_prompt(&tokens);
7289 let logits = Self::catch_mlx("prefill", || backend.forward(&tokens[offset..], offset))
7290 .map_err(DriveError::BackendCorrupted)?;
7291 let mut next_token =
7292 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7293 let ttft_ms = Some(start.elapsed().as_millis() as u64);
7294
7295 tracing::info!(
7300 prompt_tokens = tokens.len(),
7301 max_tokens = params.max_tokens,
7302 prefill_ms = ttft_ms,
7303 timeout_secs = timeout.map(|t| t.as_secs()),
7304 "local decode starting"
7305 );
7306
7307 let mut natural_stop = false;
7310 let mut timed_out = false;
7311 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
7312 let mut last_heartbeat = std::time::Duration::ZERO;
7313 for _ in 0..params.max_tokens {
7314 if eos_ids.contains(&next_token) {
7315 natural_stop = true;
7316 break;
7317 }
7318
7319 generated.push(next_token);
7320
7321 if !params.stop.is_empty() {
7322 let text_so_far = backend
7323 .decode(&generated)
7324 .map_err(DriveError::Recoverable)?;
7325 if params.stop.iter().any(|s| text_so_far.contains(s)) {
7326 natural_stop = true;
7327 break;
7328 }
7329 }
7330
7331 let elapsed = start.elapsed();
7332 if deadline_exceeded(elapsed, timeout) {
7333 timed_out = true;
7334 break;
7335 }
7336 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7337 last_heartbeat = elapsed;
7338 tracing::info!(
7339 completion_tokens = generated.len(),
7340 max_tokens = params.max_tokens,
7341 elapsed_secs = elapsed.as_secs_f64(),
7342 tokens_per_sec =
7343 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7344 "local decode in progress"
7345 );
7346 }
7347
7348 let pos = tokens.len() + generated.len() - 1;
7349 let logits = Self::catch_mlx("forward", || backend.forward(&[next_token], pos))
7350 .map_err(DriveError::BackendCorrupted)?;
7351 next_token =
7352 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7353 }
7354
7355 let elapsed = start.elapsed();
7356 let tokens_per_sec = generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON);
7357 if timed_out {
7358 tracing::warn!(
7359 elapsed_secs = elapsed.as_secs_f64(),
7360 timeout_secs = timeout.map(|t| t.as_secs()),
7361 completion_tokens = generated.len(),
7362 max_tokens = params.max_tokens,
7363 tokens_per_sec,
7364 "local decode hit its wall-clock ceiling and was cut short; \
7365 returning the partial response. Raise or disable it with \
7366 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
7367 );
7368 } else {
7369 tracing::info!(
7370 elapsed_secs = elapsed.as_secs_f64(),
7371 completion_tokens = generated.len(),
7372 max_tokens = params.max_tokens,
7373 tokens_per_sec,
7374 natural_stop,
7375 "local decode finished"
7376 );
7377 }
7378
7379 let decoded = backend
7380 .decode(&generated)
7381 .map_err(DriveError::Recoverable)?;
7382 let text = tasks::generate::truncate_at_stop(&decoded, ¶ms.stop);
7383 let stop_reason = Some(
7384 if timed_out {
7385 LOCAL_DECODE_TIMEOUT_STOP_REASON
7386 } else if natural_stop {
7387 "stop"
7388 } else {
7389 "length"
7390 }
7391 .to_string(),
7392 );
7393 Ok(LocalGeneration {
7394 text: tasks::generate::strip_thinking(&text, params.thinking),
7395 ttft_ms,
7396 stop_reason,
7397 prompt_tokens: tokens.len(),
7398 completion_tokens: generated.len(),
7399 })
7400 }
7401
7402 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7405 async fn generate_mlx(
7415 &self,
7416 req: GenerateRequest,
7417 model_id: &str,
7418 reservation: &mut resource_policy::LocalLoadReservation,
7419 ) -> Result<(String, Option<TokenUsage>, Option<u64>, Option<String>), InferenceError> {
7420 let schema = self
7421 .unified_registry()
7422 .get(model_id)
7423 .cloned()
7424 .ok_or_else(|| {
7425 InferenceError::InferenceFailed(format!(
7426 "generate_mlx: unknown schema id {model_id}"
7427 ))
7428 })?;
7429 let (handle, _retention) = self.ensure_text_backend(&schema, reservation).await?;
7430 let formatted = tasks::generate::render_chat_prompt(&req);
7437 let params = req.params.clone();
7438 let model_id = model_id.to_string();
7439 let cache = self.text_backends.clone();
7440 let device_guard = Self::mlx_device_lock().lock_owned().await;
7441 let detached_lease = reservation.detached_lease();
7442
7443 run_admitted_blocking(detached_lease, move || {
7449 let _device_guard = device_guard;
7450 let mut guard = handle.lock().map_err(|_| {
7451 InferenceError::InferenceFailed(format!(
7452 "MLX backend mutex poisoned for {model_id}"
7453 ))
7454 })?;
7455 let backend: &mut backend::SwiftLmBackend = &mut guard;
7456 let ctx_window = backend.context_length() as u64;
7457 match Self::drive_generation(backend, &formatted, ¶ms) {
7458 Ok(gen) => {
7459 let usage = TokenUsage {
7460 prompt_tokens: gen.prompt_tokens as u64,
7461 completion_tokens: gen.completion_tokens as u64,
7462 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7463 context_window: ctx_window,
7464 ..Default::default()
7465 };
7466 Ok((gen.text, Some(usage), gen.ttft_ms, gen.stop_reason))
7467 }
7468 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
7469 Err(backend::local::DriveError::BackendCorrupted(error)) => {
7470 drop(guard);
7471 cache.invalidate(&model_id);
7472 Err(error)
7473 }
7474 }
7475 })
7476 .await
7477 .map_err(|error| {
7478 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
7479 })?
7480 }
7481
7482 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7487 async fn generate_local(
7488 &self,
7489 req: GenerateRequest,
7490 model_id: &str,
7491 reservation: &mut resource_policy::LocalLoadReservation,
7492 ) -> Result<
7493 (
7494 String,
7495 Vec<tasks::generate::ToolCall>,
7496 Option<TokenUsage>,
7497 Option<u64>,
7498 Option<String>,
7499 ),
7500 InferenceError,
7501 > {
7502 let schema = self
7503 .unified_registry()
7504 .get(model_id)
7505 .cloned()
7506 .ok_or_else(|| {
7507 InferenceError::InferenceFailed(format!(
7508 "generate_local: unknown schema id {model_id}"
7509 ))
7510 })?;
7511 let (handle, _retention) = self.ensure_local_backend(&schema, reservation).await?;
7512 let params = req.params.clone();
7513 let model_id = model_id.to_string();
7514 let cache = self.local_backends.clone();
7515 let device_guard = Self::mlx_device_lock().lock_owned().await;
7516 let detached_lease = reservation.detached_lease();
7517
7518 run_admitted_blocking(detached_lease, move || {
7519 let _device_guard = device_guard;
7520 let mut guard = handle.lock().map_err(|_| {
7521 InferenceError::InferenceFailed(format!(
7522 "local backend mutex poisoned for {model_id}"
7523 ))
7524 })?;
7525 let formatted = guard.render_prompt(&req)?;
7526 let backend: &mut dyn backend::local::TextDecoder = &mut **guard;
7527 let outcome = Self::drive_generation(backend, &formatted, ¶ms);
7528 match outcome {
7529 Ok(gen) => {
7530 let (clean, tool_calls) = guard.parse_tool_calls(&gen.text);
7531 let usage = TokenUsage {
7532 prompt_tokens: gen.prompt_tokens as u64,
7533 completion_tokens: gen.completion_tokens as u64,
7534 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7535 context_window: guard.context_length() as u64,
7536 ..Default::default()
7537 };
7538 Ok((clean, tool_calls, Some(usage), gen.ttft_ms, gen.stop_reason))
7539 }
7540 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
7541 Err(backend::local::DriveError::BackendCorrupted(error)) => {
7542 drop(guard);
7543 cache.invalidate(&model_id);
7544 Err(error)
7545 }
7546 }
7547 })
7548 .await
7549 .map_err(|error| {
7550 InferenceError::InferenceFailed(format!("native local task panicked: {error}"))
7551 })?
7552 }
7553
7554 #[allow(dead_code)] fn apply_top_k_top_p(probs: &mut [f32], top_k: usize, top_p: f64) {
7562 if top_k > 0 && top_k < probs.len() {
7564 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
7565 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
7566 let allowed: std::collections::HashSet<usize> =
7567 indexed[..top_k].iter().map(|(i, _)| *i).collect();
7568 for (i, p) in probs.iter_mut().enumerate() {
7569 if !allowed.contains(&i) {
7570 *p = 0.0;
7571 }
7572 }
7573 let sum: f32 = probs.iter().sum();
7574 if sum > 0.0 {
7575 for p in probs.iter_mut() {
7576 *p /= sum;
7577 }
7578 }
7579 }
7580
7581 if top_p < 1.0 {
7583 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
7584 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
7585 let mut cumsum = 0.0f32;
7586 let mut cutoff_idx = indexed.len();
7587 for (i, &(_, p)) in indexed.iter().enumerate() {
7588 cumsum += p;
7589 if cumsum > top_p as f32 {
7590 cutoff_idx = i + 1;
7591 break;
7592 }
7593 }
7594 let allowed: std::collections::HashSet<usize> =
7595 indexed[..cutoff_idx].iter().map(|(i, _)| *i).collect();
7596 for (i, p) in probs.iter_mut().enumerate() {
7597 if !allowed.contains(&i) {
7598 *p = 0.0;
7599 }
7600 }
7601 let sum: f32 = probs.iter().sum();
7602 if sum > 0.0 {
7603 for p in probs.iter_mut() {
7604 *p /= sum;
7605 }
7606 }
7607 }
7608 }
7609
7610 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7612 fn sample_from_logits(logits: &[f32], params: &GenerateParams) -> Result<u32, InferenceError> {
7613 if params.temperature <= 0.0 {
7614 let (idx, _) = logits
7616 .iter()
7617 .enumerate()
7618 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
7619 .ok_or_else(|| InferenceError::InferenceFailed("empty logits".into()))?;
7620 return Ok(idx as u32);
7621 }
7622
7623 let temp = params.temperature as f32;
7625 let max_logit = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
7626 let mut probs: Vec<f32> = logits
7627 .iter()
7628 .map(|&l| ((l - max_logit) / temp).exp())
7629 .collect();
7630 let sum: f32 = probs.iter().sum();
7631 for p in &mut probs {
7632 *p /= sum;
7633 }
7634
7635 Self::apply_top_k_top_p(&mut probs, params.top_k, params.top_p);
7641
7642 use rand::Rng;
7644 let mut rng = rand::rng();
7645 let r: f32 = rng.random();
7646 let mut cumsum = 0.0;
7647 for (i, &p) in probs.iter().enumerate() {
7648 cumsum += p;
7649 if cumsum >= r {
7650 return Ok(i as u32);
7651 }
7652 }
7653 Ok((probs.len() - 1) as u32)
7654 }
7655
7656 pub async fn embed(&self, req: EmbedRequest) -> Result<Vec<Vec<f32>>, InferenceError> {
7659 let instruction = req
7660 .instruction
7661 .as_deref()
7662 .unwrap_or("Retrieve relevant memory facts");
7663 let embedding_model = self
7664 .preferred_model_for_capability(ModelCapability::Embed)
7665 .unwrap_or(&self.config.embedding_model);
7666 let admission_schema = self
7667 .resolve_schema(embedding_model)
7668 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model.to_string()))?
7669 .clone();
7670 let estimated_tokens = req.texts.iter().map(|text| text.len().div_ceil(4)).sum();
7671 let mut reservation = self.reserve_local_request(&admission_schema, estimated_tokens)?;
7673
7674 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7675 {
7676 let model_id = self.ensure_mlx_embedding_backend().await?;
7677 let schema = self
7678 .unified_registry()
7679 .get(&model_id)
7680 .cloned()
7681 .ok_or_else(|| {
7682 InferenceError::InferenceFailed(format!("embed: unknown schema id {model_id}"))
7683 })?;
7684 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7685 let _device_guard = Self::mlx_device_lock().lock_owned().await;
7689 let mut guard = handle.lock().map_err(|_| {
7690 InferenceError::InferenceFailed(format!(
7691 "MLX embedding backend mutex poisoned for {model_id}"
7692 ))
7693 })?;
7694 let backend: &mut backend::SwiftLmBackend = &mut guard;
7695
7696 let mut results = Vec::with_capacity(req.texts.len());
7697 for text in &req.texts {
7698 let embedding = if req.is_query {
7699 backend.embed_query(text, instruction)?
7700 } else {
7701 backend.embed_one(text)?
7702 };
7703 results.push(embedding);
7704 }
7705 Ok(results)
7706 }
7707
7708 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7709 {
7710 self.ensure_embedding_backend(&mut reservation).await?;
7711 let mut write = self.embedding_backend.write().await;
7712 let backend = write.as_mut().unwrap();
7713
7714 let mut results = Vec::with_capacity(req.texts.len());
7715 for text in &req.texts {
7716 let embedding = if req.is_query {
7717 backend.embed_query(text, instruction)?
7718 } else {
7719 backend.embed_one(text)?
7720 };
7721 results.push(embedding);
7722 }
7723 Ok(results)
7724 }
7725 }
7726
7727 pub async fn rerank(&self, req: RerankRequest) -> Result<RerankResult, InferenceError> {
7761 if req.documents.is_empty() {
7762 return Ok(RerankResult {
7763 ranked: Vec::new(),
7764 model_used: None,
7765 });
7766 }
7767
7768 let model_name = match req.model.clone() {
7769 Some(m) => m,
7770 None => self
7771 .preferred_model_for_capability(ModelCapability::Rerank)
7772 .map(str::to_string)
7773 .ok_or_else(|| {
7774 InferenceError::InferenceFailed(
7775 "no reranker model available — pull a Qwen3-Reranker model first".into(),
7776 )
7777 })?,
7778 };
7779
7780 let schema = self.resolve_schema_name_first(&model_name).ok_or_else(|| {
7781 InferenceError::InferenceFailed(format!("rerank: unknown reranker model {model_name}"))
7782 })?;
7783 if !schema.has_capability(ModelCapability::Rerank) {
7784 return Err(InferenceError::InferenceFailed(format!(
7785 "model {} does not declare the Rerank capability",
7786 schema.name
7787 )));
7788 }
7789
7790 let instruction = req.instruction.as_deref().unwrap_or(
7791 "Given a web search query, retrieve relevant passages that answer the query",
7792 );
7793
7794 let mut scored: Vec<RerankedDocument> = Vec::with_capacity(req.documents.len());
7795 for (idx, doc) in req.documents.iter().enumerate() {
7796 let prompt = rerank_prompt(instruction, &req.query, doc);
7797 let gen_req = GenerateRequest {
7798 prompt,
7799 model: Some(schema.id.clone()),
7800 params: tasks::generate::GenerateParams {
7801 temperature: 0.0,
7802 max_tokens: 3,
7806 thinking: tasks::generate::ThinkingMode::Off,
7807 ..Default::default()
7808 },
7809 context: None,
7810 context_stable_prefix: None,
7811 tools: None,
7812 images: None,
7813 messages: None,
7814 cache_control: false,
7815 response_format: None,
7816 intent: None,
7817 client_ref: None,
7818 expected_row_digest: None,
7819 expected_catalog_revision: None,
7820 caller: None,
7821 work_context: None,
7822 };
7823 let out = self.generate(gen_req).await?;
7824 let score = score_from_rerank_output(&out, &schema.name);
7825 scored.push(RerankedDocument {
7826 index: idx,
7827 score,
7828 document: doc.clone(),
7829 });
7830 }
7831
7832 scored.sort_by(|a, b| {
7835 b.score
7836 .partial_cmp(&a.score)
7837 .unwrap_or(std::cmp::Ordering::Equal)
7838 .then_with(|| a.index.cmp(&b.index))
7839 });
7840 if let Some(n) = req.top_n {
7841 scored.truncate(n);
7842 }
7843
7844 Ok(RerankResult {
7845 ranked: scored,
7846 model_used: Some(schema.name),
7847 })
7848 }
7849
7850 pub async fn ground(&self, req: GroundRequest) -> Result<GroundResult, InferenceError> {
7860 let model_name = match req.model.clone() {
7861 Some(m) => m,
7862 None => self
7863 .preferred_model_for_capability(ModelCapability::Grounding)
7864 .map(str::to_string)
7865 .ok_or_else(|| {
7866 InferenceError::InferenceFailed(
7867 "no grounding-capable model available — pull a Qwen2.5-VL model first"
7868 .into(),
7869 )
7870 })?,
7871 };
7872
7873 let gen_req = GenerateRequest {
7874 prompt: req.prompt.clone(),
7875 model: Some(model_name),
7876 params: GenerateParams::default(),
7877 context: None,
7878 context_stable_prefix: None,
7879 tools: None,
7880 images: Some(vec![req.image.clone()]),
7881 messages: None,
7882 cache_control: false,
7883 response_format: None,
7884 intent: None,
7885 client_ref: None,
7886 expected_row_digest: None,
7887 expected_catalog_revision: None,
7888 caller: None,
7889 work_context: None,
7890 };
7891 let result = self.generate_tracked(gen_req).await?;
7892 Ok(GroundResult {
7893 boxes: result.bounding_boxes,
7894 raw_text: result.text,
7895 model_used: Some(result.model_used),
7896 })
7897 }
7898
7899 pub async fn classify(
7902 &self,
7903 req: ClassifyRequest,
7904 ) -> Result<Vec<ClassifyResult>, InferenceError> {
7905 let registry = if let Some(context) = &req.work_context {
7906 Arc::new(
7907 self.work_context_routing_snapshot(context, req.model.as_deref())
7908 .await
7909 .0,
7910 )
7911 } else {
7912 self.unified_registry()
7913 };
7914 let model = match req.model.clone().or_else(|| {
7915 self.preferred_model_for_capability(ModelCapability::Classify)
7916 .map(str::to_string)
7917 }) {
7918 Some(m) => m,
7919 None => match default_system_one_classifier(registry.all()) {
7923 Some(m) => {
7924 debug!(model = %m, "classify defaulted to an available System One model");
7925 m
7926 }
7927 None => {
7928 let m = self.router.route_small(&self.registry);
7929 debug!(model = %m, "auto-routed classify request");
7930 m
7931 }
7932 },
7933 };
7934 let schema = self
7935 .resolve_schema(&model)
7936 .ok_or_else(|| InferenceError::ModelNotFound(model.clone()))?
7937 .clone();
7938
7939 if let ModelSource::Proprietary { protocol, .. } = &schema.source {
7940 if protocol.wire == ProprietaryWire::SystemOne {
7941 return self.classify_via_system_one(req, &schema).await;
7942 }
7943 }
7944 if !schema.is_local() {
7945 return self.classify_via_generate(req, &schema.id).await;
7946 }
7947
7948 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7951 {
7952 return self.classify_via_generate(req, &model).await;
7953 }
7954
7955 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7956 {
7957 let mut reservation =
7959 self.reserve_local_request(&schema, req.text.len().div_ceil(4))?;
7960 self.ensure_backend(&schema, &mut reservation).await?;
7961 let mut write = self.backend.write().await;
7962 let backend = write.get_mut(&schema.id).ok_or_else(|| {
7963 InferenceError::InferenceFailed(format!(
7964 "candle backend missing after ensure_backend for {}",
7965 schema.id
7966 ))
7967 })?;
7968 let result = tasks::classify::classify(backend, req).await?;
7969 Ok(result)
7970 }
7971 }
7972
7973 fn refuse_system_one_generate(&self, req: &GenerateRequest) -> Result<(), InferenceError> {
7982 let Some(model) = req.model.as_deref() else {
7983 return Ok(());
7984 };
7985 let model = model.strip_prefix(EXACT_MODEL_ID_PREFIX).unwrap_or(model);
7986 let schema = self.resolve_schema(model);
7987 match schema.as_ref().map(|s| &s.source) {
7988 Some(ModelSource::Proprietary { protocol, .. })
7989 if protocol.wire == ProprietaryWire::SystemOne =>
7990 {
7991 Err(InferenceError::UnsupportedMode {
7992 mode: "generate",
7993 backend: "parslee-system-one",
7994 reason: "a System One model returns typed decisions, not text; use classify",
7995 })
7996 }
7997 _ => Ok(()),
7998 }
7999 }
8000
8001 pub async fn option_probabilities(
8020 &self,
8021 model: &str,
8022 prompt: &str,
8023 options: &[String],
8024 ) -> Result<OptionProbabilities, InferenceError> {
8025 if options.is_empty() {
8026 return Err(InferenceError::InvalidClassifyLabels(
8027 "no options given".into(),
8028 ));
8029 }
8030 let schema = self
8031 .resolve_schema(model)
8032 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
8033 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8034 {
8035 let _ = (schema, prompt);
8036 Err(InferenceError::UnsupportedMode {
8037 mode: "option-logits",
8038 backend: "local",
8039 reason: "option probabilities read logits from the native MLX text backend, \
8040 which is Apple Silicon only",
8041 })
8042 }
8043 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8044 {
8045 if !schema.is_local() || !schema.family.to_lowercase().starts_with("qwen") {
8046 return Err(InferenceError::UnsupportedMode {
8047 mode: "option-logits",
8048 backend: "local",
8049 reason: "option probabilities need a local Qwen-family model: the prompt is \
8050 rendered with Qwen's chat template",
8051 });
8052 }
8053 let req = GenerateRequest {
8054 prompt: prompt.to_string(),
8055 model: Some(schema.id.clone()),
8056 params: tasks::generate::GenerateParams {
8057 thinking: tasks::generate::ThinkingMode::Off,
8058 ..Default::default()
8059 },
8060 ..Default::default()
8061 };
8062 let formatted = tasks::generate::render_chat_prompt(&req);
8063 let mut reservation =
8064 self.reserve_local_request(&schema, formatted.len().div_ceil(4))?;
8065 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
8066 let options = options.to_vec();
8067 let model_id = schema.id.clone();
8068 let device_guard = Self::mlx_device_lock().lock_owned().await;
8069 let detached_lease = reservation.detached_lease();
8070 run_admitted_blocking(detached_lease, move || {
8071 let _device_guard = device_guard;
8072 let mut guard = handle.lock().map_err(|_| {
8073 InferenceError::InferenceFailed(format!(
8074 "MLX backend mutex poisoned for {model_id}"
8075 ))
8076 })?;
8077 let backend: &mut backend::SwiftLmBackend = &mut guard;
8078 let result = tasks::classify::score_options(backend, &formatted, &options);
8079 backend.clear_kv_cache();
8080 result
8081 })
8082 .await
8083 .map_err(|error| {
8084 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
8085 })?
8086 }
8087 }
8088
8089 async fn classify_via_system_one(
8094 &self,
8095 req: ClassifyRequest,
8096 schema: &ModelSchema,
8097 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8098 let body = tasks::classify::system_one_request_body(&schema.name, &req.text, &req.labels)?;
8099 let trace = {
8102 let mut tracker = self.outcome_tracker.write().await;
8103 tracker.record_start(
8104 &schema.id,
8105 InferenceTask::Classify,
8106 "explicit system-one model",
8107 )
8108 };
8109 let start = Instant::now();
8110 let result = match self
8111 .remote_backend
8112 .with_work_context(req.work_context.clone())
8113 .parslee_system_one(schema, &body)
8114 .await
8115 {
8116 Ok(response) => tasks::classify::system_one_results(&response, &req.labels)
8117 .map(|results| (results, response)),
8118 Err(error) => Err(error),
8119 };
8120 let mut tracker = self.outcome_tracker.write().await;
8121 match result {
8122 Ok((results, response)) => {
8123 let usage = |key: &str| {
8124 response
8125 .pointer(&format!("/usage/{key}"))
8126 .and_then(|t| t.as_u64())
8127 .unwrap_or(0) as usize
8128 };
8129 tracker.record_complete(
8132 &trace,
8133 start.elapsed().as_millis() as u64,
8134 usage("input_tokens"),
8135 usage("output_tokens").max(1),
8136 );
8137 Ok(results)
8138 }
8139 Err(error) => {
8140 record_dispatch_failure(&mut tracker, &trace, &error);
8141 Err(error)
8142 }
8143 }
8144 }
8145
8146 async fn classify_via_generate(
8147 &self,
8148 req: ClassifyRequest,
8149 model: &str,
8150 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8151 let labels_str = req
8152 .labels
8153 .iter()
8154 .enumerate()
8155 .map(|(i, l)| format!("{}. {}", i + 1, l))
8156 .collect::<Vec<_>>()
8157 .join("\n");
8158
8159 let prompt = format!(
8160 "Classify the following text into one of these categories:\n\
8161 {labels_str}\n\n\
8162 Text: {}\n\n\
8163 Respond with ONLY the category name, nothing else.",
8164 req.text
8165 );
8166
8167 let gen_req = GenerateRequest {
8168 prompt,
8169 model: Some(model.to_string()),
8170 params: tasks::generate::GenerateParams {
8171 temperature: 0.0,
8172 max_tokens: 32,
8173 thinking: tasks::generate::ThinkingMode::Off,
8176 ..Default::default()
8177 },
8178 context: None,
8179 context_stable_prefix: None,
8180 tools: None,
8181 images: None,
8182 messages: None,
8183 cache_control: false,
8184 response_format: None,
8185 intent: None,
8186 client_ref: None,
8187 expected_row_digest: None,
8188 expected_catalog_revision: None,
8189 caller: None,
8190 work_context: req.work_context.clone(),
8191 };
8192
8193 let response = self.generate(gen_req).await?;
8194 tasks::classify::score_reply(&response, &req.labels)
8195 }
8196
8197 pub async fn transcribe(
8199 &self,
8200 req: TranscribeRequest,
8201 ) -> Result<TranscribeResult, InferenceError> {
8202 let candidates =
8203 self.speech_candidates(ModelCapability::SpeechToText, req.model.as_deref())?;
8204 let mut last_error = None;
8205 let mut local_resource_block = None;
8206
8207 for schema in candidates {
8208 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8210 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8211 SpeechCandidateAdmission::SkipBlocked(error) => {
8212 local_resource_block = Some(error.to_string());
8213 last_error = Some(error);
8214 continue;
8215 }
8216 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8217 };
8218 let result = match &schema.source {
8219 ModelSource::Mlx { .. } => {
8220 self.transcribe_local_mlx(&schema, &req, reservation.as_mut())
8221 .await
8222 }
8223 ModelSource::WhisperCpp { model } => {
8224 self.transcribe_whisper(&schema, model, &req, reservation.as_mut())
8225 .await
8226 }
8227 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8228 self.transcribe_elevenlabs(&schema, &req).await
8229 }
8230 _ => Err(InferenceError::InferenceFailed(format!(
8231 "speech-to-text not implemented for model source: {}",
8232 schema.id
8233 ))),
8234 };
8235
8236 match result {
8237 Ok(mut result) => {
8238 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8239 if let Some(reason) = local_resource_block.take() {
8240 result.routing_explanation = Some(format!(
8241 "Local speech recognition was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8242 ));
8243 }
8244 }
8245 return Ok(result);
8246 }
8247 Err(err) if req.model.is_some() => return Err(err),
8248 Err(err) => last_error = Some(err),
8249 }
8250 }
8251
8252 Err(last_error.unwrap_or_else(|| {
8253 InferenceError::InferenceFailed("no speech-to-text models available".into())
8254 }))
8255 }
8256
8257 pub async fn synthesize(
8259 &self,
8260 req: SynthesizeRequest,
8261 ) -> Result<SynthesizeResult, InferenceError> {
8262 let candidates =
8263 self.speech_candidates(ModelCapability::TextToSpeech, req.model.as_deref())?;
8264 let mut last_error = None;
8265 let mut local_resource_block = None;
8266
8267 for schema in candidates {
8268 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8270 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8271 SpeechCandidateAdmission::SkipBlocked(error) => {
8272 local_resource_block = Some(error.to_string());
8273 last_error = Some(error);
8274 continue;
8275 }
8276 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8277 };
8278 let result = match &schema.source {
8279 ModelSource::Mlx { .. } => {
8280 self.synthesize_local_mlx(&schema, &req, reservation.as_mut())
8281 .await
8282 }
8283 ModelSource::WindowsSpeech {} => {
8284 self.synthesize_windows_speech(&schema, &req).await
8285 }
8286 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8287 self.synthesize_elevenlabs(&schema, &req).await
8288 }
8289 _ => Err(InferenceError::InferenceFailed(format!(
8290 "text-to-speech not implemented for model source: {}",
8291 schema.id
8292 ))),
8293 };
8294
8295 match result {
8296 Ok(mut result) => {
8297 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8298 if let Some(reason) = local_resource_block.take() {
8299 result.routing_explanation = Some(format!(
8300 "Local speech synthesis was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8301 ));
8302 }
8303 }
8304 return Ok(result);
8305 }
8306 Err(err) if req.model.is_some() => return Err(err),
8307 Err(err) => last_error = Some(err),
8308 }
8309 }
8310
8311 Err(last_error.unwrap_or_else(|| {
8312 InferenceError::InferenceFailed("no text-to-speech models available".into())
8313 }))
8314 }
8315
8316 async fn synthesize_windows_speech(
8322 &self,
8323 schema: &ModelSchema,
8324 req: &SynthesizeRequest,
8325 ) -> Result<SynthesizeResult, InferenceError> {
8326 #[cfg(target_os = "windows")]
8327 {
8328 let text = req.text.clone();
8329 let voice = req.voice.clone().unwrap_or_default();
8330 let rate = req.speed.unwrap_or(1.0) as f64;
8331 let bytes =
8332 tokio::task::spawn_blocking(move || winrt_synthesize_wav(&text, &voice, rate))
8333 .await
8334 .map_err(|e| {
8335 InferenceError::InferenceFailed(format!("winrt tts join: {e}"))
8336 })??;
8337 let dest = requested_or_temp_output(req.output_path.as_deref(), "wav")?;
8338 ensure_parent_dir(&dest)?;
8339 std::fs::write(&dest, &bytes)?;
8340 Ok(SynthesizeResult {
8341 audio_path: dest.to_string_lossy().to_string(),
8342 media_type: "audio/wav".to_string(),
8343 model_used: Some(schema.name.clone()),
8344 voice_used: req.voice.clone(),
8345 routing_explanation: None,
8346 })
8347 }
8348 #[cfg(not(target_os = "windows"))]
8349 {
8350 let _ = (schema, req);
8351 Err(InferenceError::InferenceFailed(
8352 "Windows OS TTS is only available on Windows".into(),
8353 ))
8354 }
8355 }
8356
8357 pub async fn generate_image(
8359 &self,
8360 req: GenerateImageRequest,
8361 ) -> Result<GenerateImageResult, InferenceError> {
8362 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8379 {
8380 use crate::backend::external_flux;
8381 tracing::info!(
8382 model = req.model.as_deref().unwrap_or("<default>"),
8383 "routing image generation to mflux"
8384 );
8385 let schema = self
8386 .media_generation_candidates(
8387 ModelCapability::ImageGeneration,
8388 req.model.as_deref(),
8389 )?
8390 .into_iter()
8391 .next()
8392 .ok_or_else(|| {
8393 InferenceError::InferenceFailed("no image generation models available".into())
8394 })?;
8395 let mut reservation = self.reserve_local_request(&schema, 0)?;
8397 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8398 reservation
8399 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8400 .map_err(InferenceError::from)?;
8401 let mut req = req;
8402 req.model = self
8403 .resolve_external_hf_repo(req.model.as_deref(), ModelCapability::ImageGeneration);
8404 external_flux::generate_image(&req)
8405 }
8406
8407 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8412 {
8413 let _ = &req;
8414 Err(InferenceError::InferenceFailed(
8415 "local image generation requires Apple Silicon (mflux)".into(),
8416 ))
8417 }
8418 }
8419
8420 pub async fn generate_image_batch(
8436 &self,
8437 req: GenerateImageRequest,
8438 ) -> Result<Vec<GenerateImageResult>, InferenceError> {
8439 let count = req.variant_count.unwrap_or(1).max(1);
8440 if count == 1 {
8441 return self.generate_image(req).await.map(|r| vec![r]);
8442 }
8443 let base_seed = req.seed.unwrap_or(0);
8444 let mut results = Vec::with_capacity(count as usize);
8445 for i in 0..count {
8446 let mut variant_req = req.clone();
8451 variant_req.seed = Some(base_seed.wrapping_add(i as u64));
8452 variant_req.variant_count = Some(1);
8456 results.push(self.generate_image(variant_req).await?);
8457 }
8458 Ok(results)
8459 }
8460
8461 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8473 fn mlx_device_lock() -> Arc<tokio::sync::Mutex<()>> {
8474 static MLX_DEVICE_LOCK: std::sync::OnceLock<Arc<tokio::sync::Mutex<()>>> =
8475 std::sync::OnceLock::new();
8476 MLX_DEVICE_LOCK
8477 .get_or_init(|| Arc::new(tokio::sync::Mutex::new(())))
8478 .clone()
8479 }
8480
8481 pub async fn generate_video(
8483 &self,
8484 req: GenerateVideoRequest,
8485 ) -> Result<GenerateVideoResult, InferenceError> {
8486 if let Err(msg) = req.validate() {
8489 return Err(InferenceError::InferenceFailed(format!(
8490 "invalid GenerateVideoRequest: {}",
8491 msg
8492 )));
8493 }
8494 #[allow(unused_variables)]
8496 let requires_audio_conditioning = req.requires_audio_passthrough_opt_in();
8497 let candidates = self
8500 .media_generation_candidates(ModelCapability::VideoGeneration, req.model.as_deref())?;
8501 let mut last_error = None;
8502
8503 for schema in candidates {
8504 let mut reservation = self.reserve_local_request(&schema, 0)?;
8506 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8507 reservation
8508 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8509 .map_err(InferenceError::from)?;
8510 let result = match &schema.source {
8511 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8512 ModelSource::Mlx { hf_repo, .. } => {
8513 if !crate::backend::external_ltx::is_available() {
8527 return Err(InferenceError::InferenceFailed(
8528 "video generation requires the external `ltx-2-mlx` CLI on PATH"
8529 .to_string(),
8530 ));
8531 }
8532 let mut req = req.clone();
8533 req.model = Some(hf_repo.clone());
8534 crate::backend::external_ltx::generate_video(&req)
8535 }
8536 _ => Err(InferenceError::InferenceFailed(format!(
8537 "video generation not implemented for model source: {}",
8538 schema.id
8539 ))),
8540 };
8541
8542 match result {
8543 Ok(result) => return Ok(result),
8544 Err(err) if req.model.is_some() => return Err(err),
8545 Err(err) => last_error = Some(err),
8546 }
8547 }
8548
8549 Err(last_error.unwrap_or_else(|| {
8550 InferenceError::InferenceFailed("no video generation models available".into())
8551 }))
8552 }
8553
8554 pub fn response_format_rejection_reason(
8562 &self,
8563 model: &str,
8564 rf: &crate::tasks::generate::ResponseFormat,
8565 ) -> Option<String> {
8566 let registry = (*self.unified_registry()).clone();
8567 let schema = registry
8568 .list()
8569 .into_iter()
8570 .find(|s| s.id == model || s.name == model)?;
8571 match &schema.source {
8572 ModelSource::Proprietary { .. } => {
8573 Some("the Parslee gateway does not accept response_format".to_string())
8574 }
8575 ModelSource::RemoteApi { protocol, .. } => {
8576 let handler = crate::protocol::handler_for(*protocol);
8577 (!handler.supports_response_format(rf)).then(|| {
8578 format!(
8579 "the {} protocol rejects response_format",
8580 handler.protocol_name()
8581 )
8582 })
8583 }
8584 _ => None,
8585 }
8586 }
8587
8588 pub fn list_models_unified(&self) -> Vec<ModelInfo> {
8593 let hardware = HardwareInfo::detect();
8594 let policy = self.active_local_resource_policy();
8595 self.list_models_unified_for(&hardware, &policy.policy)
8596 }
8597
8598 pub fn model_fit(&self, schema: &ModelSchema) -> recommend::ModelFit {
8602 recommend::model_fit(
8603 schema,
8604 &HardwareInfo::detect(),
8605 Some(&self.active_local_resource_policy().policy),
8606 )
8607 }
8608
8609 pub fn list_models_unified_for(
8613 &self,
8614 hardware: &HardwareInfo,
8615 policy: &resource_policy::ResourcePolicy,
8616 ) -> Vec<ModelInfo> {
8617 self.list_models_unified_for_inner(hardware, policy, false)
8618 }
8619
8620 pub fn list_models_unified_for_explicit_status(&self) -> Vec<ModelInfo> {
8641 let hardware = HardwareInfo::detect();
8642 let policy = self.active_local_resource_policy();
8643 self.list_models_unified_for_inner(&hardware, &policy.policy, true)
8644 }
8645
8646 fn list_models_unified_for_inner(
8647 &self,
8648 hardware: &HardwareInfo,
8649 policy: &resource_policy::ResourcePolicy,
8650 authoritative_credentials: bool,
8651 ) -> Vec<ModelInfo> {
8652 let mut registry = (*self.unified_registry()).clone();
8653 registry.prune_missing_on_disk_models();
8654 if authoritative_credentials {
8655 registry.refresh_availability_for_explicit_status();
8656 } else {
8657 registry.refresh_availability();
8658 }
8659 registry
8660 .list()
8661 .iter()
8662 .map(|schema| {
8663 let mut info = ModelInfo::from(*schema).with_fit(recommend::model_fit(
8664 schema,
8665 hardware,
8666 Some(policy),
8667 ));
8668 if !authoritative_credentials {
8669 info.credential_required = None;
8686 } else if let Some(env) = info.credential_required.as_deref() {
8687 if registry.credential_state_is_unknown(env) {
8698 info.credential_required = None;
8699 }
8700 }
8701 if schema.downloads_weights() {
8702 let enabled = self
8703 .model_management
8704 .car_enabled(&schema.id)
8705 .unwrap_or(false);
8706 if !enabled {
8707 info.available = false;
8711 }
8712 if enabled
8713 && self
8714 .model_management
8715 .load_receipt(&schema.id)
8716 .is_ok_and(|receipt| receipt.is_none())
8717 {
8718 if let Ok(_mutation) = self.model_management.begin_mutation(&schema.id) {
8725 if let Some(path) = registry.existing_local_artifact(&schema.id) {
8726 if std::fs::symlink_metadata(&path)
8727 .is_ok_and(|metadata| metadata.file_type().is_symlink())
8728 {
8729 let generation = self
8730 .resource_policy_generation
8731 .load(std::sync::atomic::Ordering::Acquire);
8732 let _ = self.model_management.record_managed_artifact(
8733 &schema.id,
8734 model_source_identity(schema),
8735 None,
8736 generation,
8737 true,
8738 path,
8739 );
8740 }
8741 }
8742 }
8743 }
8744 let receipt = self.model_management.load_receipt(&schema.id);
8745 let directory_cleanup_unsupported =
8746 !model_management::directory_removal_supported()
8747 && receipt.as_ref().is_ok_and(|receipt| {
8748 receipt.as_ref().is_some_and(|receipt| {
8749 receipt.artifact_kind
8750 == model_management::ManagedArtifactKind::Directory
8751 })
8752 });
8753 info.car_enabled = enabled;
8754 info.can_remove = enabled
8755 && self
8756 .model_management
8757 .can_remove(&schema.id)
8758 .unwrap_or(false);
8759 info.in_use = self.local_model_in_use(&schema.id);
8760 info.management_evidence = if !enabled {
8761 Some("disabled_tombstone".into())
8762 } else if directory_cleanup_unsupported {
8763 Some("install_receipt_directory_cleanup_unsupported".into())
8764 } else if receipt.is_ok_and(|receipt| receipt.is_some()) {
8765 Some("install_receipt".into())
8766 } else if info.weights_ready {
8767 Some("shared_or_hand_installed".into())
8768 } else {
8769 None
8770 };
8771 }
8772 info
8773 })
8774 .collect()
8775 }
8776
8777 fn local_model_in_use(&self, model_id: &str) -> bool {
8781 self.local_admission.active_request_count(model_id) > 0
8782 || self.local_admission.is_resident(model_id)
8783 || self.local_admission.teardown_pending(model_id)
8784 || self.model_management.model_in_use(model_id).unwrap_or(true)
8785 }
8786
8787 pub fn retire_plan(&self, model_id: &str) -> Result<retire::RetirePlan, InferenceError> {
8791 self.retire_plan_in(model_id, &hf_cache::hub_dir())
8792 }
8793
8794 pub fn retire_plan_in(
8796 &self,
8797 model_id: &str,
8798 hub: &Path,
8799 ) -> Result<retire::RetirePlan, InferenceError> {
8800 self.retire_plan_with(model_id, hub, None)
8801 .map(|(plan, _)| plan)
8802 }
8803
8804 fn retire_plan_with(
8809 &self,
8810 model_id: &str,
8811 hub: &Path,
8812 held: Option<&std::collections::BTreeSet<String>>,
8813 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
8814 if self.unified_registry().get(model_id).is_none() {
8815 return Err(InferenceError::ModelNotFound(model_id.to_string()));
8816 }
8817 let (refreshed, owned) = self.retire_snapshot(hub);
8818 self.retire_plan_on(&refreshed, &owned, model_id, hub, held)
8819 }
8820
8821 fn retire_snapshot(&self, hub: &Path) -> (UnifiedRegistry, retire::Ownership) {
8826 let mut refreshed = (*self.unified_registry()).clone();
8827 refreshed.refresh_availability();
8828 let owned = retire::ownership(
8829 retire::catalog_ownership(),
8830 retire::recorded_files(&self.config.state_root, hub),
8831 );
8832 (refreshed, owned)
8833 }
8834
8835 fn retire_plan_on(
8837 &self,
8838 refreshed: &UnifiedRegistry,
8839 owned: &retire::Ownership,
8840 model_id: &str,
8841 hub: &Path,
8842 held: Option<&std::collections::BTreeSet<String>>,
8843 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
8844 let schema = refreshed
8845 .get(model_id)
8846 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
8847 let registry: Vec<&ModelSchema> = refreshed.all().collect();
8848 let installed = |m: &ModelSchema| {
8849 m.downloads_weights()
8850 && m.weights_ready
8851 && self.model_management.car_enabled(&m.id).unwrap_or(true)
8852 };
8853 let primary = retire::primary_whole_repo(schema);
8859 let expands = |repo: &str| {
8863 hf_cache::checked_repo_dir_in(hub, repo).is_some()
8864 && owned.get(repo) == Some(&retire::Owned::Whole)
8865 };
8866 let mut targets: Vec<&ModelSchema> = vec![schema];
8867 targets.extend(registry.iter().copied().filter(|m| {
8868 m.id != schema.id
8869 && primary.is_some_and(|repo| {
8870 retire::primary_whole_repo(m) == Some(repo) && expands(repo)
8871 })
8872 }));
8873 let mut plan = retire::plan_hub_retirement(&retire::RetireInputs {
8874 targets: &targets,
8875 registry: ®istry,
8876 installed: &installed,
8877 owned,
8878 hub,
8879 });
8880 if plan.refusals.iter().any(|r| {
8881 matches!(
8882 r,
8883 retire::RetireRefusal::NothingToRetire | retire::RetireRefusal::Unsupported { .. }
8884 )
8885 }) {
8886 return Ok((plan, std::collections::BTreeSet::new()));
8887 }
8888
8889 let models_dir = self
8893 .config
8894 .models_dir
8895 .canonicalize()
8896 .unwrap_or_else(|_| self.config.models_dir.clone());
8897 for target in &targets {
8898 let mut projections = Vec::new();
8899 if retire::single_component(&target.name) {
8900 projections.push(models_dir.join(&target.name));
8901 } else {
8902 plan.refusals.push(retire::RetireRefusal::UnsafePath {
8903 path: self.config.models_dir.join(&target.name),
8904 why: "the model's name is not a single path component".into(),
8905 });
8906 }
8907 if let Ok(adopted) = self.model_management.adopted_projection_path(&target.id) {
8908 projections.push(adopted);
8909 }
8910 let receipt_path = self
8915 .model_management
8916 .load_receipt(&target.id)
8917 .ok()
8918 .flatten()
8919 .map(|r| r.managed_path);
8920 for path in projections {
8921 if std::fs::symlink_metadata(&path).is_err()
8922 || plan.projections.iter().any(|p| p.path == path)
8923 || plan.user_content.contains(&path)
8924 {
8925 continue;
8926 }
8927 let receipted = receipt_path
8932 .as_ref()
8933 .is_some_and(|managed| retire::same_entry(managed, &path));
8934 if receipted || retire::links_only(&path) {
8935 plan.freed_bytes += retire::tree_bytes(&path);
8936 plan.projections.push(retire::Projection {
8937 path,
8938 receipt_for: receipted.then(|| target.id.clone()),
8939 });
8940 } else {
8941 plan.user_content.push(path);
8942 }
8943 }
8944 }
8945
8946 if targets.iter().any(|t| self.local_model_in_use(&t.id)) {
8947 plan.refusals.push(retire::RetireRefusal::InUse);
8948 }
8949 let repos: std::collections::HashSet<String> = targets
8953 .iter()
8954 .flat_map(|t| retire::hf_references(t))
8955 .map(|r| r.repo)
8956 .collect();
8957 let lock_keys: std::collections::BTreeSet<String> = registry
8958 .iter()
8959 .filter(|m| {
8960 targets.iter().any(|t| t.id == m.id)
8961 || retire::hf_references(m)
8962 .iter()
8963 .any(|r| repos.contains(&r.repo))
8964 })
8965 .flat_map(|m| [m.id.clone(), m.name.clone()])
8966 .collect();
8967 match held {
8968 None => {
8969 if lock_keys
8970 .iter()
8971 .any(|key| crate::download::model_lock_held(key))
8972 {
8973 plan.refusals.push(retire::RetireRefusal::Downloading);
8974 }
8975 }
8976 Some(held) => {
8977 if !lock_keys.is_subset(held) {
8978 plan.refusals.push(retire::RetireRefusal::StateChanged);
8979 }
8980 }
8981 }
8982 for default in self.lane_defaults().defaults {
8983 if targets.iter().any(|t| t.id == default.model_id) {
8984 let use_case = serde_json::to_value(default.use_case)
8985 .ok()
8986 .and_then(|v| v.as_str().map(str::to_string))
8987 .unwrap_or_default();
8988 plan.refusals
8989 .push(retire::RetireRefusal::LaneDefault { use_case });
8990 }
8991 }
8992 let protected = self.protected_models(refreshed, false);
8996 for target in &targets {
8997 if let Some(why) = protected.get(&target.id) {
8998 plan.refusals
8999 .push(retire::RetireRefusal::Protected { why: why.clone() });
9000 }
9001 }
9002 if plan.deletions.is_empty() && plan.projections.is_empty() && plan.refusals.is_empty() {
9004 plan.refusals.push(retire::RetireRefusal::NothingToRetire);
9005 }
9006 plan.seal();
9007 Ok((plan, lock_keys))
9008 }
9009
9010 pub async fn retire_model(
9016 &self,
9017 model_id: &str,
9018 dry_run: bool,
9019 expect: Option<&str>,
9020 ) -> Result<retire::RetireOutcome, InferenceError> {
9021 self.retire_model_in(model_id, &hf_cache::hub_dir(), dry_run, expect)
9022 .await
9023 }
9024
9025 pub async fn retire_model_in(
9041 &self,
9042 model_id: &str,
9043 hub: &Path,
9044 dry_run: bool,
9045 expect: Option<&str>,
9046 ) -> Result<retire::RetireOutcome, InferenceError> {
9047 let not_executed = |plan| retire::RetireOutcome {
9048 plan,
9049 executed: false,
9050 removed: Vec::new(),
9051 errors: Vec::new(),
9052 };
9053 let (first, keys) = self.retire_plan_with(model_id, hub, None)?;
9054 if dry_run || first.is_refused() {
9055 return Ok(not_executed(first));
9056 }
9057 let _concierge = self.concierge_action_lock.lock().await;
9061 let mut locks = Vec::with_capacity(keys.len());
9062 for key in &keys {
9063 match crate::download::try_acquire_model_lock(key) {
9064 Some(guard) => locks.push(guard),
9065 None => {
9066 let mut refused = first;
9067 refused.refusals.push(retire::RetireRefusal::Downloading);
9068 return Ok(not_executed(refused));
9069 }
9070 }
9071 }
9072 let (mut plan, _) = self.retire_plan_with(model_id, hub, Some(&keys))?;
9073 if expect.is_some_and(|digest| digest != plan.digest) {
9074 plan.refusals.push(retire::RetireRefusal::StateChanged);
9075 }
9076 if plan.is_refused() {
9077 return Ok(not_executed(plan));
9078 }
9079 let targets: Vec<String> = std::iter::once(plan.model_id.clone())
9080 .chain(plan.also_retires.iter().cloned())
9081 .collect();
9082 let mut maintenance = Vec::with_capacity(targets.len());
9083 for id in &targets {
9084 match self.prepare_local_model_removal(id).await {
9085 Ok(guard) => maintenance.push(guard),
9086 Err(error) => {
9087 tracing::info!(model = %id, %error, "retirement refused: the model could not be released");
9088 plan.refusals.push(retire::RetireRefusal::InUse);
9089 return Ok(not_executed(plan));
9090 }
9091 }
9092 }
9093
9094 let started_at = std::time::SystemTime::now()
9097 .duration_since(std::time::UNIX_EPOCH)
9098 .map(|d| d.as_secs())
9099 .unwrap_or(0);
9100 for id in &targets {
9101 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9102 seq: 0,
9103 kind: action_ledger::ConciergeActionKind::RetireStarted,
9104 model_id: id.clone(),
9105 use_case: None,
9106 project: None,
9107 prior_model_id: None,
9108 detail: format!(
9109 "retiring with {}; plan {}; {} bytes planned",
9110 targets.join(", "),
9111 plan.digest,
9112 plan.freed_bytes
9113 ),
9114 timestamp: started_at,
9115 });
9116 }
9117
9118 let mut removed = Vec::new();
9119 let mut errors = Vec::new();
9120
9121 for projection in &plan.projections {
9124 let result = match &projection.receipt_for {
9125 Some(id) => {
9126 let result = async {
9129 self.model_management
9130 .note_retiring(id)
9131 .map_err(|e| e.to_string())?;
9132 self.remove_model_from_car_inner(id, true)
9133 .await
9134 .map_err(|e| e.to_string())?;
9135 self.model_management
9136 .clear_tombstone(id)
9137 .map_err(|e| e.to_string())?;
9138 self.model_management.retired(id);
9139 Ok::<_, String>(vec![projection.path.clone()])
9140 }
9141 .await;
9142 result
9143 }
9144 None => retire::remove_links_only_tree(&projection.path, &self.config.models_dir),
9145 };
9146 match result {
9147 Ok(paths) => removed.extend(paths),
9148 Err(why) => {
9149 errors.push(format!("{}: {why}", projection.path.display()));
9150 break;
9151 }
9152 }
9153 }
9154
9155 let hub_canonical = hub.canonicalize().unwrap_or_else(|_| hub.to_path_buf());
9156 if errors.is_empty() {
9157 'hub: for deletion in &plan.deletions {
9158 match deletion {
9159 retire::HubDeletion::Repo { path, .. } => {
9160 let quarantine = path.with_file_name(format!(
9161 "{}{:016x}",
9162 model_management::RETIRE_QUARANTINE_PREFIX,
9163 rand::random::<u64>()
9164 ));
9165 let verify = || retire::verify_repo_dir(path, hub);
9166 match self.model_management.detach_and_remove_tree(
9167 path,
9168 &quarantine,
9169 model_id,
9170 verify,
9171 ) {
9172 Ok(()) => removed.push(path.clone()),
9173 Err(failure) => {
9174 if failure.detached {
9175 removed.push(path.clone());
9178 }
9179 errors.push(format!("{}: {}", path.display(), failure.error));
9180 break 'hub;
9181 }
9182 }
9183 }
9184 retire::HubDeletion::Files {
9185 repo,
9186 entries,
9187 blobs,
9188 ..
9189 } => {
9190 let Some(dir) = hf_cache::checked_repo_dir_in(&hub_canonical, repo) else {
9191 errors.push(format!("{repo}: not a Hub repo id"));
9192 break 'hub;
9193 };
9194 for blob in blobs {
9198 let checked =
9199 retire::verify_blob(blob, &dir.join("blobs")).and_then(|()| {
9200 model_management::unlink_checked(
9201 blob,
9202 model_management::UnlinkKind::File,
9203 )
9204 .map_err(|e| e.to_string())
9205 });
9206 match checked {
9207 Ok(()) => removed.push(blob.clone()),
9208 Err(why) => {
9209 errors.push(format!("{}: {why}", blob.display()));
9210 break 'hub;
9211 }
9212 }
9213 }
9214 let snapshots = dir.join("snapshots");
9215 for entry in entries {
9216 let checked = retire::verify_entry(entry, &snapshots).and_then(|()| {
9217 model_management::unlink_checked(
9218 entry,
9219 model_management::UnlinkKind::FileOrLink,
9220 )
9221 .map_err(|e| e.to_string())
9222 });
9223 match checked {
9224 Ok(()) => removed.push(entry.clone()),
9225 Err(why) => {
9226 errors.push(format!("{}: {why}", entry.display()));
9227 break 'hub;
9228 }
9229 }
9230 }
9231 for entry in entries {
9234 let mut at = entry.parent();
9235 while let Some(parent) = at {
9236 if !parent.starts_with(&snapshots) || parent == snapshots {
9237 break;
9238 }
9239 if std::fs::remove_dir(parent).is_err() {
9240 break;
9241 }
9242 at = parent.parent();
9243 }
9244 }
9245 retire::prune_empty_repo(&dir);
9246 }
9247 }
9248 }
9249 }
9250 drop(maintenance);
9251 drop(locks);
9252
9253 if !removed.is_empty() {
9254 let now = std::time::SystemTime::now()
9255 .duration_since(std::time::UNIX_EPOCH)
9256 .map(|d| d.as_secs())
9257 .unwrap_or(0);
9258 for id in &targets {
9259 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9260 seq: 0,
9261 kind: action_ledger::ConciergeActionKind::Retire,
9262 model_id: id.clone(),
9263 use_case: None,
9264 project: None,
9265 prior_model_id: None,
9266 detail: format!(
9267 "retired with {}; plan {}; {} bytes planned; {} paths removed{}",
9268 targets.join(", "),
9269 plan.digest,
9270 plan.freed_bytes,
9271 removed.len(),
9272 if errors.is_empty() {
9273 String::new()
9274 } else {
9275 format!("; stopped: {}", errors.join("; "))
9276 }
9277 ),
9278 timestamp: now,
9279 });
9280 }
9281 }
9282 Ok(retire::RetireOutcome {
9283 executed: !removed.is_empty(),
9284 plan,
9285 removed,
9286 errors,
9287 })
9288 }
9289
9290 pub fn set_external_pins(&self, pins: std::collections::BTreeMap<String, Vec<String>>) {
9298 *self
9299 .external_pins
9300 .write()
9301 .unwrap_or_else(std::sync::PoisonError::into_inner) = pins;
9302 }
9303
9304 pub fn start_usage_tracking(&self) {
9308 self.model_management.start_usage_tracking();
9309 self.model_management.beat();
9310 }
9311
9312 fn recent_concierge_actions(&self, now: u64) -> Vec<portfolio::RecentAction> {
9316 use action_ledger::ConciergeActionKind as Kind;
9317 let since = now.saturating_sub(30 * 24 * 60 * 60);
9318 let actions = crate::action_ledger::read_actions(&self.action_ledger_path(), 0);
9319 let mut out: Vec<portfolio::RecentAction> = Vec::new();
9320 for (i, a) in actions.iter().enumerate() {
9321 if a.timestamp < since {
9322 continue;
9323 }
9324 let (kind, undo, interrupted) = match a.kind {
9325 Kind::Retire => (
9326 "retire",
9327 (!a.model_id.starts_with("repo:"))
9328 .then(|| format!("car models pull {}", a.model_id)),
9329 false,
9330 ),
9331 Kind::RetireStarted => {
9332 let finished = actions[i + 1..]
9333 .iter()
9334 .any(|later| later.kind == Kind::Retire && later.model_id == a.model_id);
9335 if finished {
9336 continue;
9337 }
9338 ("retire", None, true)
9339 }
9340 Kind::DiscardPartials => ("discard_partials", None, false),
9341 Kind::ResourcePolicy => (
9342 "resource_policy",
9343 a.prior_model_id
9344 .as_ref()
9345 .map(|prior| format!("car models resource-policy set {prior}")),
9346 false,
9347 ),
9348 Kind::AutoUpgrade => (
9349 "upgrade",
9350 match (a.use_case, &a.prior_model_id) {
9351 (Some(use_case), _) => serde_json::to_value(use_case)
9352 .ok()
9353 .and_then(|v| v.as_str().map(str::to_owned))
9354 .map(|lane| format!("car concierge rollback {lane}")),
9355 (None, Some(prior)) => {
9356 Some(format!("keep using {prior}; nothing was switched"))
9357 }
9358 (None, None) => None,
9359 },
9360 false,
9361 ),
9362 Kind::RevokedLaneCleared => ("lane_cleared", None, false),
9365 Kind::Install | Kind::SetDefault | Kind::ClearDefault | Kind::Rollback => continue,
9366 };
9367 out.push(portfolio::RecentAction {
9368 at: a.timestamp,
9369 kind: kind.to_string(),
9370 subject: a.model_id.clone(),
9371 detail: a.detail.clone(),
9372 undo,
9373 use_case: a.use_case.and_then(|use_case| {
9374 serde_json::to_value(use_case)
9375 .ok()
9376 .and_then(|v| v.as_str().map(str::to_owned))
9377 }),
9378 prior: a.prior_model_id.clone(),
9379 interrupted,
9380 });
9381 }
9382 out.reverse();
9383 out
9384 }
9385
9386 pub async fn set_resource_policy_by_user(
9391 &self,
9392 policy: resource_policy::ResourcePolicy,
9393 ) -> Result<(), resource_policy::ResourcePolicyError> {
9394 let _concierge = self.concierge_action_lock.lock().await;
9395 let repository =
9396 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9397 repository.clear_concierge_marker()?;
9398 resource_policy::ResourcePolicyRepository::save(&repository, &policy)?;
9399 self.apply_local_resource_policy(policy);
9400 Ok(())
9401 }
9402
9403 async fn apply_one_curated_upgrade(
9412 &self,
9413 hub: &Path,
9414 nudge: &crate::nudge::NudgeState,
9415 now: u64,
9416 ) -> Result<Option<maintenance::Upgraded>, String> {
9417 let findings = self.detect_upgrades().await;
9418 self.apply_one_curated_upgrade_from(findings, hub, nudge, now)
9419 .await
9420 }
9421
9422 async fn apply_one_curated_upgrade_from(
9424 &self,
9425 findings: Vec<crate::upgrade::UpgradeFinding>,
9426 hub: &Path,
9427 nudge: &crate::nudge::NudgeState,
9428 now: u64,
9429 ) -> Result<Option<maintenance::Upgraded>, String> {
9430 use crate::upgrade::UpgradeSource;
9431 for finding in findings {
9432 let label = match finding.source {
9433 UpgradeSource::Benchmark => "benchmark upgrade",
9434 UpgradeSource::Curated | UpgradeSource::Upstream => "curated upgrade",
9435 };
9436 let key = format!("{}=>{}", finding.from_id, finding.to_id);
9437 let backing_off = self
9438 .failed_upgrades
9439 .lock()
9440 .unwrap_or_else(std::sync::PoisonError::into_inner)
9441 .get(&key)
9442 .is_some_and(|at| now.saturating_sub(*at) < 24 * 60 * 60);
9443 if finding.source == UpgradeSource::Upstream
9444 || finding.trust_tier != crate::schema::TrustTier::Curated
9445 || !finding.target_pullable
9446 || nudge.dismissed.contains(&key)
9447 || backing_off
9448 {
9449 continue;
9450 }
9451 let Some(target) = self.unified_registry().get(&finding.to_id).cloned() else {
9452 continue;
9453 };
9454 let fit = self.model_fit(&target);
9455 if !fit.platform_compatible || fit.fit == recommend::ModelFitStatus::TooBig {
9456 continue;
9457 }
9458 let lanes: Vec<crate::lane_defaults::LaneDefault> = self
9459 .lane_defaults()
9460 .defaults
9461 .into_iter()
9462 .filter(|d| d.model_id == finding.from_id)
9463 .collect();
9464 if lanes.is_empty() {
9465 continue;
9466 }
9467 let present = self
9469 .unified_registry()
9470 .existing_local_artifact(&finding.to_id)
9471 .is_some();
9472 if !present && crate::download::check_disk_space(hub, target.size_mb()).is_err() {
9473 continue;
9474 }
9475 let mut switched = Vec::new();
9476 for lane in &lanes {
9477 if let Err(e) = self
9478 .apply_concierge(lane.use_case, &finding.to_id, lane.project.clone())
9479 .await
9480 {
9481 self.failed_upgrades
9482 .lock()
9483 .unwrap_or_else(std::sync::PoisonError::into_inner)
9484 .insert(key, now);
9485 if let Some(first) = lanes.first().filter(|_| !switched.is_empty()) {
9487 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9488 seq: 0,
9489 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
9490 model_id: finding.to_id.clone(),
9491 use_case: Some(first.use_case),
9492 project: first.project.clone(),
9493 prior_model_id: Some(finding.from_id.clone()),
9494 detail: format!(
9495 "{label}, partial: switched {}; then {e}",
9496 switched.join(", ")
9497 ),
9498 timestamp: now,
9499 });
9500 }
9501 return Err(format!(
9502 "upgrading {} to {} (fetch, then switch the lane): {e}; retried in a day",
9503 finding.from_id, finding.to_id
9504 ));
9505 }
9506 switched.push(
9507 serde_json::to_value(lane.use_case)
9508 .ok()
9509 .and_then(|v| v.as_str().map(str::to_owned))
9510 .unwrap_or_default(),
9511 );
9512 }
9513 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9514 seq: 0,
9515 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
9516 model_id: finding.to_id.clone(),
9517 use_case: lanes.first().map(|l| l.use_case),
9518 project: lanes.first().and_then(|l| l.project.clone()),
9519 prior_model_id: Some(finding.from_id.clone()),
9520 detail: format!("{label}: {}", finding.reason),
9521 timestamp: now,
9522 });
9523 return Ok(Some(maintenance::Upgraded {
9524 from: finding.from_id,
9525 to: finding.to_id,
9526 lanes: switched,
9527 }));
9528 }
9529 Ok(None)
9530 }
9531
9532 async fn size_memory_policy(
9541 &self,
9542 hw: &HardwareInfo,
9543 observable_since: Option<u64>,
9544 apply: bool,
9545 now: u64,
9546 ) -> Option<maintenance::PolicySizing> {
9547 use resource_policy::{ResourcePolicy, ResourcePolicyLoadSource};
9548 let since = observable_since?;
9549 let _concierge = self.concierge_action_lock.lock().await;
9550 let repository =
9551 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9552 let loaded = repository.load_with_evidence().ok()?;
9553 let concierge = repository.concierge_choice();
9554 let user_chose = match loaded.source {
9555 ResourcePolicyLoadSource::MissingDefault => false,
9556 ResourcePolicyLoadSource::CorruptDefault => true,
9557 ResourcePolicyLoadSource::Loaded => {
9558 concierge.as_ref().map(|c| &c.policy) != Some(&loaded.policy)
9559 }
9560 };
9561 let everyday = ResourcePolicy::everyday();
9562 let local_focused = ResourcePolicy::local_focused();
9563 let fit = |m: &ModelSchema, policy: &ResourcePolicy| {
9564 recommend::model_fit(m, hw, Some(policy)).fit
9565 };
9566 let raised_at = concierge
9567 .as_ref()
9568 .filter(|c| !c.reverted && c.policy == local_focused)
9569 .map(|c| c.at);
9570 let mut evidence = maintenance::PolicyEvidence::default();
9571 let mut days = std::collections::BTreeSet::new();
9572 for e in crate::outcome::read_ledger(
9573 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9574 0,
9575 ) {
9576 if e.timestamp < since
9577 || e.attribution != Some(crate::outcome::UnattributedReason::ResourceBlocked)
9578 {
9579 continue;
9580 }
9581 let registry = self.unified_registry();
9582 let Some(m) = registry.get(&e.model_id).filter(|m| m.is_local()) else {
9583 continue;
9584 };
9585 if fit(m, &loaded.policy) == recommend::ModelFitStatus::TooBig
9590 && fit(m, &local_focused) == recommend::ModelFitStatus::Fits
9591 {
9592 evidence.ceiling_refusals += 1;
9593 days.insert(e.timestamp / (24 * 60 * 60));
9594 }
9595 if raised_at.is_some_and(|at| e.timestamp >= at)
9596 && fit(m, &everyday) == recommend::ModelFitStatus::Fits
9597 {
9598 evidence.live_refusals_since_raise += 1;
9599 }
9600 }
9601 if let Some(at) = raised_at {
9603 for e in crate::outcome::read_ledger(
9604 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9605 0,
9606 ) {
9607 let admitted_by_raise = self.unified_registry().get(&e.model_id).is_some_and(|m| {
9608 m.is_local() && fit(m, &everyday) == recommend::ModelFitStatus::TooBig
9609 });
9610 if e.timestamp < at || !admitted_by_raise {
9611 continue;
9612 }
9613 match (e.success, e.attribution) {
9614 (Some(true), _) => evidence.admitted_successes_since_raise += 1,
9615 (Some(false), None) => evidence.admitted_failures_since_raise += 1,
9616 _ => {}
9617 }
9618 }
9619 }
9620 evidence.ceiling_refusal_days = days.len() as u64;
9621 let decision = maintenance::decide_resource_policy(
9622 &loaded.policy,
9623 user_chose,
9624 concierge.as_ref(),
9625 hw.total_ram_mb,
9626 &evidence,
9627 )?;
9628 let (target, reason, reverted) = match decision {
9629 maintenance::PolicyMove::Raise(reason) => (local_focused, reason, false),
9630 maintenance::PolicyMove::Revert(reason) => (everyday, reason, true),
9631 };
9632 let mut sizing = maintenance::PolicySizing {
9633 from: loaded.policy.profile.clone(),
9634 to: target.profile.clone(),
9635 reason,
9636 applied: false,
9637 };
9638 if apply {
9639 match repository.save_concierge_choice(&target, &sizing.reason, now, reverted) {
9640 Ok(()) => {
9641 self.apply_local_resource_policy(target);
9642 sizing.applied = true;
9643 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9644 seq: 0,
9645 kind: action_ledger::ConciergeActionKind::ResourcePolicy,
9646 model_id: String::new(),
9647 use_case: None,
9648 project: None,
9649 prior_model_id: serde_json::to_value(&sizing.from)
9650 .ok()
9651 .and_then(|v| v.as_str().map(str::to_owned)),
9652 detail: format!("{:?} -> {:?}: {}", sizing.from, sizing.to, sizing.reason),
9653 timestamp: now,
9654 });
9655 }
9656 Err(error) => tracing::warn!(%error, "could not change the memory policy"),
9657 }
9658 }
9659 Some(sizing)
9660 }
9661
9662 async fn retire_orphan_repo(&self, repo: &str, hub: &Path) -> Result<u64, String> {
9669 let _concierge = self.concierge_action_lock.lock().await;
9670 let (refreshed, owned) = self.retire_snapshot(hub);
9671 let referenced: std::collections::HashSet<String> = refreshed
9672 .all()
9673 .flat_map(retire::hf_references)
9674 .map(|r| r.repo)
9675 .collect();
9676 let orphan = retire::unreferenced_repos(hub, &referenced, &owned)
9677 .into_iter()
9678 .find(|o| o.repo == repo && o.owner == retire::OrphanOwner::Car)
9679 .ok_or("no longer an orphan CAR owns")?;
9680 if retire::transfer_in_progress(&orphan.path)? {
9681 return Err("a transfer is writing into it".into());
9682 }
9683 let repo_dir = orphan
9688 .path
9689 .canonicalize()
9690 .map_err(|e| format!("cannot resolve the repo: {e}"))?;
9691 if let Some(user) = retire::path_resolving_into(&self.config.models_dir, &repo_dir) {
9692 return Err(format!("{} still resolves into it", user.display()));
9693 }
9694 for row in refreshed.all().filter(|m| m.is_local()) {
9695 let inside = refreshed
9696 .existing_local_artifact(&row.id)
9697 .and_then(|path| path.canonicalize().ok())
9698 .is_some_and(|path| path.starts_with(&repo_dir));
9699 if inside {
9700 return Err(format!("the model {} still loads from it", row.id));
9701 }
9702 }
9703 let label = format!("repo:{repo}");
9704 let now = std::time::SystemTime::now()
9705 .duration_since(std::time::UNIX_EPOCH)
9706 .map(|d| d.as_secs())
9707 .unwrap_or(0);
9708 let entry = |kind, detail: String| action_ledger::ConciergeActionEntry {
9709 seq: 0,
9710 kind,
9711 model_id: label.clone(),
9712 use_case: None,
9713 project: None,
9714 prior_model_id: None,
9715 detail,
9716 timestamp: now,
9717 };
9718 self.record_concierge_action(entry(
9719 action_ledger::ConciergeActionKind::RetireStarted,
9720 format!("orphaned repo; {} bytes planned", orphan.bytes),
9721 ));
9722 let quarantine = orphan.path.with_file_name(format!(
9723 "{}{:016x}",
9724 model_management::RETIRE_QUARANTINE_PREFIX,
9725 rand::random::<u64>()
9726 ));
9727 let verify = || retire::verify_repo_dir(&orphan.path, hub);
9728 self.model_management
9729 .detach_and_remove_tree(&orphan.path, &quarantine, &label, verify)
9730 .map_err(|failure| failure.error.to_string())?;
9731 self.record_concierge_action(entry(
9732 action_ledger::ConciergeActionKind::Retire,
9733 format!("orphaned repo no model uses; {} bytes", orphan.bytes),
9734 ));
9735 Ok(orphan.bytes)
9736 }
9737
9738 pub fn usage_heartbeat(&self) {
9742 self.model_management.beat();
9743 }
9744
9745 pub fn effective_update_policy(
9752 &self,
9753 prefs: &crate::update_prefs::UpdatePreferences,
9754 start_clock: bool,
9755 ) -> (crate::update_prefs::UpdatePolicy, Option<u64>) {
9756 use crate::update_prefs::UpdatePolicy;
9757 if prefs.policy != UpdatePolicy::Auto || prefs.policy_chosen {
9758 return (prefs.policy, None);
9759 }
9760 let marker = self.config.state_root.join(AUTO_GRACE_FILE);
9761 let now = std::time::SystemTime::now()
9762 .duration_since(std::time::UNIX_EPOCH)
9763 .map(|d| d.as_secs())
9764 .unwrap_or(0);
9765 let since = std::fs::read_to_string(&marker)
9766 .ok()
9767 .and_then(|text| text.trim().parse::<u64>().ok())
9768 .unwrap_or_else(|| {
9769 if start_clock {
9770 let _ = std::fs::create_dir_all(&self.config.state_root);
9771 let _ = std::fs::write(&marker, now.to_string());
9772 }
9773 now
9774 });
9775 match maintenance::auto_grace_until(since, now, AUTO_GRACE_SECS) {
9776 Some(until) => (UpdatePolicy::Notify, Some(until)),
9777 None => (UpdatePolicy::Auto, None),
9778 }
9779 }
9780
9781 pub async fn maintenance_pass(&self, dry_run: bool) -> maintenance::MaintenanceReport {
9790 self.maintenance_pass_in(&hf_cache::hub_dir(), &HardwareInfo::detect(), dry_run)
9791 .await
9792 }
9793
9794 pub async fn maintenance_pass_in(
9796 &self,
9797 hub: &Path,
9798 hw: &HardwareInfo,
9799 dry_run: bool,
9800 ) -> maintenance::MaintenanceReport {
9801 let prefs = self.update_prefs();
9802 let (policy, grace_until) = self.effective_update_policy(&prefs, !dry_run);
9803 let nudge = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
9804 let disk = crate::download::disk_space(hub).map(|(free_bytes, total_bytes)| {
9805 maintenance::DiskSpace {
9806 free_bytes,
9807 total_bytes,
9808 }
9809 });
9810 let mut report = self
9811 .maintenance_pass_with(hub, hw, dry_run, policy, &nudge, disk)
9812 .await;
9813 report.auto_grace_until = grace_until;
9814 report
9815 }
9816
9817 pub async fn maintenance_pass_with(
9820 &self,
9821 hub: &Path,
9822 hw: &HardwareInfo,
9823 dry_run: bool,
9824 policy: crate::update_prefs::UpdatePolicy,
9825 nudge: &crate::nudge::NudgeState,
9826 disk: Option<maintenance::DiskSpace>,
9827 ) -> maintenance::MaintenanceReport {
9828 if !dry_run {
9829 self.enforce_catalog_revocations().await;
9830 }
9831 let mut report = maintenance::MaintenanceReport {
9832 policy,
9833 dry_run,
9834 ..Default::default()
9835 };
9836 if policy == crate::update_prefs::UpdatePolicy::Off {
9837 return report;
9838 }
9839 if !dry_run {
9840 report.errors.extend(self.resume_retirements().await);
9841 }
9842 let portfolio = self.portfolio_in(hub, hw).await;
9843 let now = portfolio.policy.now;
9844 report.disk = disk;
9845 let was_pressured = self
9846 .disk_pressured
9847 .load(std::sync::atomic::Ordering::Relaxed);
9848 report.pressured = maintenance::disk_pressure(report.disk, was_pressured);
9849 if !dry_run {
9850 self.disk_pressured
9851 .store(report.pressured, std::sync::atomic::Ordering::Relaxed);
9852 }
9853 let keep = maintenance::kept_models(nudge, now);
9854 let churned = crate::action_ledger::read_actions(&self.action_ledger_path(), 0)
9857 .into_iter()
9858 .filter(|a| {
9859 matches!(
9860 a.kind,
9861 action_ledger::ConciergeActionKind::Retire
9862 | action_ledger::ConciergeActionKind::RetireStarted
9863 )
9864 })
9865 .map(|a| a.model_id)
9866 .collect();
9867 let ctx = maintenance::MaintenanceContext {
9868 policy,
9869 pressured: report.pressured,
9870 keep,
9871 churned,
9872 superseded_unpressured_secs:
9873 maintenance::MaintenanceContext::DEFAULT_SUPERSEDED_UNPRESSURED_SECS,
9874 };
9875 report.plan = maintenance::decide(&portfolio, &ctx);
9876 report.portfolio = Some(portfolio.clone());
9877 report.resource_policy = self
9878 .size_memory_policy(
9879 hw,
9880 portfolio.policy.tracking_since,
9881 !dry_run && policy == crate::update_prefs::UpdatePolicy::Auto,
9882 now,
9883 )
9884 .await;
9885 if dry_run || policy != crate::update_prefs::UpdatePolicy::Auto {
9886 return report;
9887 }
9888 if let Some(act) = &report.plan.act {
9889 match self
9890 .retire_model_in(&act.model_id, hub, false, Some(&act.digest))
9891 .await
9892 {
9893 Ok(outcome) => {
9894 report.retired = outcome.executed.then(|| maintenance::Retired {
9895 model_id: act.model_id.clone(),
9896 also_retired: outcome.plan.also_retires.clone(),
9897 freed_bytes: outcome.plan.freed_bytes,
9898 restore: format!("car models pull {}", act.model_id),
9899 });
9900 if !outcome.executed {
9901 report.errors.push(format!(
9904 "{}: not retired this pass ({:?})",
9905 act.model_id, outcome.plan.refusals
9906 ));
9907 }
9908 report.errors.extend(outcome.errors);
9909 }
9910 Err(error) => report.errors.push(format!("{}: {error}", act.model_id)),
9911 }
9912 }
9913 if report.retired.is_none() && report.pressured {
9916 let mut orphans: Vec<_> = portfolio
9919 .orphans
9920 .iter()
9921 .filter(|o| o.owner == retire::OrphanOwner::Car)
9922 .collect();
9923 orphans.sort_by(|a, b| b.bytes.cmp(&a.bytes));
9924 for orphan in orphans {
9925 match self.retire_orphan_repo(&orphan.repo, hub).await {
9926 Ok(freed_bytes) => {
9927 report.orphan_retired = Some(maintenance::RetiredOrphan {
9928 repo: orphan.repo.clone(),
9929 freed_bytes,
9930 });
9931 break;
9932 }
9933 Err(error) => report.errors.push(format!("{}: {error}", orphan.repo)),
9934 }
9935 }
9936 }
9937 if report.retired.is_none() && report.orphan_retired.is_none() && !report.pressured {
9939 match self.apply_one_curated_upgrade(hub, nudge, now).await {
9940 Ok(upgraded) => report.upgraded = upgraded,
9941 Err(error) => report.errors.push(error),
9942 }
9943 }
9944 let owned = retire::ownership(
9945 retire::catalog_ownership(),
9946 retire::recorded_files(&self.config.state_root, hub),
9947 );
9948 let (discarded, errors) = retire::discard_abandoned_partials(hub, &owned);
9949 report.errors.extend(errors);
9950 if !discarded.is_empty() {
9951 let bytes: u64 = discarded.iter().map(|(_, b)| b).sum();
9952 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9953 seq: 0,
9954 kind: action_ledger::ConciergeActionKind::DiscardPartials,
9955 model_id: String::new(),
9956 use_case: None,
9957 project: None,
9958 prior_model_id: None,
9959 detail: format!(
9960 "{} abandoned partial downloads, {bytes} bytes",
9961 discarded.len()
9962 ),
9963 timestamp: now,
9964 });
9965 report.partials_discarded = discarded.into_iter().map(|(path, _)| path).collect();
9966 }
9967 report
9968 }
9969
9970 pub async fn portfolio(&self) -> portfolio::Portfolio {
9973 self.portfolio_in(&hf_cache::hub_dir(), &HardwareInfo::detect())
9974 .await
9975 }
9976
9977 pub async fn portfolio_in(&self, hub: &Path, hw: &HardwareInfo) -> portfolio::Portfolio {
9979 let now = std::time::SystemTime::now()
9980 .duration_since(std::time::UNIX_EPOCH)
9981 .map(|d| d.as_secs())
9982 .unwrap_or(0);
9983 let observable_since = self
9986 .model_management
9987 .usage_tracking_since()
9988 .zip(self.model_management.usage_covered_since())
9989 .map(|(tracking, covered)| tracking.max(covered));
9990 let policy = portfolio::PortfolioPolicy::new(now, observable_since);
9991 let (refreshed, owned) = self.retire_snapshot(hub);
9992 let protected = self.protected_models(&refreshed, true);
9993 let mut ledger_last: std::collections::HashMap<String, u64> =
9994 std::collections::HashMap::new();
9995 let mut ledger_failures: std::collections::HashMap<String, Vec<u64>> =
10002 std::collections::HashMap::new();
10003 for entry in crate::outcome::read_ledger(
10004 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
10005 0,
10006 ) {
10007 if entry.counts_against_model() {
10008 ledger_failures
10009 .entry(entry.model_id.clone())
10010 .or_default()
10011 .push(entry.timestamp);
10012 }
10013 let last = ledger_last.entry(entry.model_id).or_default();
10014 *last = (*last).max(entry.timestamp);
10015 }
10016 let installed_at = |m: &ModelSchema| -> Option<u64> {
10019 let repo = retire::hf_references(m).into_iter().next()?.repo;
10020 let snapshots = hf_cache::checked_repo_dir_in(hub, &repo)?.join("snapshots");
10021 std::fs::read_dir(snapshots)
10022 .ok()?
10023 .filter_map(Result::ok)
10024 .filter_map(|e| e.metadata().ok()?.modified().ok())
10025 .filter_map(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
10026 .map(|d| d.as_secs())
10027 .max()
10028 };
10029 let installed = |m: &ModelSchema| {
10030 m.is_local()
10031 && m.downloads_weights()
10032 && m.weights_ready
10033 && self.model_management.car_enabled(&m.id).unwrap_or(true)
10034 };
10035 let runnable = |m: &ModelSchema| {
10036 let fit = recommend::model_fit(m, hw, None);
10037 fit.platform_compatible && fit.fit != recommend::ModelFitStatus::TooBig
10038 };
10039 let outcomes: std::collections::HashMap<String, (u64, u64)> = {
10041 let tracker = self.outcome_tracker.read().await;
10042 refreshed
10043 .all()
10044 .filter(|m| installed(m))
10045 .filter_map(|m| {
10046 tracker
10047 .profile(&m.id)
10048 .map(|p| (m.id.clone(), (p.success_count, p.fail_count)))
10049 })
10050 .collect()
10051 };
10052 let has_run = |id: &str| outcomes.get(id).is_some_and(|(success, _)| *success > 0);
10060 let dismissed =
10062 crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path())
10063 .dismissed;
10064 let superseded: std::collections::HashMap<String, String> = refreshed
10065 .available_upgrades()
10066 .into_iter()
10067 .filter(|u| !dismissed.contains(&format!("{}=>{}", u.from_id, u.to_id)))
10068 .filter(|u| {
10069 refreshed
10070 .get(&u.to_id)
10071 .is_some_and(|to| installed(to) && runnable(to) && has_run(&to.id))
10072 })
10073 .map(|u| (u.from_id, u.to_id))
10074 .collect();
10075 let superseded: std::collections::HashMap<String, String> = superseded
10077 .iter()
10078 .filter(|(from, to)| superseded.get(*to) != Some(*from))
10079 .map(|(from, to)| (from.clone(), to.clone()))
10080 .collect();
10081 let budget = recommend::memory_budget_mb(hw);
10082 let keep = maintenance::kept_models(
10083 &crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path()),
10084 now,
10085 );
10086 let mut facts = Vec::new();
10087 for m in refreshed.all().filter(|m| installed(m)) {
10088 let fit = recommend::model_fit(m, hw, None);
10089 let success_count = outcomes.get(&m.id).map_or(0, |(success, _)| *success);
10090 let since_install = installed_at(m)
10094 .unwrap_or(0)
10095 .max(policy.tracking_since.unwrap_or(u64::MAX));
10096 let fail_count = ledger_failures.get(&m.id).map_or(0, |times| {
10097 times.iter().filter(|t| **t >= since_install).count() as u64
10098 });
10099 let retire = self
10100 .retire_plan_on(&refreshed, &owned, &m.id, hub, None)
10101 .ok()
10102 .map(|(plan, _)| portfolio::RetireSummary {
10103 freed_bytes: plan.freed_bytes,
10104 refusals: plan
10105 .refusals
10106 .iter()
10107 .map(|r| {
10108 serde_json::to_value(r)
10109 .ok()
10110 .and_then(|v| {
10111 v.get("reason").and_then(|r| r.as_str()).map(str::to_owned)
10112 })
10113 .unwrap_or_else(|| format!("{r:?}"))
10114 })
10115 .collect(),
10116 also_retires: plan.also_retires.clone(),
10117 digest: plan.digest,
10118 });
10119 facts.push(portfolio::ModelFacts {
10120 model_id: m.id.clone(),
10121 name: m.name.clone(),
10122 generates_text: m.has_capability(ModelCapability::Generate),
10123 platform_compatible: fit.platform_compatible,
10124 never_fits: fit.fit == recommend::ModelFitStatus::TooBig,
10125 estimated_peak_mb: fit.estimated_peak_mb,
10126 hardware_budget_mb: budget,
10127 protected: protected.get(&m.id).cloned(),
10128 kept: keep.contains(&m.id),
10129 last_used: self.model_management.last_used(&m.id),
10130 ledger_last_used: ledger_last.get(&m.id).copied(),
10131 success_count,
10132 fail_count,
10133 superseded_by: superseded.get(&m.id).cloned(),
10134 retire,
10135 });
10136 }
10137 let referenced: std::collections::HashSet<String> = refreshed
10138 .all()
10139 .flat_map(retire::hf_references)
10140 .map(|r| r.repo)
10141 .collect();
10142 let mut assessed = portfolio::assess_all(&facts, &policy);
10143 assessed.orphans = retire::unreferenced_repos(hub, &referenced, &owned);
10144 assessed.recent_actions = self.recent_concierge_actions(now);
10145 assessed.revoked_pins = self.revoked_pins(&refreshed);
10146 assessed.revoked_copies =
10147 retire::revoked_copies(&self.config.state_root, &self.config.models_dir);
10148 assessed
10149 }
10150
10151 fn revoked_pins(&self, registry: &UnifiedRegistry) -> Vec<portfolio::RevokedPin> {
10159 let pins = self
10160 .external_pins
10161 .read()
10162 .unwrap_or_else(std::sync::PoisonError::into_inner)
10163 .clone();
10164 if pins.is_empty() {
10165 return Vec::new();
10166 }
10167 let names =
10168 catalog::load_revoked_names(&catalog::revoked_models_path(&self.config.state_root));
10169 pins.into_iter()
10170 .filter_map(|(pin, mut agents)| {
10171 let resolved = registry
10172 .get(&pin)
10173 .or_else(|| registry.find_by_name(&pin))
10174 .map(|schema| schema.id.clone());
10175 let model_id = if registry.is_revoked(&pin) {
10176 pin.clone()
10177 } else if let Some(id) = resolved {
10178 id
10179 } else {
10180 names.get(&pin)?.clone()
10181 };
10182 if !registry.is_revoked(&model_id) {
10183 return None;
10184 }
10185 agents.sort();
10186 agents.dedup();
10187 Some(portfolio::RevokedPin {
10188 pin,
10189 model_id,
10190 agents,
10191 })
10192 })
10193 .collect()
10194 }
10195
10196 fn protected_models(
10203 &self,
10204 registry: &UnifiedRegistry,
10205 include_lanes: bool,
10206 ) -> std::collections::BTreeMap<String, String> {
10207 let mut out = std::collections::BTreeMap::new();
10208 let mut protect = |key: &str, why: String| {
10209 let Some(schema) = registry.get(key).or_else(|| registry.find_by_name(key)) else {
10210 return;
10211 };
10212 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
10215 if let Some(twin) = registry.resolve_mlx_equivalent(schema) {
10216 out.entry(twin.id.clone()).or_insert_with(|| why.clone());
10217 }
10218 out.entry(schema.id.clone()).or_insert(why);
10219 };
10220 let lanes = if include_lanes {
10221 self.lane_defaults().defaults
10222 } else {
10223 Vec::new()
10224 };
10225 for lane in lanes {
10226 let use_case = serde_json::to_value(lane.use_case)
10227 .ok()
10228 .and_then(|v| v.as_str().map(str::to_owned))
10229 .unwrap_or_else(|| format!("{:?}", lane.use_case));
10230 protect(&lane.model_id, format!("the {use_case} lane default"));
10231 }
10232 let config = &self.config;
10233 for (key, what) in [
10234 (Some(&config.generation_model), "the generation model"),
10235 (
10236 config.preferred_generation_model.as_ref(),
10237 "the preferred generation model",
10238 ),
10239 (Some(&config.embedding_model), "the embedding model"),
10240 (
10241 config.preferred_embedding_model.as_ref(),
10242 "the preferred embedding model",
10243 ),
10244 (
10245 Some(&config.classification_model),
10246 "the classification model",
10247 ),
10248 (
10249 config.preferred_classification_model.as_ref(),
10250 "the preferred classification model",
10251 ),
10252 ] {
10253 if let Some(key) = key {
10254 protect(key, format!("{what} in CAR's configuration"));
10255 }
10256 }
10257 for (capability, what) in [
10258 (
10259 ModelCapability::SpeechToText,
10260 "the local speech-to-text default",
10261 ),
10262 (
10263 ModelCapability::TextToSpeech,
10264 "the local text-to-speech default",
10265 ),
10266 ] {
10267 if let Some(name) = self.speech_health_default_name(capability, true, false) {
10268 protect(&name, what.to_string());
10269 }
10270 }
10271 protect(
10272 LOCAL_TOOL_MODEL_PULL_ID,
10273 "the local tool model CAR's setup guidance installs".to_string(),
10274 );
10275 let pins = self
10276 .external_pins
10277 .read()
10278 .unwrap_or_else(std::sync::PoisonError::into_inner)
10279 .clone();
10280 for (key, agents) in pins {
10281 protect(&key, pinned_by(&agents));
10282 }
10283 out
10284 }
10285
10286 pub async fn resume_retirements(&self) -> Vec<String> {
10291 let _concierge = self.concierge_action_lock.lock().await;
10292 let mut errors: Vec<String> = self
10293 .model_management
10294 .resume_detached_trees()
10295 .into_iter()
10296 .map(|e| e.to_string())
10297 .collect();
10298 errors.extend(
10299 self.model_management
10300 .resume_retiring()
10301 .into_iter()
10302 .map(|e| e.to_string()),
10303 );
10304 errors
10305 }
10306
10307 pub fn model_management_store(&self) -> &model_management::ModelManagementStore {
10308 &self.model_management
10309 }
10310
10311 fn ensure_model_enabled(&self, model_id: &str) -> Result<(), InferenceError> {
10312 if self.model_management.car_enabled(model_id)? {
10313 return Ok(());
10314 }
10315 Err(InferenceError::InferenceFailed(format!(
10316 "local model {model_id} was removed from CAR; reinstall it before use"
10317 )))
10318 }
10319
10320 pub fn available_model_upgrades(&self) -> Vec<ModelUpgrade> {
10322 self.unified_registry().available_upgrades()
10323 }
10324
10325 pub async fn check_upgrade_nudge(
10331 &self,
10332 inference_active: bool,
10333 ) -> (crate::nudge::NudgeDecision, crate::nudge::NudgeState) {
10334 let findings = self.detect_upgrades().await;
10335 let mut prefs = self.update_prefs();
10336 prefs.policy = self.effective_update_policy(&prefs, false).0;
10337 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10338 let now = std::time::SystemTime::now()
10339 .duration_since(std::time::UNIX_EPOCH)
10340 .map(|d| d.as_secs())
10341 .unwrap_or(0);
10342 let decision = crate::nudge::decide_nudge(
10343 &findings,
10344 &prefs,
10345 &state,
10346 now,
10347 crate::nudge::DEFAULT_THROTTLE_SECS,
10348 inference_active,
10349 );
10350 (decision, state)
10351 }
10352
10353 pub fn dismiss_upgrade_nudge(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10356 let path = crate::nudge::NudgeState::default_path();
10357 let mut state = crate::nudge::NudgeState::load_from(&path);
10358 state.dismiss(dismiss_key);
10359 state
10360 .save_to(&path)
10361 .map_err(InferenceError::InferenceFailed)
10362 }
10363
10364 pub async fn check_concierge(
10371 &self,
10372 inference_active: bool,
10373 ) -> (
10374 Vec<crate::concierge::ConciergeSuggestion>,
10375 crate::nudge::NudgeState,
10376 ) {
10377 let prefs = self.update_prefs();
10378 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10379 let hw = crate::hardware::HardwareInfo::detect();
10380 let schemas = self.list_schemas();
10381 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10382 let now = std::time::SystemTime::now()
10383 .duration_since(std::time::UNIX_EPOCH)
10384 .map(|d| d.as_secs())
10385 .unwrap_or(0);
10386 let suggestions = crate::concierge::decide_concierge(
10387 &refs,
10388 &hw,
10389 crate::concierge::DEFAULT_WATCHED_USE_CASES,
10390 crate::intent::QualityTier::Balanced,
10391 &prefs,
10392 &state,
10393 now,
10394 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
10395 inference_active,
10396 );
10397 (suggestions, state)
10398 }
10399
10400 pub fn dismiss_concierge_suggestion(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10405 self.dismiss_upgrade_nudge(dismiss_key)
10406 }
10407
10408 pub fn dismiss_concierge_labeled(
10412 &self,
10413 dismiss_key: &str,
10414 reason: crate::concierge::DismissReason,
10415 ) -> Result<(), String> {
10416 let path = crate::nudge::NudgeState::default_path();
10417 let mut state = crate::nudge::NudgeState::load_from(&path);
10418 let now = std::time::SystemTime::now()
10419 .duration_since(std::time::UNIX_EPOCH)
10420 .map(|d| d.as_secs())
10421 .unwrap_or(0);
10422 state.dismiss_labeled(dismiss_key, reason, now);
10423 state.save_to(&path).map_err(|e| e.to_string())
10424 }
10425
10426 pub async fn check_canaries(&self) -> Vec<crate::intent::UseCase> {
10434 use crate::action_ledger::ConciergeActionKind;
10435 use crate::concierge::{
10436 canary_verdict, CanaryVerdict, CANARY_MIN_SAMPLES, CANARY_REGRESSION_MARGIN,
10437 };
10438 use std::collections::BTreeMap;
10439
10440 let actions = self.concierge_actions(0);
10441 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
10442 let entries = crate::outcome::read_ledger(&ledger_path, 0);
10443
10444 let mut latest: BTreeMap<
10446 crate::intent::UseCase,
10447 &crate::action_ledger::ConciergeActionEntry,
10448 > = BTreeMap::new();
10449 for a in &actions {
10450 if a.project.is_some() {
10451 continue; }
10453 if let Some(uc) = a.use_case {
10454 latest.insert(uc, a);
10455 }
10456 }
10457
10458 let mut reverts: Vec<(crate::intent::UseCase, u64)> = Vec::new();
10463 let tracker = self.outcome_tracker.read().await;
10464 for (uc, a) in latest {
10465 if a.kind != ConciergeActionKind::SetDefault {
10468 continue;
10469 }
10470 let Some(prior) = a.prior_model_id.as_deref() else {
10471 continue; };
10473
10474 let (mut succ, mut total) = (0u64, 0u64);
10476 for e in &entries {
10477 if e.timestamp < a.timestamp
10478 || e.model_id != a.model_id
10479 || crate::usage_profile::use_case_for_task(e.task) != uc
10480 {
10481 continue;
10482 }
10483 match e.success {
10484 Some(true) => {
10485 succ += 1;
10486 total += 1;
10487 }
10488 Some(false) => total += 1,
10489 None => {}
10490 }
10491 }
10492 let new_rate = if total == 0 {
10493 None
10494 } else {
10495 Some(succ as f64 / total as f64)
10496 };
10497
10498 let (mut base_succ, mut base_total) = (0u64, 0u64);
10503 if let Some(profile) = tracker.profile(prior) {
10504 for t in [
10505 crate::outcome::InferenceTask::Generate,
10506 crate::outcome::InferenceTask::Embed,
10507 crate::outcome::InferenceTask::Classify,
10508 crate::outcome::InferenceTask::Code,
10509 crate::outcome::InferenceTask::Reasoning,
10510 ] {
10511 if crate::usage_profile::use_case_for_task(t) != uc {
10512 continue;
10513 }
10514 if let Some(ts) = profile.task_stats(t) {
10515 base_succ += ts.successes;
10516 base_total += ts.successes + ts.failures;
10517 }
10518 }
10519 }
10520 if base_total == 0 {
10523 continue;
10524 }
10525 let baseline = base_succ as f64 / base_total as f64;
10526
10527 if canary_verdict(
10528 new_rate,
10529 total,
10530 baseline,
10531 CANARY_MIN_SAMPLES,
10532 CANARY_REGRESSION_MARGIN,
10533 ) == CanaryVerdict::Revert
10534 {
10535 reverts.push((uc, a.seq));
10539 }
10540 }
10541 drop(tracker);
10542
10543 let mut reverted = Vec::new();
10546 for (uc, seq) in reverts {
10547 if self.rollback_lane_inner(uc, None, Some(seq)).await.is_ok() {
10548 tracing::info!(lane = ?uc, "concierge canary: auto-reverted a worse model switch");
10549 reverted.push(uc);
10550 }
10551 }
10552 reverted
10553 }
10554
10555 pub async fn concierge_ask(
10572 &self,
10573 question: Option<&str>,
10574 ) -> Result<concierge_answer::ConciergeAnswer, String> {
10575 let maintenance = self.maintenance_pass(true).await;
10576 let portfolio = match maintenance.portfolio.clone() {
10577 Some(portfolio) => portfolio,
10578 None => self.portfolio().await,
10579 };
10580 let memory_policy =
10581 serde_json::to_value(&self.active_local_resource_policy().policy.profile)
10582 .ok()
10583 .and_then(|v| v.as_str().map(str::to_owned))
10584 .unwrap_or_else(|| "unknown".into());
10585 let summary = concierge_answer::summarize(&portfolio, &maintenance, &memory_policy);
10586 let Some(question) = question else {
10587 return Ok(concierge_answer::ConciergeAnswer {
10588 answer: summary.clone(),
10589 summary,
10590 model_used: None,
10591 ungrounded: Vec::new(),
10592 });
10593 };
10594 use std::fmt::Write as _;
10595 let status = self.concierge_status(false).await;
10596 let hw = crate::hardware::HardwareInfo::detect();
10597 let schemas = self.list_schemas();
10598 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10599
10600 let mut evidence = String::new();
10601 evidence.push_str("OBSERVED USAGE (last 30 days):\n");
10602 if status.lanes.is_empty() {
10603 evidence.push_str(" (no usage recorded yet)\n");
10604 }
10605 for lane in &status.lanes {
10606 let rate = lane
10607 .success_rate()
10608 .map(|r| format!("{:.0}% success", r * 100.0))
10609 .unwrap_or_else(|| "no resolved signal".into());
10610 let _ = writeln!(
10611 evidence,
10612 " {:?}: {} calls, {}{}",
10613 lane.use_case,
10614 lane.calls,
10615 rate,
10616 if lane.failing_models.is_empty() {
10617 String::new()
10618 } else {
10619 format!(
10620 ", failing on {}",
10621 lane.failing_models
10622 .iter()
10623 .cloned()
10624 .collect::<Vec<_>>()
10625 .join(", ")
10626 )
10627 }
10628 );
10629 }
10630 evidence.push_str("\nMODEL HEALTH:\n");
10631 for m in &status.models {
10632 let success = match m.success_rate {
10633 Some(r) => format!("{:.0}% success", r * 100.0),
10634 None => "no resolved signal".to_string(),
10635 };
10636 let _ = writeln!(
10637 evidence,
10638 " {}: {} calls, {}, {:.0}ms avg{}",
10639 m.model_id,
10640 m.calls,
10641 success,
10642 m.avg_latency_ms,
10643 if m.excluded { " (excluded)" } else { "" }
10644 );
10645 }
10646 let mut menu_lanes: Vec<crate::intent::UseCase> =
10652 crate::concierge::DEFAULT_WATCHED_USE_CASES.to_vec();
10653 for lane in &status.lanes {
10654 if !menu_lanes.contains(&lane.use_case) {
10655 menu_lanes.push(lane.use_case);
10656 }
10657 }
10658 evidence.push_str("\nGROUNDED CANDIDATES (fit verified for this machine):\n");
10659 for uc in menu_lanes {
10660 let set = crate::recommend::recommend(
10661 &refs,
10662 &hw,
10663 uc,
10664 crate::intent::QualityTier::Balanced,
10665 crate::intent::Privacy::OnDevice,
10666 );
10667 for p in set.picks.iter().take(3) {
10668 let _ = writeln!(
10669 evidence,
10670 " [{:?}] {} — {}{}",
10671 uc,
10672 p.display_name,
10673 if p.already_installed {
10674 "installed"
10675 } else {
10676 "available"
10677 },
10678 if p.fit == crate::recommend::FitStatus::Fits {
10679 ", fits"
10680 } else {
10681 ", does NOT fit"
10682 }
10683 );
10684 }
10685 }
10686 if let Some(s) = &status.decision.suggestion {
10687 let _ = writeln!(evidence, "\nCURRENT SUGGESTION: {}", s.message);
10688 }
10689
10690 evidence.push_str("\nLOCAL MODELS ON THIS MACHINE:\n");
10691 evidence.push_str(&summary);
10692
10693 let prompt = format!(
10694 "You are CAR's model concierge. Answer the user's question ONLY from the \
10695 EVIDENCE provided as context: this machine's local models and what each is \
10696 doing, what maintenance would do, observed usage and health, and the \
10697 grounded candidate menu (the only models you may recommend acquiring). \
10698 Name models by the ids the evidence uses. NEVER invent a model and NEVER \
10699 claim a model fits or is better than the evidence states. If the evidence \
10700 doesn't answer the question, say so. Answer in a few short sentences or \
10701 bullets.\n\nQUESTION: {question}"
10702 );
10703 let req = crate::tasks::generate::GenerateRequest {
10704 prompt,
10705 context: Some(evidence.clone()),
10706 intent: Some(crate::intent::IntentHint {
10707 task: Some(crate::intent::TaskHint::Chat),
10708 prefer_local: true,
10709 ..Default::default()
10710 }),
10711 params: crate::tasks::generate::GenerateParams {
10712 temperature: 0.2,
10713 max_tokens: 600,
10714 workload: crate::tasks::generate::RoutingWorkload::Background,
10715 ..Default::default()
10716 },
10717 ..Default::default()
10718 };
10719 match self.generate_tracked(req).await {
10720 Ok(result) => {
10721 let reply = concierge_answer::trim_repetition(&result.text);
10722 let ungrounded = concierge_answer::ungrounded_ids(&reply, &evidence);
10723 let answer = if ungrounded.is_empty() {
10724 reply
10725 } else {
10726 format!(
10727 "{reply}\n\n(Not verified for this machine: {}. Only the models listed \
10728 above are checked.)",
10729 ungrounded.join(", ")
10730 )
10731 };
10732 Ok(concierge_answer::ConciergeAnswer {
10733 answer,
10734 summary,
10735 model_used: Some(result.model_used),
10736 ungrounded,
10737 })
10738 }
10739 Err(error) => {
10740 tracing::warn!(%error, "concierge.ask: no model could answer; returning the facts");
10741 Ok(concierge_answer::ConciergeAnswer {
10742 answer: format!(
10743 "{summary}\n(No model could answer the question itself: {error}.)"
10744 ),
10745 summary,
10746 model_used: None,
10747 ungrounded: Vec::new(),
10748 })
10749 }
10750 }
10751 }
10752
10753 pub async fn refresh_catalog(&self) -> Result<usize, String> {
10763 let url = catalog::catalog_url();
10764 let pubkey = catalog::catalog_public_key();
10765 let verified = self.fetch_checked_catalog(&url, &pubkey).await?;
10770 let path = crate::catalog::cache_path(&self.config.state_root);
10771 self.adopt_verified_catalog(&path, &verified, &pubkey).await
10772 }
10773
10774 async fn fetch_checked_catalog(
10779 &self,
10780 url: &str,
10781 pubkey: &str,
10782 ) -> Result<catalog::VerifiedCatalog, String> {
10783 use catalog::CheckFailure::{Older, Rejected, Unreachable};
10784 let http = crate::tls_client::catalog_refresh_client();
10785 let fetched = match crate::catalog::fetch_signed(&http, url).await {
10788 Err(error) => Err((Unreachable, error)),
10789 Ok((bytes, sig)) => {
10790 crate::catalog::verify_fetched(bytes, sig, pubkey).map_err(|e| (Rejected, e))
10791 }
10792 };
10793 let cached = crate::catalog::load_doc(
10797 &crate::catalog::cache_path(&self.config.state_root),
10798 Some(pubkey),
10799 )
10800 .map(|doc| doc.version);
10801 let outcome = match &fetched {
10802 Ok(verified) => match cached {
10803 Some(cached) if verified.version() < cached => Err((
10804 Older,
10805 format!(
10806 "the source served catalog v{}, older than the cached v{cached}",
10807 verified.version()
10808 ),
10809 )),
10810 _ => Ok(verified.version()),
10811 },
10812 Err((failure, error)) => Err((*failure, error.clone())),
10813 };
10814 catalog::record_check(
10815 &catalog::status_path(&self.config.state_root),
10816 url,
10817 outcome
10818 .as_ref()
10819 .map(|v| *v)
10820 .map_err(|(f, e)| (*f, e.as_str())),
10821 now_unix(),
10822 );
10823 fetched.map_err(|(_, error)| error)
10824 }
10825
10826 pub fn mark_catalog_loop_alive(&self) {
10828 catalog::record_loop_alive(&catalog::status_path(&self.config.state_root), now_unix());
10829 }
10830
10831 pub fn mark_daemon_alive_for_catalog(&self) {
10834 catalog::record_daemon_alive(&catalog::status_path(&self.config.state_root), now_unix());
10835 }
10836
10837 pub fn catalog_freshness(&self) -> catalog::CatalogFreshness {
10840 catalog::catalog_freshness(&catalog::load_status(&catalog::status_path(
10841 &self.config.state_root,
10842 )))
10843 }
10844
10845 pub async fn refresh_catalog_on_schedule(&self) -> Result<Option<usize>, String> {
10850 if matches!(self.update_prefs().policy, update_prefs::UpdatePolicy::Off) {
10851 catalog::record_off(&catalog::status_path(&self.config.state_root), now_unix());
10852 return Ok(None);
10853 }
10854 self.refresh_catalog_from(&catalog::catalog_url(), &catalog::catalog_public_key())
10855 .await
10856 }
10857
10858 async fn refresh_catalog_from(&self, url: &str, pubkey: &str) -> Result<Option<usize>, String> {
10860 let verified = self.fetch_checked_catalog(url, pubkey).await?;
10861 let path = crate::catalog::cache_path(&self.config.state_root);
10862 let cached = crate::catalog::load_doc(&path, Some(pubkey)).map(|doc| doc.version);
10863 if cached.is_some_and(|version| verified.version() <= version) {
10864 return Ok(None);
10865 }
10866 self.adopt_verified_catalog(&path, &verified, pubkey)
10867 .await
10868 .map(Some)
10869 }
10870
10871 async fn adopt_verified_catalog(
10878 &self,
10879 path: &Path,
10880 verified: &catalog::VerifiedCatalog,
10881 pubkey: &str,
10882 ) -> Result<usize, String> {
10883 let count = crate::catalog::install_if_newer(path, verified, pubkey).await?;
10884 let newest = crate::catalog::load_verified(path, Some(pubkey))
10885 .ok_or_else(|| format!("re-reading the verified catalog at {}", path.display()))?;
10886 let revoked = newest.revoked().to_vec();
10887 self.unlink_revoked_signed_models(&revoked).await;
10888 self.swap_signed_catalog(newest);
10889 self.clear_revoked_lane_defaults(&revoked).await;
10890 Ok(count)
10891 }
10892
10893 pub async fn enforce_catalog_revocations(&self) {
10898 let revoked = self.current_registry().revoked_ids();
10899 self.clear_revoked_lane_defaults(&revoked).await;
10900 }
10901
10902 async fn clear_revoked_lane_defaults(&self, revoked: &[String]) {
10910 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
10911 if revoked.is_empty() {
10912 return;
10913 }
10914 let _guard = self.concierge_action_lock.lock().await;
10915 let now = std::time::SystemTime::now()
10916 .duration_since(std::time::UNIX_EPOCH)
10917 .map(|d| d.as_secs())
10918 .unwrap_or(0);
10919 for lane in self.lane_defaults().defaults {
10920 if !revoked.contains(&lane.model_id) {
10921 continue;
10922 }
10923 match self.clear_lane_default(lane.project.as_deref(), lane.use_case) {
10924 Ok(true) => self.record_concierge_action(ConciergeActionEntry {
10925 seq: 0,
10926 kind: ConciergeActionKind::RevokedLaneCleared,
10927 model_id: lane.model_id.clone(),
10928 use_case: Some(lane.use_case),
10929 project: lane.project.clone(),
10930 prior_model_id: Some(lane.model_id.clone()),
10931 detail: "revoked by the signed model catalog".into(),
10932 timestamp: now,
10933 }),
10934 Ok(false) => {}
10935 Err(error) => {
10936 tracing::warn!(model = %lane.model_id, %error, "could not clear a revoked lane default")
10937 }
10938 }
10939 }
10940 for pin in self.revoked_pins(&self.current_registry()) {
10941 tracing::warn!(model = %pin.model_id, pin = %pin.pin, agents = ?pin.agents, "a pinned model was revoked by the signed catalog; re-point it");
10942 }
10943 }
10944
10945 async fn unlink_revoked_signed_models(&self, revoked: &[String]) {
10954 let registry = self.current_registry();
10955 let names_path = catalog::revoked_names_path(&self.config.state_root);
10956 let mut names = catalog::load_revoked_names(&names_path);
10957 let models_path = catalog::revoked_models_path(&self.config.state_root);
10958 let mut models = catalog::load_revoked_names(&models_path);
10959 names.retain(|_, id| revoked.contains(id));
10961 models.retain(|_, id| revoked.contains(id));
10962 let mut installed = Vec::new();
10963 for id in revoked {
10964 let Some(row) = registry.get(id) else {
10965 continue;
10966 };
10967 if registry.signed_origin_of(id).is_none() || registry.is_builtin(id) {
10968 continue;
10969 }
10970 if !retire::single_component(&row.name) {
10975 continue;
10976 }
10977 models.insert(row.name.clone(), id.clone());
10978 if row.is_local() && row.downloads_weights() {
10979 names.insert(row.name.clone(), id.clone());
10980 installed.push((id.clone(), row.name.clone()));
10981 }
10982 }
10983 for (path, record) in [(&names_path, &names), (&models_path, &models)] {
10984 if let Err(error) = catalog::save_revoked_names(path, record) {
10985 tracing::warn!(path = %path.display(), %error, "could not record revoked model names");
10986 }
10987 }
10988 let models_dir = self.config.models_dir.clone();
10989 for (id, name) in installed {
10990 let projection = models_dir.join(&name);
10991 if std::fs::symlink_metadata(&projection).is_err() || !retire::links_only(&projection) {
10992 continue;
10993 }
10994 match self.model_management.load_receipt(&id) {
10995 Ok(Some(receipt))
10996 if receipt.artifact_kind == model_management::ManagedArtifactKind::Symlink =>
10997 {
10998 if let Err(error) = self.remove_model_from_car(&id).await {
10999 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
11000 }
11001 }
11002 Ok(Some(_)) => {}
11003 Ok(None) => {
11004 if let Err(error) = retire::remove_links_only_tree(&projection, &models_dir) {
11005 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
11006 }
11007 }
11008 Err(error) => {
11009 tracing::warn!(%id, %error, "could not read a revoked model's receipt")
11010 }
11011 }
11012 }
11013 }
11014
11015 pub async fn discover_models(&self) -> Result<usize, String> {
11024 use crate::schema::{ModelSource, TrustTier};
11025 let template = self
11032 .unified_registry()
11033 .all()
11034 .filter(|m| {
11035 m.provider.eq_ignore_ascii_case("openai")
11036 && m.trust_tier == TrustTier::Curated
11037 && matches!(m.source, ModelSource::RemoteApi { .. })
11038 })
11039 .max_by(|a, b| {
11040 a.capabilities
11041 .len()
11042 .cmp(&b.capabilities.len())
11043 .then_with(|| a.id.cmp(&b.id))
11044 })
11045 .cloned();
11046 let Some(template) = template else {
11047 return Ok(0); };
11049 let (endpoint, api_key_env) = match &template.source {
11050 ModelSource::RemoteApi {
11051 endpoint,
11052 api_key_env,
11053 ..
11054 } => (endpoint.clone(), api_key_env.clone()),
11055 _ => return Ok(0),
11056 };
11057 let Some(models_url) = crate::discovery::models_url_from_endpoint(&endpoint) else {
11058 return Ok(0);
11059 };
11060 let key = match car_secrets::resolve_env_or_keychain(&api_key_env) {
11061 Some(k) if !k.is_empty() => k,
11062 _ => return Ok(0), };
11064
11065 let http = reqwest::Client::builder()
11069 .timeout(std::time::Duration::from_secs(20))
11070 .build()
11071 .map_err(|e| format!("discovery client: {e}"))?;
11072 let ids = crate::discovery::fetch_model_ids(&http, &models_url, &key).await?;
11073
11074 let cache = crate::discovery::cache_path(&self.config.state_models_dir());
11076 let mut cached = crate::discovery::load_cache(&cache);
11077 let mut have: std::collections::HashSet<String> =
11078 cached.iter().map(|m| m.id.clone()).collect();
11079 for id in ids {
11080 if !crate::discovery::is_chat_model(&id) {
11081 continue;
11082 }
11083 let schema = crate::discovery::discovered_schema("openai", &id, &template);
11084 if self.unified_registry().get(&schema.id).is_some() {
11090 continue;
11091 }
11092 if have.insert(schema.id.clone()) {
11093 cached.push(schema);
11094 }
11095 }
11096 let count = cached.len();
11097 crate::discovery::save_cache(&cache, &cached)?;
11098 Ok(count)
11099 }
11100
11101 pub fn lane_defaults(&self) -> crate::lane_defaults::LaneDefaults {
11103 self.lane_defaults_cache.read().unwrap().clone()
11104 }
11105
11106 pub fn lane_default(
11110 &self,
11111 project: Option<&str>,
11112 use_case: crate::intent::UseCase,
11113 ) -> Option<String> {
11114 self.lane_defaults_cache
11115 .read()
11116 .unwrap()
11117 .resolve(project, use_case)
11118 .map(str::to_string)
11119 }
11120
11121 fn model_is_excluded(
11122 exclude_set: &std::collections::HashSet<String>,
11123 registry: &UnifiedRegistry,
11124 model: &str,
11125 ) -> bool {
11126 registry
11127 .get(model)
11128 .or_else(|| registry.find_by_name(model))
11129 .map(|schema| exclude_set.contains(&schema.id))
11130 .unwrap_or_else(|| exclude_set.contains(model))
11131 }
11132
11133 fn lane_pin_for(
11139 &self,
11140 req: &GenerateRequest,
11141 routing_registry: &UnifiedRegistry,
11142 ) -> Option<String> {
11143 let task = req.intent.as_ref().and_then(|h| h.task)?;
11144 let use_case = crate::usage_profile::use_case_for_task_hint(task);
11145 let id = self.lane_default(None, use_case)?;
11146 let known = routing_registry
11151 .get(&id)
11152 .or_else(|| routing_registry.find_by_name(&id));
11153 match known {
11154 Some(s) if s.available_now() => Some(id),
11155 _ => None,
11156 }
11157 }
11158
11159 pub fn set_lane_default(
11162 &self,
11163 project: Option<String>,
11164 use_case: crate::intent::UseCase,
11165 model_id: &str,
11166 ) -> Result<(), String> {
11167 if self.current_registry().is_revoked(model_id) {
11170 return Err(format!(
11171 "{model_id} was revoked by the signed model catalog and cannot be a lane default"
11172 ));
11173 }
11174 let now = std::time::SystemTime::now()
11175 .duration_since(std::time::UNIX_EPOCH)
11176 .map(|d| d.as_secs())
11177 .unwrap_or(0);
11178 let mut defaults = self.lane_defaults_cache.write().unwrap();
11181 defaults.set(project, use_case, model_id.to_string(), now);
11182 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11183 .map_err(|e| e.to_string())
11184 }
11185
11186 pub fn clear_lane_default(
11189 &self,
11190 project: Option<&str>,
11191 use_case: crate::intent::UseCase,
11192 ) -> Result<bool, String> {
11193 let mut defaults = self.lane_defaults_cache.write().unwrap();
11194 let removed = defaults.clear(project, use_case);
11195 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11196 .map_err(|e| e.to_string())?;
11197 Ok(removed)
11198 }
11199
11200 pub async fn user_set_lane_default(
11208 &self,
11209 use_case: crate::intent::UseCase,
11210 model_id: &str,
11211 project: Option<String>,
11212 ) -> Result<(), String> {
11213 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11214 let _guard = self.concierge_action_lock.lock().await;
11215 let prior = self.lane_default(project.as_deref(), use_case);
11216 self.set_lane_default(project.clone(), use_case, model_id)?;
11217 self.record_concierge_action(ConciergeActionEntry {
11218 seq: 0,
11219 kind: ConciergeActionKind::SetDefault,
11220 model_id: model_id.to_string(),
11221 use_case: Some(use_case),
11222 project,
11223 prior_model_id: prior,
11224 detail: "user set lane default".into(),
11225 timestamp: now_unix(),
11226 });
11227 Ok(())
11228 }
11229
11230 pub async fn user_clear_lane_default(
11235 &self,
11236 use_case: crate::intent::UseCase,
11237 project: Option<String>,
11238 ) -> Result<bool, String> {
11239 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11240 let _guard = self.concierge_action_lock.lock().await;
11241 let prior = self.lane_default(project.as_deref(), use_case);
11242 let removed = self.clear_lane_default(project.as_deref(), use_case)?;
11243 if removed {
11244 self.record_concierge_action(ConciergeActionEntry {
11245 seq: 0,
11246 kind: ConciergeActionKind::ClearDefault,
11247 model_id: prior.clone().unwrap_or_default(),
11248 use_case: Some(use_case),
11249 project,
11250 prior_model_id: prior,
11251 detail: "user cleared lane default".into(),
11252 timestamp: now_unix(),
11253 });
11254 }
11255 Ok(removed)
11256 }
11257
11258 pub fn concierge_actions(
11260 &self,
11261 limit: usize,
11262 ) -> Vec<crate::action_ledger::ConciergeActionEntry> {
11263 crate::action_ledger::read_actions(&self.action_ledger_path(), limit)
11264 }
11265
11266 fn lane_defaults_path(&self) -> PathBuf {
11268 self.config
11269 .state_root
11270 .join(crate::lane_defaults::LANE_DEFAULTS_FILE)
11271 }
11272
11273 fn action_ledger_path(&self) -> PathBuf {
11276 self.config
11277 .state_root
11278 .join(crate::action_ledger::ACTION_LEDGER_FILE)
11279 }
11280
11281 fn record_concierge_action(&self, mut entry: crate::action_ledger::ConciergeActionEntry) {
11282 entry.seq = self
11283 .concierge_action_seq
11284 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
11285 if let Err(e) = crate::action_ledger::append_action(&self.action_ledger_path(), &entry) {
11286 tracing::debug!("record concierge action failed: {e}");
11287 }
11288 }
11289
11290 pub async fn apply_concierge(
11302 &self,
11303 use_case: crate::intent::UseCase,
11304 model_id: &str,
11305 project: Option<String>,
11306 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11307 self.with_pinned_registry(self.apply_concierge_unpinned(use_case, model_id, project))
11308 .await
11309 }
11310
11311 async fn apply_concierge_unpinned(
11312 &self,
11313 use_case: crate::intent::UseCase,
11314 model_id: &str,
11315 project: Option<String>,
11316 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11317 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11318 let now = || {
11319 std::time::SystemTime::now()
11320 .duration_since(std::time::UNIX_EPOCH)
11321 .map(|d| d.as_secs())
11322 .unwrap_or(0)
11323 };
11324 if let Some(schema) = self
11329 .list_schemas()
11330 .into_iter()
11331 .find(|s| s.id == model_id || s.name == model_id)
11332 {
11333 let serves = use_case
11334 .required_capabilities()
11335 .iter()
11336 .all(|c| schema.capabilities.contains(c));
11337 if !serves {
11338 return Err(format!(
11339 "model '{model_id}' does not serve the {use_case:?} lane"
11340 ));
11341 }
11342 }
11343
11344 if self
11352 .unified_registry()
11353 .existing_local_artifact(model_id)
11354 .is_none()
11355 {
11356 self.pull_model(model_id).await.map_err(|e| e.to_string())?;
11357 }
11358
11359 let _guard = self.concierge_action_lock.lock().await;
11362 let prior = self.lane_default(project.as_deref(), use_case);
11365 self.record_concierge_action(ConciergeActionEntry {
11366 seq: 0, kind: ConciergeActionKind::Install,
11368 model_id: model_id.to_string(),
11369 use_case: Some(use_case),
11370 project: project.clone(),
11371 prior_model_id: None,
11372 detail: "concierge apply: ensured model present".into(),
11373 timestamp: now(),
11374 });
11375
11376 self.set_lane_default(project.clone(), use_case, model_id)?;
11378 self.record_concierge_action(ConciergeActionEntry {
11379 seq: 0, kind: ConciergeActionKind::SetDefault,
11381 model_id: model_id.to_string(),
11382 use_case: Some(use_case),
11383 project: project.clone(),
11384 prior_model_id: prior.clone(),
11385 detail: "concierge apply: set lane default".into(),
11386 timestamp: now(),
11387 });
11388
11389 Ok(crate::action_ledger::ConciergeApplyResult {
11390 model_id: model_id.to_string(),
11391 use_case,
11392 installed: true,
11393 set_default: true,
11394 prior_model_id: prior,
11395 })
11396 }
11397
11398 pub async fn rollback_lane(
11403 &self,
11404 use_case: crate::intent::UseCase,
11405 project: Option<String>,
11406 ) -> Result<Option<String>, String> {
11407 self.rollback_lane_inner(use_case, project, None).await
11408 }
11409
11410 async fn rollback_lane_inner(
11416 &self,
11417 use_case: crate::intent::UseCase,
11418 project: Option<String>,
11419 expected_anchor_seq: Option<u64>,
11420 ) -> Result<Option<String>, String> {
11421 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11422 let _guard = self.concierge_action_lock.lock().await;
11423 let actions = self.concierge_actions(0);
11428 let anchor = actions.iter().rev().find(|a| {
11429 matches!(
11430 a.kind,
11431 ConciergeActionKind::SetDefault | ConciergeActionKind::Rollback
11432 ) && a.use_case == Some(use_case)
11433 && a.project == project
11434 });
11435 let set = match anchor {
11436 None => return Err("no prior set-default to roll back".into()),
11437 Some(a) if a.kind == ConciergeActionKind::Rollback => {
11438 return Err("already rolled back to the prior default; nothing to undo".into())
11439 }
11440 Some(a) => a,
11441 };
11442 if let Some(seq) = expected_anchor_seq {
11445 if set.seq != seq {
11446 return Err("lane default changed since the canary decision; not reverting".into());
11447 }
11448 }
11449 let prior = set.prior_model_id.clone();
11450 let now = std::time::SystemTime::now()
11451 .duration_since(std::time::UNIX_EPOCH)
11452 .map(|d| d.as_secs())
11453 .unwrap_or(0);
11454
11455 match &prior {
11456 Some(m) => self.set_lane_default(project.clone(), use_case, m)?,
11457 None => {
11458 self.clear_lane_default(project.as_deref(), use_case)?;
11459 }
11460 }
11461 let detail = match &prior {
11462 Some(_) => "concierge rollback: restored prior lane default",
11463 None => "concierge rollback: cleared lane default (no prior)",
11464 };
11465 self.record_concierge_action(ConciergeActionEntry {
11466 seq: 0, kind: ConciergeActionKind::Rollback,
11468 model_id: prior.clone().unwrap_or_default(),
11469 use_case: Some(use_case),
11470 project,
11471 prior_model_id: Some(set.model_id.clone()),
11472 detail: detail.into(),
11473 timestamp: now,
11474 });
11475 Ok(prior)
11476 }
11477
11478 pub async fn concierge_status(
11483 &self,
11484 inference_active: bool,
11485 ) -> crate::concierge::ConciergeStatus {
11486 const USAGE_WINDOW_SECS: u64 = 30 * 24 * 60 * 60;
11488
11489 let prefs = self.update_prefs();
11490 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
11491 let hw = crate::hardware::HardwareInfo::detect();
11492 let schemas = self.list_schemas();
11493 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11494 let now = std::time::SystemTime::now()
11495 .duration_since(std::time::UNIX_EPOCH)
11496 .map(|d| d.as_secs())
11497 .unwrap_or(0);
11498
11499 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
11500 let entries = crate::outcome::read_ledger(&ledger_path, 0);
11501 let usage =
11502 crate::usage_profile::UsageProfile::from_ledger(&entries, now, USAGE_WINDOW_SECS);
11503
11504 let decision = crate::concierge::evaluate_concierge(
11505 &refs,
11506 &hw,
11507 &usage,
11508 crate::intent::QualityTier::Balanced,
11509 &prefs,
11510 &state,
11511 now,
11512 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
11513 inference_active,
11514 );
11515
11516 let tracker = self.outcome_tracker.read().await;
11517 let models = tracker
11518 .export_profiles()
11519 .iter()
11520 .map(|p| crate::concierge::ModelHealth {
11521 model_id: p.model_id.clone(),
11522 calls: p.total_calls,
11523 success_rate: p.success_rate_resolved(),
11527 avg_latency_ms: p.avg_latency_ms(),
11528 quality: p.ema_quality,
11529 excluded: tracker.is_excluded(&p.model_id),
11530 })
11531 .collect();
11532 drop(tracker);
11533
11534 const STALE_VERIFY_SECS: u64 = 14 * 24 * 60 * 60;
11539 let actions = self.concierge_actions(0);
11540 let mut latest: std::collections::BTreeMap<
11541 crate::intent::UseCase,
11542 &crate::action_ledger::ConciergeActionEntry,
11543 > = std::collections::BTreeMap::new();
11544 for a in &actions {
11545 if a.project.is_none() {
11546 if let Some(uc) = a.use_case {
11547 latest.insert(uc, a);
11548 }
11549 }
11550 }
11551 let mut pending_verification = Vec::new();
11552 for (uc, a) in latest {
11553 if a.kind != crate::action_ledger::ConciergeActionKind::SetDefault {
11554 continue;
11555 }
11556 if now.saturating_sub(a.timestamp) < STALE_VERIFY_SECS {
11557 continue; }
11559 let resolved = entries
11560 .iter()
11561 .filter(|e| {
11562 e.timestamp >= a.timestamp
11563 && e.model_id == a.model_id
11564 && crate::usage_profile::use_case_for_task(e.task) == uc
11565 && e.success.is_some()
11566 })
11567 .count() as u64;
11568 if resolved < crate::concierge::CANARY_MIN_SAMPLES {
11569 pending_verification.push(crate::concierge::PendingVerification {
11570 use_case: uc,
11571 model_id: a.model_id.clone(),
11572 set_at: a.timestamp,
11573 resolved_samples: resolved,
11574 needed: crate::concierge::CANARY_MIN_SAMPLES,
11575 });
11576 }
11577 }
11578
11579 crate::concierge::ConciergeStatus {
11580 lanes: usage.active_lanes().into_iter().cloned().collect(),
11581 decision,
11582 models,
11583 pending_verification,
11584 catalog: self.catalog_freshness(),
11585 revoked_pins: self.revoked_pins(&self.current_registry()),
11586 }
11587 }
11588
11589 pub async fn detect_upgrades(&self) -> Vec<crate::upgrade::UpgradeFinding> {
11593 let prefs = self.update_prefs();
11594 let curated = self.unified_registry().available_upgrades();
11595 let schemas = self.list_schemas();
11596 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11597 let probe = crate::upgrade::HuggingFaceProbe::new();
11598 let now = std::time::SystemTime::now()
11599 .duration_since(std::time::UNIX_EPOCH)
11600 .map(|d| d.as_secs())
11601 .unwrap_or(0);
11602 let mut findings = crate::upgrade::detect_upgrades(
11603 curated,
11604 &refs,
11605 &prefs,
11606 &probe,
11607 &crate::upgrade::UpgradeCache::default_path(),
11608 now,
11609 crate::upgrade::DEFAULT_TTL_SECS,
11610 )
11611 .await;
11612 let lane_models: std::collections::HashSet<String> = self
11617 .lane_defaults()
11618 .defaults
11619 .into_iter()
11620 .map(|d| d.model_id)
11621 .collect();
11622 let in_use = |m: &ModelSchema| {
11623 lane_models.contains(&m.id)
11624 || self.model_management.last_used(&m.id).is_some_and(|at| {
11625 now.saturating_sub(at) < portfolio::PortfolioPolicy::DEFAULT_IDLE_AFTER_SECS
11626 })
11627 };
11628 let installed: Vec<&ModelSchema> = refs
11629 .iter()
11630 .copied()
11631 .filter(|m| {
11632 m.is_local()
11633 && m.downloads_weights()
11634 && m.weights_ready
11635 && self.model_management.car_enabled(&m.id).unwrap_or(true)
11636 && in_use(m)
11637 })
11638 .collect();
11639 let fit = |m: &ModelSchema| {
11640 let fit = self.model_fit(m);
11641 fit.platform_compatible.then_some(fit.fit)
11642 };
11643 let registry = self.unified_registry();
11644 let found = crate::upgrade::suggest_only_unless_builtin(
11645 crate::upgrade::benchmark_findings(&installed, &refs, &fit),
11646 &|id| registry.is_builtin(id),
11647 );
11648 for finding in found {
11649 let duplicate = findings
11650 .iter()
11651 .any(|f| f.from_id == finding.from_id && f.to_id == finding.to_id);
11652 if !duplicate {
11653 findings.push(finding);
11654 }
11655 }
11656 findings
11657 }
11658
11659 pub fn list_schemas(&self) -> Vec<ModelSchema> {
11662 self.catalog_registry_snapshot()
11663 .list()
11664 .into_iter()
11665 .cloned()
11666 .collect()
11667 }
11668
11669 pub fn catalog_snapshot(&self) -> Result<CatalogSnapshot, String> {
11673 CatalogSnapshot::new(self.unified_registry().list().into_iter().cloned())
11674 }
11675
11676 pub fn registered_schema(&self, id: &str) -> Option<ModelSchema> {
11681 self.unified_registry().registered_schema(id).cloned()
11682 }
11683
11684 pub fn list_models(&self) -> Vec<models::ModelInfo> {
11685 self.registry.list_models()
11686 }
11687
11688 pub fn knows_model(&self, name: &str) -> bool {
11701 self.model_schema(name).is_some()
11702 }
11703
11704 pub fn model_schema(&self, name: &str) -> Option<ModelSchema> {
11722 self.resolve_schema(name)
11723 }
11724
11725 fn first_installed_local_model(&self, needs_tools: bool) -> Option<String> {
11738 let registry = self.unified_registry();
11746 let mut candidates: Vec<_> = registry
11747 .all()
11748 .filter(|s| s.is_local() && !s.is_vllm_mlx())
11755 .filter(|s| !s.is_foundation_models() || s.available)
11771 .filter(|s| s.has_capability(ModelCapability::Generate))
11772 .filter(|s| !needs_tools || s.has_capability(ModelCapability::ToolUse))
11776 .filter(|s| self.model_management.car_enabled(&s.id).unwrap_or(false))
11777 .filter(|s| self.unified_registry().ready_without_download(&s.id) == Some(true))
11778 .collect();
11779 candidates.sort_by_key(|s| s.size_mb());
11781
11782 if let Some(avail) = crate::hardware::available_ram_mb() {
11794 const WORKING_RESERVE_MB: u64 = 2048;
11795 candidates.retain(|s| s.size_mb().saturating_add(WORKING_RESERVE_MB) <= avail);
11796 }
11797
11798 candidates.first().map(|s| s.name.clone())
11799 }
11800
11801 pub async fn pull_model(&self, name: &str) -> Result<std::path::PathBuf, InferenceError> {
11803 self.pull_model_with_progress(name, &crate::download::ProgressSink::none())
11804 .await
11805 }
11806
11807 pub async fn pull_model_with_progress(
11812 &self,
11813 name: &str,
11814 sink: &crate::download::ProgressSink,
11815 ) -> Result<std::path::PathBuf, InferenceError> {
11816 self.with_pinned_registry(async {
11817 let pulled = self.pull_model_with_progress_unpinned(name, sink).await;
11818 if pulled.is_ok() {
11819 self.keep_pulled_row_if_withdrawn(name);
11820 }
11821 pulled
11822 })
11823 .await
11824 }
11825
11826 async fn pull_model_with_progress_unpinned(
11827 &self,
11828 name: &str,
11829 sink: &crate::download::ProgressSink,
11830 ) -> Result<std::path::PathBuf, InferenceError> {
11831 let schema = self
11832 .resolve_schema_name_first(name)
11833 .ok_or_else(|| InferenceError::ModelNotFound(name.to_string()))?;
11834 let _mutation = self.model_management.begin_mutation(&schema.id)?;
11835 if let Some(receipt) = self.model_management.load_receipt(&schema.id)? {
11836 let _ = self.model_management.can_remove(&schema.id)?;
11840 self.model_management.clear_tombstone(&schema.id)?;
11841 return Ok(receipt.managed_path);
11842 }
11843 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
11844 return Ok(receipt.managed_path);
11845 }
11846 let expected_managed = self.model_management.models_dir().join(&schema.name);
11847 if std::fs::symlink_metadata(&expected_managed).is_ok() {
11848 return Err(InferenceError::InferenceFailed(format!(
11849 "model {} has a pre-existing unreceipted artifact at {}; use models.adopt or move it before pulling",
11850 schema.id,
11851 expected_managed.display()
11852 )));
11853 }
11854 let staging = self.model_management.create_install_staging(&schema.id)?;
11855 let installed = match self
11856 .unified_registry()
11857 .ensure_local_with_progress_staged(&schema.id, sink, &staging)
11858 .await
11859 {
11860 Ok(installed) => installed,
11861 Err(error) => {
11862 self.model_management.discard_install_staging(&staging);
11863 return Err(error);
11864 }
11865 };
11866 let installed_is_staging = match (installed.canonicalize(), staging.canonicalize()) {
11867 (Ok(installed), Ok(staging)) => installed == staging,
11868 _ => false,
11869 };
11870 let generation = self
11871 .resource_policy_generation
11872 .load(std::sync::atomic::Ordering::Acquire);
11873 let managed_path = self.model_management.models_dir().join(&schema.name);
11874 let managed = if installed_is_staging {
11875 let receipt = self.model_management.install_receipt_for_publication(
11876 &schema.id,
11877 model_source_identity(&schema),
11878 generation,
11879 false,
11880 managed_path,
11881 model_management::ManagedArtifactKind::Directory,
11882 &staging,
11883 )?;
11884 self.model_management
11885 .begin_install_intent(receipt, Some(&staging))?;
11886 self.model_management
11887 .publish_install_staging(&schema.id, &staging, &schema.name)?
11888 } else {
11889 self.model_management.discard_install_staging(&staging);
11890 let receipt = self.model_management.install_receipt_for_publication(
11891 &schema.id,
11892 model_source_identity(&schema),
11893 generation,
11894 false,
11895 managed_path,
11896 model_management::ManagedArtifactKind::Symlink,
11897 &installed,
11898 )?;
11899 self.model_management.begin_install_intent(receipt, None)?;
11900 self.model_management
11901 .materialize_managed_projection(&schema.name, &installed)?
11902 };
11903 let receipt = self
11904 .model_management
11905 .resume_install_intent(&schema.id)?
11906 .ok_or_else(|| {
11907 InferenceError::InferenceFailed(format!(
11908 "model {} publication completed without a durable install intent",
11909 schema.id
11910 ))
11911 })?;
11912 debug_assert_eq!(receipt.managed_path, managed);
11913 Ok(receipt.managed_path)
11914 }
11915
11916 pub async fn adopt_model_into_car(
11920 &self,
11921 model_id: &str,
11922 ) -> Result<model_management::InstallReceipt, InferenceError> {
11923 let schema = self
11924 .resolve_schema(model_id)
11925 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
11926 if !schema.is_local() || !schema.downloads_weights() {
11927 return Err(InferenceError::InferenceFailed(format!(
11928 "model {} is not a CAR-manageable local artifact",
11929 schema.id
11930 )));
11931 }
11932 let existing = self
11933 .unified_registry()
11934 .existing_local_artifact(&schema.id)
11935 .ok_or_else(|| {
11936 InferenceError::InferenceFailed(format!(
11937 "model {} has no usable local artifact to adopt",
11938 schema.id
11939 ))
11940 })?;
11941 let _mutation = self.model_management.begin_mutation(&schema.id)?;
11942 let existing_is_managed_symlink = std::fs::symlink_metadata(&existing)
11943 .is_ok_and(|metadata| metadata.file_type().is_symlink())
11944 && existing.parent().is_some_and(|parent| {
11945 parent.canonicalize().ok() == self.model_management.models_dir().canonicalize().ok()
11946 });
11947 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
11948 return Ok(receipt);
11949 }
11950 let generation = self
11951 .resource_policy_generation
11952 .load(std::sync::atomic::Ordering::Acquire);
11953 let managed = if existing_is_managed_symlink {
11954 existing
11955 } else {
11956 let managed = self.model_management.adopted_projection_path(&schema.id)?;
11957 let receipt = self.model_management.install_receipt_for_publication(
11958 &schema.id,
11959 model_source_identity(&schema),
11960 generation,
11961 true,
11962 managed,
11963 model_management::ManagedArtifactKind::Symlink,
11964 &existing,
11965 )?;
11966 self.model_management.begin_install_intent(receipt, None)?;
11967 self.model_management
11968 .materialize_adopted_projection(&schema.id, &existing)?
11969 };
11970 if existing_is_managed_symlink {
11971 self.model_management
11972 .record_managed_artifact(
11973 &schema.id,
11974 model_source_identity(&schema),
11975 None,
11976 generation,
11977 true,
11978 managed,
11979 )
11980 .map_err(InferenceError::from)
11981 } else {
11982 self.model_management
11983 .resume_install_intent(&schema.id)?
11984 .ok_or_else(|| {
11985 InferenceError::InferenceFailed(format!(
11986 "model {} adoption completed without a durable install intent",
11987 schema.id
11988 ))
11989 })
11990 }
11991 }
11992
11993 pub async fn remove_model_from_car(
11996 &self,
11997 model_id: &str,
11998 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
11999 self.remove_model_from_car_inner(model_id, false).await
12000 }
12001
12002 async fn remove_model_from_car_inner(
12006 &self,
12007 model_id: &str,
12008 maintenance_held: bool,
12009 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
12010 let schema = self
12011 .resolve_schema(model_id)
12012 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
12013 if !schema.is_local() || !schema.downloads_weights() {
12014 return Err(InferenceError::InferenceFailed(format!(
12015 "model {} is not a CAR-owned local artifact",
12016 schema.id
12017 )));
12018 }
12019 if self.model_management.load_receipt(&schema.id)?.is_none() {
12020 if self.model_management.car_enabled(&schema.id)? {
12021 return Err(model_management::ModelManagementError::MissingReceipt {
12022 model_id: schema.id.clone(),
12023 }
12024 .into());
12025 }
12026 let generation = self
12030 .resource_policy_generation
12031 .load(std::sync::atomic::Ordering::Acquire)
12032 .saturating_add(1);
12033 return self
12034 .model_management
12035 .begin_mutation(&schema.id)?
12036 .remove(generation)
12037 .map_err(InferenceError::from);
12038 }
12039 let mutation = self.model_management.begin_mutation(&schema.id)?;
12040 let _maintenance = if maintenance_held {
12041 None
12042 } else {
12043 Some(
12044 self.prepare_local_model_removal(&schema.id)
12045 .await
12046 .map_err(|error| InferenceError::InferenceFailed(error.to_string()))?,
12047 )
12048 };
12049 let generation = self
12050 .resource_policy_generation
12051 .load(std::sync::atomic::Ordering::Acquire)
12052 .saturating_add(1);
12053 mutation.remove(generation).map_err(InferenceError::from)
12054 }
12055
12056 pub fn update_prefs(&self) -> crate::update_prefs::UpdatePreferences {
12061 let cwd = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("."));
12062 crate::update_prefs::UpdatePreferences::load_effective(&cwd).unwrap_or_else(|error| {
12063 tracing::warn!(%error, "update preferences unreadable; acting as notify");
12066 crate::update_prefs::UpdatePreferences {
12067 policy: crate::update_prefs::UpdatePolicy::Notify,
12068 policy_chosen: true,
12069 ..Default::default()
12070 }
12071 })
12072 }
12073
12074 pub fn set_update_prefs(
12076 &self,
12077 prefs: &crate::update_prefs::UpdatePreferences,
12078 ) -> Result<(), InferenceError> {
12079 prefs.save().map_err(InferenceError::InferenceFailed)
12080 }
12081
12082 #[deprecated(note = "use async remove_model_from_car for receipt-backed safe removal")]
12086 pub fn remove_model(&self, name: &str) -> Result<(), InferenceError> {
12087 Err(InferenceError::InferenceFailed(format!(
12088 "legacy removal for {name} is disabled; use async receipt-backed remove_model_from_car"
12089 )))
12090 }
12091
12092 pub fn register_model(&mut self, schema: ModelSchema) {
12098 self.unified_registry_mut().register(schema);
12099 }
12100
12101 pub fn register_user_model(&mut self, schema: ModelSchema) {
12103 self.unified_registry_mut().register_user_model(schema);
12104 }
12105
12106 pub async fn discover_vllm_mlx_models(&mut self) -> usize {
12109 let config = vllm_mlx::VllmMlxConfig::default();
12110 if !config.auto_discover {
12111 return 0;
12112 }
12113 vllm_mlx::discover_and_register(&config, self.unified_registry_mut()).await
12114 }
12115
12116 pub fn outcome_tracker(&self) -> Arc<RwLock<OutcomeTracker>> {
12118 self.outcome_tracker.clone()
12119 }
12120
12121 async fn auto_save_outcomes(&self) {
12129 const OUTCOME_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
12130
12131 if crate::offload::is_offload_worker() {
12139 return;
12140 }
12141
12142 let due = {
12144 let last = self.last_outcome_flush.lock().unwrap();
12145 last.is_none_or(|t| t.elapsed() >= OUTCOME_FLUSH_INTERVAL)
12146 };
12147 if due {
12148 match self.persist_outcomes().await {
12149 Ok(did) => {
12150 if did {
12151 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12152 }
12153 }
12154 Err(e) => tracing::debug!("auto-save outcomes failed: {}", e),
12155 }
12156 }
12157
12158 if let Err(e) = self.save_key_pool_stats().await {
12159 tracing::debug!("auto-save key pool stats failed: {}", e);
12160 }
12161 }
12162
12163 async fn persist_outcomes(&self) -> Result<bool, std::io::Error> {
12169 const PENDING_TTL_SECS: u64 = 300;
12170 let entries = {
12174 let mut tracker = self.outcome_tracker.write().await;
12175 tracker.sweep_pending(PENDING_TTL_SECS);
12176 tracker.drain_ledger()
12177 };
12178 let mut did = false;
12179 let ledger_disabled = std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some();
12184 if !entries.is_empty() && !ledger_disabled {
12185 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12186 let _guard = self.ledger_io_lock.lock().await;
12187 crate::outcome::append_ledger_entries(&ledger_path, &entries)?;
12188 did = true;
12189 }
12190 let profiles_path = self.config.state_models_dir().join("outcome_profiles.json");
12192 let wrote = {
12193 let mut tracker = self.outcome_tracker.write().await;
12194 tracker.save_if_dirty(&profiles_path)?
12195 };
12196 Ok(did || wrote)
12197 }
12198
12199 pub async fn save_outcomes(&self) -> Result<(), std::io::Error> {
12204 let tracker = self.outcome_tracker.read().await;
12205 let path = self.config.state_models_dir().join("outcome_profiles.json");
12206 tracker.save_to_file(&path)
12207 }
12208
12209 pub async fn flush_outcomes(&self) -> Result<bool, std::io::Error> {
12215 self.flush_outcomes_inner().await
12216 }
12217
12218 pub async fn reconcile_outcome_ledger(
12234 &self,
12235 ) -> Result<outcome::LedgerReconciliation, std::io::Error> {
12236 if crate::offload::is_offload_worker()
12237 || std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some()
12238 || !self.outcome_profiles_loaded
12239 {
12240 return Ok(outcome::LedgerReconciliation::default());
12241 }
12242 let models_dir = self.config.state_models_dir();
12243 let ledger = models_dir.join("outcome_ledger.jsonl");
12244 let _ = std::fs::remove_file(ledger.with_extension("jsonl.migrated"));
12247 self.flush_outcomes_inner().await?;
12248 let seeds = quality_seeds(&models_dir);
12249 let _io = self.ledger_io_lock.lock().await;
12250 let registry = &self.unified_registry();
12251 let is_registered = |id: &str| registry.get(id).is_some();
12252 let mut tracker = self.outcome_tracker.write().await;
12253 let plan = outcome::plan_reconcile(&ledger, &tracker, &is_registered, &seeds);
12254 if plan.ledger.is_none() && plan.profiles.is_none() {
12255 return Ok(plan.report);
12256 }
12257 if let Some(entries) = &plan.ledger {
12258 outcome::write_reconciled_ledger(&ledger, entries)?;
12259 }
12260 if let Some(profiles) = plan.profiles {
12261 tracker.commit_reconciled(profiles);
12262 tracker.save_to_file(&models_dir.join("outcome_profiles.json"))?;
12263 }
12264 tracing::info!(
12265 reclassified = plan.report.reclassified,
12266 unregistered = plan.report.unregistered,
12267 profiles_rebuilt = plan.report.profiles_rebuilt,
12268 profiles_removed = plan.report.profiles_removed,
12269 "reconciled the outcome ledger with model profiles"
12270 );
12271 Ok(plan.report)
12272 }
12273
12274 async fn flush_outcomes_inner(&self) -> Result<bool, std::io::Error> {
12275 let did = self.persist_outcomes().await?;
12276 if did {
12277 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12278 }
12279 Ok(did)
12280 }
12281
12282 pub async fn prune_outcome_ledger(&self, max_entries: usize) -> std::io::Result<()> {
12285 let path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12286 let _guard = self.ledger_io_lock.lock().await;
12287 crate::outcome::prune_ledger(&path, max_entries)
12288 }
12289
12290 pub async fn save_key_pool_stats(&self) -> Result<(), std::io::Error> {
12292 let path = self.config.state_models_dir().join("key_pool_stats.json");
12293 self.remote_backend.key_pool.save_stats(&path).await
12294 }
12295
12296 pub async fn key_pool_stats(
12298 &self,
12299 ) -> std::collections::HashMap<String, Vec<key_pool::KeyStats>> {
12300 self.remote_backend.key_pool.all_stats().await
12301 }
12302
12303 pub async fn export_profiles(&self) -> Vec<ModelProfile> {
12305 let tracker = self.outcome_tracker.read().await;
12306 tracker.export_profiles()
12307 }
12308
12309 pub fn outcome_scoreboard(&self) -> crate::scoreboard::Scoreboard {
12317 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12318 let entries = crate::outcome::read_ledger(&ledger_path, 0);
12319 crate::calibration::ShadowCalibration::from_ledger(&entries).emit();
12323 crate::scoreboard::Scoreboard::from_ledger(&entries, |id| {
12324 let s = self.resolve_schema(id)?;
12325 scoreboard_price_model(&s)
12326 })
12327 }
12328
12329 pub async fn import_profiles(&self, profiles: Vec<ModelProfile>) {
12331 let mut tracker = self.outcome_tracker.write().await;
12332 tracker.import_profiles(profiles);
12333 }
12334
12335 pub async fn prepare_speech_runtime(&self) -> Result<PathBuf, InferenceError> {
12358 self.ensure_speech_runtime().await.map(|r| r.root)
12364 }
12365
12366 pub fn set_speech_policy(&mut self, policy: SpeechPolicy) {
12368 self.speech_policy = policy;
12369 }
12370
12371 pub fn set_routing_config(&mut self, config: RoutingConfig) {
12372 self.adaptive_router.set_config(config);
12373 }
12374
12375 pub async fn install_curated_speech(
12377 &mut self,
12378 ) -> Result<Vec<SpeechInstallReport>, InferenceError> {
12379 if let Err(error) = self.prepare_speech_runtime().await {
12392 tracing::warn!(
12393 %error,
12394 "managed speech runtime could not be provisioned; installing the \
12395 models that do not depend on it"
12396 );
12397 }
12398 let schemas = self.list_schemas();
12399 let mut repos = Vec::new();
12400 for schema in &schemas {
12401 if !schema.is_mlx() || !schema.tags.iter().any(|tag| tag == "speech") {
12402 continue;
12403 }
12404 if !cfg!(all(
12419 target_os = "macos",
12420 target_arch = "aarch64",
12421 not(car_skip_mlx)
12422 )) {
12423 continue;
12424 }
12425 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
12426 if !repos.iter().any(|existing: &String| existing == hf_repo) {
12427 repos.push(hf_repo.clone());
12428 }
12429 }
12430 }
12431
12432 let mut installed = Vec::new();
12433 for repo in repos {
12434 let model_for_record = self
12435 .unified_registry()
12436 .all()
12437 .find(|m| matches!(&m.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo))
12438 .map(|m| m.id.clone())
12439 .unwrap_or_else(|| repo.clone());
12440 let _guard = crate::download::acquire_model_lock(&model_for_record).await;
12443 let (snapshot, fetched) =
12444 crate::retire::collecting_fetches(download_hf_repo_snapshot(&repo)).await;
12445 crate::retire::record_download(
12446 &self.config.state_root,
12447 &crate::hf_cache::hub_dir(),
12448 &model_for_record,
12449 &fetched,
12450 );
12451 let (snapshot_path, files_downloaded) = snapshot?;
12452 let name = schemas
12453 .iter()
12454 .find(|schema| {
12455 matches!(&schema.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo)
12456 })
12457 .map(|schema| schema.name.clone())
12458 .unwrap_or_else(|| repo.clone());
12459 installed.push(SpeechInstallReport {
12460 name,
12461 hf_repo: repo,
12462 snapshot_path,
12463 files_downloaded,
12464 });
12465 }
12466
12467 for schema in &schemas {
12471 if !schema.tags.iter().any(|tag| tag == "speech") {
12472 continue;
12473 }
12474 if let ModelSource::WhisperCpp { model } = &schema.source {
12475 let model = model.clone();
12476 let name = schema.name.clone();
12477 let path = tokio::task::spawn_blocking(move || car_whisper::ensure_model(&model))
12478 .await
12479 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))?
12480 .map_err(|e| {
12481 InferenceError::InferenceFailed(format!("whisper model fetch: {e}"))
12482 })?;
12483 installed.push(SpeechInstallReport {
12484 name,
12485 hf_repo: "ggerganov/whisper.cpp".to_string(),
12486 snapshot_path: path,
12487 files_downloaded: 1,
12488 });
12489 }
12490 }
12491
12492 self.unified_registry_mut().refresh_availability();
12493 Ok(installed)
12494 }
12495
12496 pub fn speech_health(&self) -> SpeechHealthReport {
12498 let local_stt_default =
12499 self.speech_health_default_name(ModelCapability::SpeechToText, true, false);
12500 let local_tts_default =
12501 self.speech_health_default_name(ModelCapability::TextToSpeech, true, false);
12502 let remote_stt_default =
12503 self.speech_health_default_name(ModelCapability::SpeechToText, false, true);
12504 let remote_tts_default =
12505 self.speech_health_default_name(ModelCapability::TextToSpeech, false, true);
12506
12507 let mut local_models = Vec::new();
12508 let mut remote_models = Vec::new();
12509 for schema in self.list_schemas() {
12510 let capability = if schema.has_capability(ModelCapability::SpeechToText) {
12511 Some(ModelCapability::SpeechToText)
12512 } else if schema.has_capability(ModelCapability::TextToSpeech) {
12513 Some(ModelCapability::TextToSpeech)
12514 } else {
12515 None
12516 };
12517 let Some(capability) = capability else {
12518 continue;
12519 };
12520
12521 let selected_by_default = local_stt_default
12522 .as_ref()
12523 .is_some_and(|name| name == &schema.name)
12524 || local_tts_default
12525 .as_ref()
12526 .is_some_and(|name| name == &schema.name)
12527 || remote_stt_default
12528 .as_ref()
12529 .is_some_and(|name| name == &schema.name)
12530 || remote_tts_default
12531 .as_ref()
12532 .is_some_and(|name| name == &schema.name);
12533
12534 let health = SpeechModelHealth {
12535 id: schema.id.clone(),
12536 name: schema.name.clone(),
12537 provider: schema.provider.clone(),
12538 capability,
12539 is_local: schema.is_local(),
12540 available: schema.available,
12541 cached: speech_model_cached(&schema),
12542 selected_by_default,
12543 source: speech_model_source_label(&schema),
12544 };
12545 if schema.is_local() {
12546 local_models.push(health);
12547 } else {
12548 remote_models.push(health);
12549 }
12550 }
12551
12552 let runtime = {
12559 let rt =
12560 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
12561 SpeechRuntimeHealth {
12562 root: rt.root.clone(),
12563 installed: rt.is_ready(),
12564 python: rt.python.clone(),
12565 stt_command: rt.stt_program.clone(),
12566 tts_command: rt.tts_program.clone(),
12567 configured_python: std::env::var("CAR_SPEECH_PYTHON")
12568 .ok()
12569 .filter(|value| !value.trim().is_empty()),
12570 detected_python: detect_speech_python(),
12571 }
12572 };
12573
12574 SpeechHealthReport {
12575 runtime,
12576 local_models,
12577 remote_models,
12578 elevenlabs_configured: crate::tasks::transcribe::provider_configured_for_passive_status(
12582 "ELEVENLABS_API_KEY",
12583 )
12584 || crate::tasks::synthesize::provider_configured_for_passive_status(
12585 "ELEVENLABS_API_KEY",
12586 ),
12587 prefer_local: self.speech_policy.prefer_local,
12588 allow_remote_fallback: self.speech_policy.allow_remote_fallback,
12589 preferred_local_stt: self.speech_policy.preferred_local_stt.clone(),
12590 preferred_local_tts: self.speech_policy.preferred_local_tts.clone(),
12591 preferred_remote_stt: self.speech_policy.preferred_remote_stt.clone(),
12592 preferred_remote_tts: self.speech_policy.preferred_remote_tts.clone(),
12593 local_stt_default,
12594 local_tts_default,
12595 remote_stt_default,
12596 remote_tts_default,
12597 }
12598 }
12599
12600 pub async fn model_health(&self) -> ModelHealthReport {
12603 self.model_health_with(false).await
12604 }
12605
12606 pub async fn model_health_for_explicit_status(&self) -> ModelHealthReport {
12616 self.model_health_with(true).await
12617 }
12618
12619 async fn model_health_with(&self, authoritative_credentials: bool) -> ModelHealthReport {
12620 let schemas: Vec<ModelSchema> = self
12621 .catalog_registry_snapshot_with(authoritative_credentials)
12622 .list()
12623 .into_iter()
12624 .cloned()
12625 .collect();
12626 let total_models = schemas.len();
12627 let available_models = schemas
12628 .iter()
12629 .filter(|schema| schema.available_now())
12630 .count();
12631 let local_models = schemas.iter().filter(|schema| schema.is_local()).count();
12632 let remote_models = total_models.saturating_sub(local_models);
12633
12634 let defaults = vec![
12635 self.model_default_health(
12636 ModelCapability::Generate,
12637 self.preferred_model_for_capability(ModelCapability::Generate)
12638 .unwrap_or(&self.config.generation_model),
12639 ),
12640 self.model_default_health(
12641 ModelCapability::Embed,
12642 self.preferred_model_for_capability(ModelCapability::Embed)
12643 .unwrap_or(&self.config.embedding_model),
12644 ),
12645 self.model_default_health(
12646 ModelCapability::Classify,
12647 self.preferred_model_for_capability(ModelCapability::Classify)
12648 .unwrap_or(&self.config.classification_model),
12649 ),
12650 ];
12651
12652 let credential_memo: std::sync::Mutex<std::collections::HashMap<String, bool>> =
12655 std::sync::Mutex::new(std::collections::HashMap::new());
12656 let credential = |env_var: &str| -> bool {
12657 if !authoritative_credentials {
12665 return std::env::var(env_var).is_ok_and(|value| !value.trim().is_empty());
12666 }
12667 let mut memo = credential_memo
12668 .lock()
12669 .unwrap_or_else(|poisoned| poisoned.into_inner());
12670 if let Some(known) = memo.get(env_var) {
12671 return *known;
12672 }
12673 let present = crate::registry::credential_present_without_reading_it(env_var);
12674 memo.insert(env_var.to_string(), present);
12675 present
12676 };
12677
12678 let mut providers = std::collections::BTreeMap::new();
12679 for schema in &schemas {
12680 let entry =
12681 providers
12682 .entry(schema.provider.clone())
12683 .or_insert_with(|| ProviderAccumulator {
12684 configured: false,
12685 local_models: 0,
12686 remote_models: 0,
12687 available_models: 0,
12688 capabilities: std::collections::HashSet::new(),
12689 });
12690
12691 entry.configured |= model_source_configured(schema, &credential);
12692 if schema.is_local() {
12693 entry.local_models += 1;
12694 } else {
12695 entry.remote_models += 1;
12696 }
12697 if schema.available_now() {
12698 entry.available_models += 1;
12699 }
12700 for capability in &schema.capabilities {
12701 entry.capabilities.insert(*capability);
12702 }
12703 }
12704
12705 let providers = providers
12706 .into_iter()
12707 .map(|(provider, acc)| ModelProviderHealth {
12708 provider,
12709 configured: acc.configured,
12710 local_models: acc.local_models,
12711 remote_models: acc.remote_models,
12712 available_models: acc.available_models,
12713 capabilities: sort_capabilities(acc.capabilities.into_iter().collect()),
12714 })
12715 .collect();
12716
12717 let capabilities = all_model_capabilities()
12718 .into_iter()
12719 .map(|capability| {
12720 let relevant: Vec<&ModelSchema> = schemas
12721 .iter()
12722 .filter(|schema| schema.has_capability(capability))
12723 .collect();
12724 let available: Vec<&ModelSchema> = relevant
12725 .iter()
12726 .copied()
12727 .filter(|schema| schema.available_now())
12728 .collect();
12729 ModelCapabilityHealth {
12730 capability,
12731 total_models: relevant.len(),
12732 available_models: available.len(),
12733 local_available_models: available
12734 .iter()
12735 .filter(|schema| schema.is_local())
12736 .count(),
12737 remote_available_models: available
12738 .iter()
12739 .filter(|schema| !schema.is_local())
12740 .count(),
12741 }
12742 })
12743 .collect();
12744
12745 let routing = self.routing_scenarios().await;
12746 let routing_config = self.adaptive_router.config().clone();
12747 let benchmark_priors =
12748 load_benchmark_prior_health(&self.config.state_models_dir(), &schemas);
12749
12750 ModelHealthReport {
12751 total_models,
12752 available_models,
12753 local_models,
12754 remote_models,
12755 defaults,
12756 providers,
12757 capabilities,
12758 routing_prefer_local: routing_config.prefer_local,
12759 routing_quality_first_cold_start: routing_config.quality_first_cold_start,
12760 routing_min_observations: routing_config.min_observations,
12761 routing_bootstrap_min_task_observations: routing_config.bootstrap_min_task_observations,
12762 routing_bootstrap_quality_floor: routing_config.bootstrap_quality_floor,
12763 routing_quality_weight: routing_config.quality_weight,
12764 routing_latency_weight: routing_config.latency_weight,
12765 routing_cost_weight: routing_config.cost_weight,
12766 routing_scenarios: routing,
12767 benchmark_priors,
12768 speech: self.speech_health(),
12769 }
12770 }
12771
12772 async fn routing_scenarios(&self) -> Vec<RoutingScenarioHealth> {
12773 let tracker = self.outcome_tracker.read().await;
12774 let config = self.adaptive_router.config().clone();
12775 let scenarios = [
12776 (
12777 "interactive_text",
12778 "Summarize the benefits of local-first AI routing in two sentences.",
12779 "text",
12780 RoutingWorkload::Interactive,
12781 false,
12782 false,
12783 ),
12784 (
12785 "background_code",
12786 "Write a Python function named fibonacci(n) that returns the nth Fibonacci number.",
12787 "code",
12788 RoutingWorkload::Background,
12789 false,
12790 false,
12791 ),
12792 (
12793 "interactive_tool_use",
12794 "Use the provided weather tool to get the weather for Boston.",
12795 "tool_use",
12796 RoutingWorkload::Interactive,
12797 true,
12798 false,
12799 ),
12800 (
12801 "interactive_vision",
12802 "What is in this image? Answer in one word.",
12803 "vision",
12804 RoutingWorkload::Interactive,
12805 false,
12806 true,
12807 ),
12808 ];
12809
12810 let routing_registry = self.catalog_registry_snapshot();
12817
12818 scenarios
12819 .into_iter()
12820 .map(
12821 |(name, prompt, task_family, workload, has_tools, has_vision)| {
12822 let decision = self.adaptive_router.route_context_aware(
12823 prompt,
12824 0,
12825 &routing_registry,
12826 &tracker,
12827 has_tools,
12828 has_vision,
12829 workload,
12830 );
12831 let quality_first_cold_start = if has_tools || has_vision {
12832 config.quality_first_cold_start
12833 } else if task_family == "code"
12834 && matches!(workload, RoutingWorkload::Background)
12835 {
12836 false
12837 } else {
12838 config.quality_first_cold_start
12839 };
12840 RoutingScenarioHealth {
12841 name: name.to_string(),
12842 task_family: task_family.to_string(),
12843 workload,
12844 has_tools,
12845 has_vision,
12846 prefer_local: if task_family == "speech" {
12847 self.speech_policy.prefer_local
12848 } else {
12849 config.prefer_local
12850 },
12851 quality_first_cold_start,
12852 bootstrap_min_task_observations: config.bootstrap_min_task_observations,
12853 bootstrap_quality_floor: config.bootstrap_quality_floor,
12854 model_id: decision.model_id,
12855 model_name: decision.model_name,
12856 reason: decision.reason,
12857 strategy: decision.strategy,
12858 }
12859 },
12860 )
12861 .collect()
12862 }
12863
12864 pub async fn smoke_test_speech(
12866 &self,
12867 local: bool,
12868 remote: bool,
12869 ) -> Result<SpeechSmokeReport, InferenceError> {
12870 let mut report = SpeechSmokeReport::default();
12871
12872 if local {
12873 let tts = self
12874 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
12875 .ok_or_else(|| {
12876 InferenceError::InferenceFailed(
12877 "no local text-to-speech model available".into(),
12878 )
12879 })?;
12880 let stt = self
12881 .preferred_speech_schema(ModelCapability::SpeechToText, true, false)
12882 .ok_or_else(|| {
12883 InferenceError::InferenceFailed(
12884 "no local speech-to-text model available".into(),
12885 )
12886 })?;
12887 report.local = Some(
12888 self.run_speech_smoke_path("local", &tts, &stt, "Testing CAR local speech path.")
12889 .await?,
12890 );
12891 } else {
12892 report.skipped.push("local".to_string());
12893 }
12894
12895 if remote {
12896 let tts = self
12897 .preferred_speech_schema(ModelCapability::TextToSpeech, false, true)
12898 .ok_or_else(|| {
12899 InferenceError::InferenceFailed(
12900 "no remote text-to-speech model available".into(),
12901 )
12902 })?;
12903 let stt = self
12904 .preferred_speech_schema(ModelCapability::SpeechToText, false, true)
12905 .ok_or_else(|| {
12906 InferenceError::InferenceFailed(
12907 "no remote speech-to-text model available".into(),
12908 )
12909 })?;
12910 report.remote = Some(
12911 self.run_speech_smoke_path("remote", &tts, &stt, "Testing CAR remote speech path.")
12912 .await?,
12913 );
12914 } else {
12915 report.skipped.push("remote".to_string());
12916 }
12917
12918 Ok(report)
12919 }
12920
12921 fn speech_candidates(
12922 &self,
12923 capability: ModelCapability,
12924 explicit: Option<&str>,
12925 ) -> Result<Vec<ModelSchema>, InferenceError> {
12926 if let Some(model) = explicit {
12927 let schema = self
12928 .resolve_schema(model)
12929 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
12930 if !schema.has_capability(capability) {
12931 return Err(InferenceError::InferenceFailed(format!(
12932 "model {} does not support {:?}",
12933 schema.name, capability
12934 )));
12935 }
12936 return Ok(vec![schema]);
12937 }
12938
12939 let mut candidates: Vec<ModelSchema> = self
12940 .unified_registry()
12941 .query(&ModelFilter {
12942 capabilities: vec![capability],
12943 ..Default::default()
12944 })
12945 .into_iter()
12946 .cloned()
12947 .collect();
12948
12949 if candidates.is_empty() {
12950 return Err(InferenceError::InferenceFailed(format!(
12951 "no models registered for capability {:?}",
12952 capability
12953 )));
12954 }
12955
12956 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
12957 if !self.speech_policy.allow_remote_fallback
12958 && candidates.iter().any(|model| model.is_local())
12959 {
12960 candidates.retain(|model| model.is_local());
12961 }
12962
12963 Ok(candidates)
12964 }
12965
12966 #[allow(dead_code)] fn resolve_external_hf_repo(
12972 &self,
12973 explicit: Option<&str>,
12974 capability: ModelCapability,
12975 ) -> Option<String> {
12976 let id = explicit?;
12977 let schema = self.resolve_schema(id)?;
12978 if !schema.has_capability(capability) {
12979 return Some(id.to_string());
12980 }
12981 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
12982 return Some(hf_repo.clone());
12983 }
12984 Some(id.to_string())
12985 }
12986
12987 fn media_generation_candidates(
12988 &self,
12989 capability: ModelCapability,
12990 explicit: Option<&str>,
12991 ) -> Result<Vec<ModelSchema>, InferenceError> {
12992 if let Some(model) = explicit {
12993 let schema = self
12994 .resolve_schema(model)
12995 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
12996 if !schema.has_capability(capability) {
12997 return Err(InferenceError::InferenceFailed(format!(
12998 "model {} does not support {:?}",
12999 schema.name, capability
13000 )));
13001 }
13002 return Ok(vec![schema]);
13003 }
13004
13005 let mut candidates: Vec<ModelSchema> = self
13006 .unified_registry()
13007 .query(&ModelFilter {
13008 capabilities: vec![capability],
13009 local_only: true,
13010 ..Default::default()
13011 })
13012 .into_iter()
13013 .cloned()
13014 .collect();
13015 candidates.sort_by_key(|schema| (!schema.available, schema.size_mb()));
13016 if candidates.is_empty() {
13017 return Err(InferenceError::InferenceFailed(format!(
13018 "no models registered for capability {:?}",
13019 capability
13020 )));
13021 }
13022 Ok(candidates)
13023 }
13024
13025 fn preferred_speech_schema(
13026 &self,
13027 capability: ModelCapability,
13028 local_only: bool,
13029 remote_only: bool,
13030 ) -> Option<ModelSchema> {
13031 let available_only = remote_only;
13032 let mut candidates: Vec<ModelSchema> = self
13033 .unified_registry()
13034 .query(&ModelFilter {
13035 capabilities: vec![capability],
13036 available_only,
13037 ..Default::default()
13038 })
13039 .into_iter()
13040 .filter(|schema| {
13041 (!local_only || schema.is_local()) && (!remote_only || schema.is_remote())
13042 })
13043 .cloned()
13044 .collect();
13045 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
13046 candidates.into_iter().next()
13047 }
13048
13049 fn speech_health_default_name(
13050 &self,
13051 capability: ModelCapability,
13052 local_only: bool,
13053 remote_only: bool,
13054 ) -> Option<String> {
13055 let preferred = match capability {
13056 ModelCapability::SpeechToText if local_only => {
13057 self.speech_policy.preferred_local_stt.as_ref()
13058 }
13059 ModelCapability::SpeechToText if remote_only => {
13060 self.speech_policy.preferred_remote_stt.as_ref()
13061 }
13062 ModelCapability::TextToSpeech if local_only => {
13063 self.speech_policy.preferred_local_tts.as_ref()
13064 }
13065 ModelCapability::TextToSpeech if remote_only => {
13066 self.speech_policy.preferred_remote_tts.as_ref()
13067 }
13068 _ => None,
13069 };
13070
13071 preferred
13072 .filter(|name| {
13073 self.unified_registry().list().iter().any(|schema| {
13074 schema.name == **name
13075 && schema.has_capability(capability)
13076 && (!local_only || schema.is_local())
13077 && (!remote_only || schema.is_remote())
13078 })
13079 })
13080 .cloned()
13081 .or_else(|| {
13082 self.preferred_speech_schema(capability, local_only, remote_only)
13083 .map(|schema| schema.name)
13084 })
13085 }
13086
13087 fn model_default_health(
13088 &self,
13089 capability: ModelCapability,
13090 configured_model: &str,
13091 ) -> ModelDefaultHealth {
13092 let schema = self.resolve_schema_name_first(configured_model);
13093
13094 ModelDefaultHealth {
13095 capability,
13096 configured_model: configured_model.to_string(),
13097 available: schema.as_ref().is_some_and(ModelSchema::available_now),
13098 is_local: schema.as_ref().is_some_and(ModelSchema::is_local),
13099 provider: schema.map(|model| model.provider),
13100 }
13101 }
13102
13103 fn speech_sort_key(
13104 &self,
13105 capability: ModelCapability,
13106 model: &ModelSchema,
13107 ) -> (u8, u8, u8, u8, u64, u64) {
13108 let policy_preference = match capability {
13109 ModelCapability::SpeechToText if model.is_local() => {
13110 self.speech_policy.preferred_local_stt.as_ref()
13111 }
13112 ModelCapability::SpeechToText => self.speech_policy.preferred_remote_stt.as_ref(),
13113 ModelCapability::TextToSpeech if model.is_local() => {
13114 self.speech_policy.preferred_local_tts.as_ref()
13115 }
13116 ModelCapability::TextToSpeech => self.speech_policy.preferred_remote_tts.as_ref(),
13117 _ => None,
13118 };
13119 let local_rank = if self.speech_policy.prefer_local {
13120 if model.is_local() {
13121 0
13122 } else {
13123 1
13124 }
13125 } else if model.is_remote() {
13126 0
13127 } else {
13128 1
13129 };
13130 let availability_rank = if model.available {
13131 0
13132 } else if model.is_local() {
13133 1
13134 } else {
13135 2
13136 };
13137 let policy_rank: u8 = if policy_preference.is_some_and(|preferred| preferred == &model.name)
13138 {
13139 0
13140 } else {
13141 1
13142 };
13143 let speech_rank = match capability {
13144 ModelCapability::TextToSpeech => {
13160 if model.name == "Kokoro-82M-bf16" {
13161 0
13162 } else if model.name == "Kokoro-82M-6bit" {
13163 1
13164 } else if model.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
13165 3
13168 } else {
13169 2
13170 }
13171 }
13172 ModelCapability::SpeechToText => {
13173 if model.name == "Parakeet-TDT-0.6B-v3-MLX" {
13174 0
13175 } else {
13176 1
13177 }
13178 }
13179 _ => 0,
13180 };
13181 let latency_rank = model.performance.latency_p50_ms.unwrap_or(u64::MAX);
13182 let size_rank = model.cost.size_mb.unwrap_or(u64::MAX);
13183 (
13184 local_rank,
13185 availability_rank,
13186 policy_rank,
13187 speech_rank,
13188 latency_rank,
13189 size_rank,
13190 )
13191 }
13192
13193 async fn run_speech_smoke_path(
13194 &self,
13195 path: &str,
13196 tts: &ModelSchema,
13197 stt: &ModelSchema,
13198 text: &str,
13199 ) -> Result<SpeechSmokePathReport, InferenceError> {
13200 let work_dir = temp_work_dir(&format!("speech-smoke-{path}"))?;
13201 let audio_path = work_dir.join(format!("{path}.wav"));
13202 let synth = self
13203 .synthesize(SynthesizeRequest {
13204 text: text.to_string(),
13205 model: Some(tts.name.clone()),
13206 voice: default_speech_voice(tts),
13207 language: Some("en".to_string()),
13208 output_path: Some(audio_path.display().to_string()),
13209 ..SynthesizeRequest::default()
13210 })
13211 .await?;
13212 let transcript = self
13213 .transcribe(TranscribeRequest {
13214 audio_path: synth.audio_path.clone(),
13215 model: Some(stt.name.clone()),
13216 language: Some("en".to_string()),
13217 prompt: None,
13218 timestamps: false,
13219 })
13220 .await?;
13221
13222 Ok(SpeechSmokePathReport {
13223 path: path.to_string(),
13224 tts_model: synth.model_used.unwrap_or_else(|| tts.name.clone()),
13225 stt_model: transcript.model_used.unwrap_or_else(|| stt.name.clone()),
13226 audio_path: PathBuf::from(synth.audio_path),
13227 transcript: transcript.text,
13228 })
13229 }
13230
13231 async fn ensure_speech_runtime(&self) -> Result<SpeechRuntime, InferenceError> {
13232 let mut guard = self.speech_runtime.lock().await;
13233 if let Some(runtime) = guard.as_ref() {
13234 if runtime.is_ready() {
13235 return Ok(runtime.clone());
13236 }
13237 }
13238
13239 let runtime =
13240 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
13241 if !runtime.is_ready() {
13242 bootstrap_speech_runtime(&runtime).await?;
13243 }
13244 if !runtime.is_ready() {
13245 return Err(InferenceError::InferenceFailed(format!(
13246 "managed speech runtime is not ready at {}",
13247 runtime.root.display()
13248 )));
13249 }
13250
13251 *guard = Some(runtime.clone());
13252 Ok(runtime)
13253 }
13254
13255 async fn transcribe_whisper(
13264 &self,
13265 schema: &ModelSchema,
13266 model: &str,
13267 req: &TranscribeRequest,
13268 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13269 ) -> Result<TranscribeResult, InferenceError> {
13270 let model = model.to_string();
13271 let model_path = car_whisper::ensure_model(&model)
13272 .map_err(|e| InferenceError::InferenceFailed(format!("whisper download: {e}")))?;
13273 let measured_bytes = backend_cache::estimate_model_size(&model_path);
13274 let reservation = reservation.ok_or_else(|| {
13275 InferenceError::InferenceFailed("local Whisper path missing admission".into())
13276 })?;
13277 reservation
13278 .reconcile_measured_weights(measured_bytes)
13279 .map_err(InferenceError::from)?;
13280 let detached_lease = reservation.detached_lease();
13281 let language = req.language.clone().unwrap_or_else(|| "auto".to_string());
13283 let audio_path = std::path::PathBuf::from(&req.audio_path);
13284 let name = schema.name.clone();
13285 let req_language = req.language.clone();
13286 let text =
13287 run_admitted_blocking(detached_lease, move || -> Result<String, InferenceError> {
13288 let stt = car_whisper::WhisperStt::load_from_path(&model_path, &language)
13290 .map_err(|e| InferenceError::InferenceFailed(format!("whisper load: {e}")))?;
13291 stt.transcribe_file(&audio_path).map_err(|e| {
13292 InferenceError::InferenceFailed(format!("whisper transcribe: {e}"))
13293 })
13294 })
13295 .await
13296 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))??;
13297 Ok(TranscribeResult::text_only(text, Some(name), req_language))
13298 }
13299
13300 async fn transcribe_local_mlx(
13301 &self,
13302 schema: &ModelSchema,
13303 req: &TranscribeRequest,
13304 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13305 ) -> Result<TranscribeResult, InferenceError> {
13306 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13307 let reservation = reservation.ok_or_else(|| {
13308 InferenceError::InferenceFailed("local STT path missing admission".into())
13309 })?;
13310 reservation
13311 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13312 .map_err(InferenceError::from)?;
13313 self.transcribe_via_speech_runtime(schema, req, reservation.detached_lease())
13322 .await
13323 }
13324
13325 async fn transcribe_via_speech_runtime(
13332 &self,
13333 schema: &ModelSchema,
13334 req: &TranscribeRequest,
13335 detached_lease: resource_policy::DetachedLocalLease,
13336 ) -> Result<TranscribeResult, InferenceError> {
13337 {
13338 let runtime = self.ensure_speech_runtime().await?;
13339 let hf_repo = match &schema.source {
13340 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13341 _ => {
13342 return Err(InferenceError::InferenceFailed(format!(
13343 "speech runtime needs an MLX model repo; {} is not one",
13344 schema.id
13345 )))
13346 }
13347 };
13348 let output_dir = temp_work_dir("stt")?;
13349 let output_prefix = output_dir.join("transcript");
13350 let mut args = vec![
13351 "--model".to_string(),
13352 hf_repo,
13353 "--audio".to_string(),
13354 req.audio_path.clone(),
13355 "--output-path".to_string(),
13356 output_prefix.display().to_string(),
13357 "--format".to_string(),
13358 "json".to_string(),
13359 ];
13360 if let Some(language) = &req.language {
13361 args.push("--language".to_string());
13362 args.push(normalize_lang_code(language));
13363 }
13364 if let Some(prompt) = &req.prompt {
13365 args.push("--context".to_string());
13366 args.push(prompt.clone());
13367 }
13368 if req.timestamps {
13369 args.push("--verbose".to_string());
13370 }
13371
13372 let output =
13373 run_mlx_audio_command(&runtime, "stt.generate", &args, detached_lease).await?;
13374 let text = read_transcription_result(&output_prefix)?
13375 .or_else(|| extract_text_from_payload(&output.stdout))
13376 .ok_or_else(|| {
13377 InferenceError::InferenceFailed(format!(
13378 "mlx-audio transcription returned no text: {}",
13379 output.stderr
13380 ))
13381 })?;
13382
13383 Ok(TranscribeResult {
13384 text,
13385 model_used: Some(schema.name.clone()),
13386 language: req.language.clone(),
13387 words: Vec::new(),
13388 routing_explanation: None,
13389 })
13390 }
13391 }
13392
13393 async fn synthesize_local_mlx(
13394 &self,
13395 schema: &ModelSchema,
13396 req: &SynthesizeRequest,
13397 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13398 ) -> Result<SynthesizeResult, InferenceError> {
13399 let requested = req.requested_advanced_controls();
13404 let repo_supports_advanced = match &schema.source {
13405 ModelSource::Mlx { hf_repo, .. } => hf_repo.to_ascii_lowercase().contains("qwen3-tts"),
13406 _ => false,
13407 };
13408 if !requested.is_empty() && !repo_supports_advanced {
13409 if req.strict_capabilities {
13410 return Err(InferenceError::InferenceFailed(format!(
13411 "model {name} does not support Qwen3-TTS advanced controls {requested:?}; \
13412 route to a Qwen3-TTS model or set strict_capabilities = false to degrade",
13413 name = schema.name,
13414 )));
13415 }
13416 tracing::warn!(
13417 model = %schema.name,
13418 fields = ?requested,
13419 "Qwen3-TTS advanced controls set on non-Qwen3-TTS backend — ignored \
13420 (set strict_capabilities=true to error instead)"
13421 );
13422 }
13423
13424 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13425 let reservation = reservation.ok_or_else(|| {
13426 InferenceError::InferenceFailed("local TTS path missing admission".into())
13427 })?;
13428 reservation
13429 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13430 .map_err(InferenceError::from)?;
13431
13432 self.synthesize_via_speech_runtime(schema, req, reservation.detached_lease())
13442 .await
13443 }
13444
13445 async fn synthesize_via_speech_runtime(
13458 &self,
13459 schema: &ModelSchema,
13460 req: &SynthesizeRequest,
13461 detached_lease: resource_policy::DetachedLocalLease,
13462 ) -> Result<SynthesizeResult, InferenceError> {
13463 let runtime = self.ensure_speech_runtime().await?;
13464 let primary_hf_repo = match &schema.source {
13465 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13466 _ => {
13467 return Err(InferenceError::InferenceFailed(format!(
13468 "speech runtime needs an MLX model repo; {} is not one",
13469 schema.id
13470 )))
13471 }
13472 };
13473 let (produced, model_used) = match self
13474 .synthesize_local_mlx_repo(
13475 &runtime,
13476 &primary_hf_repo,
13477 schema.name.as_str(),
13478 req,
13479 detached_lease.clone(),
13480 )
13481 .await
13482 {
13483 Ok(result) => result,
13484 Err(primary_err)
13485 if primary_hf_repo == "mlx-community/Kokoro-82M-6bit"
13486 && kokoro_runtime_fallback_enabled() =>
13487 {
13488 let fallback_repo = "mlx-community/Kokoro-82M-bf16";
13489 let fallback_name = "Kokoro-82M-bf16";
13490 match self
13491 .synthesize_local_mlx_repo(
13492 &runtime,
13493 fallback_repo,
13494 fallback_name,
13495 req,
13496 detached_lease.clone(),
13497 )
13498 .await
13499 {
13500 Ok(result) => result,
13501 Err(fallback_err) => {
13502 return Err(InferenceError::InferenceFailed(format!(
13503 "{primary_err}; fallback {fallback_name} also failed: {fallback_err}"
13504 )));
13505 }
13506 }
13507 }
13508 Err(err) => return Err(err),
13509 };
13510 let final_path =
13511 materialize_audio_output(&produced, req.output_path.as_deref(), &req.format)?;
13512
13513 Ok(SynthesizeResult {
13514 audio_path: final_path.display().to_string(),
13515 media_type: media_type_for_format(&req.format),
13516 model_used: Some(model_used),
13517 voice_used: req.voice.clone(),
13518 routing_explanation: None,
13519 })
13520 }
13521
13522 async fn synthesize_local_mlx_repo(
13523 &self,
13524 runtime: &SpeechRuntime,
13525 hf_repo: &str,
13526 model_name: &str,
13527 req: &SynthesizeRequest,
13528 detached_lease: resource_policy::DetachedLocalLease,
13529 ) -> Result<(PathBuf, String), InferenceError> {
13530 let output_dir = temp_work_dir("tts")?;
13531 let mut args = vec![
13532 "--model".to_string(),
13533 hf_repo.to_string(),
13534 "--text".to_string(),
13535 req.text.clone(),
13536 "--output_path".to_string(),
13537 output_dir.display().to_string(),
13538 ];
13539 if let Some(voice) = &req.voice {
13540 args.push("--voice".to_string());
13541 args.push(voice.clone());
13542 }
13543 if let Some(speed) = req.speed {
13544 args.push("--speed".to_string());
13545 args.push(speed.to_string());
13546 }
13547 let repo_lower = hf_repo.to_ascii_lowercase();
13548 if repo_lower.contains("kokoro") {
13549 args.push("--lang_code".to_string());
13550 args.push(kokoro_lang_code(req.language.as_deref()).to_string());
13551 } else if let Some(language) = &req.language {
13552 args.push("--lang_code".to_string());
13553 args.push(normalize_lang_code(language));
13554 }
13555
13556 if repo_lower.contains("qwen3-tts") {
13562 if let Some(ref_audio) = &req.reference_audio_path {
13563 args.push("--ref_audio".to_string());
13564 args.push(ref_audio.clone());
13565 }
13566 if let Some(ref_text) = &req.reference_text {
13567 args.push("--ref_text".to_string());
13568 args.push(ref_text.clone());
13569 }
13570 if let Some(instruct) = &req.voice_instruction {
13571 args.push("--instruct".to_string());
13572 args.push(instruct.clone());
13573 }
13574 }
13575
13576 let output = if repo_lower.contains("kokoro") {
13577 let device = std::env::var("CAR_SPEECH_KOKORO_DEVICE")
13578 .or_else(|_| std::env::var("CAR_SPEECH_MLX_DEVICE"))
13579 .unwrap_or_else(|_| "cpu".to_string());
13580 let extra_env = vec![
13581 ("MLX_DEVICE".to_string(), device),
13583 ("PYTORCH_ENABLE_MPS_FALLBACK".to_string(), "1".to_string()),
13585 ];
13586 run_mlx_audio_command_with_env(
13587 runtime,
13588 "tts.generate",
13589 &args,
13590 &extra_env,
13591 detached_lease,
13592 )
13593 .await?
13594 } else {
13595 run_mlx_audio_command(runtime, "tts.generate", &args, detached_lease).await?
13596 };
13597 let produced = find_audio_file(&output_dir)?.ok_or_else(|| {
13598 let hint = if repo_lower.contains("kokoro") {
13599 ". Kokoro models may crash on GPU — try CAR_SPEECH_KOKORO_DEVICE=cpu or use the default Qwen3-TTS model"
13600 } else {
13601 ""
13602 };
13603 InferenceError::InferenceFailed(format!(
13604 "mlx-audio synthesis produced no audio file: {}{}",
13605 output.stderr, hint
13606 ))
13607 })?;
13608 Ok((produced, model_name.to_string()))
13609 }
13610
13611 async fn transcribe_elevenlabs(
13612 &self,
13613 schema: &ModelSchema,
13614 req: &TranscribeRequest,
13615 ) -> Result<TranscribeResult, InferenceError> {
13616 let (endpoint, api_key) = elevenlabs_auth(
13617 schema,
13618 crate::tasks::transcribe::resolve_provider_credential_for_request,
13619 )?;
13620 let file_name = Path::new(&req.audio_path)
13621 .file_name()
13622 .and_then(|f| f.to_str())
13623 .unwrap_or("audio.wav")
13624 .to_string();
13625 let audio_bytes = tokio::fs::read(&req.audio_path).await?;
13626 let file_part = Part::bytes(audio_bytes).file_name(file_name);
13627 let mut form = Form::new()
13628 .text("model_id", schema.name.clone())
13629 .part("file", file_part);
13630 if let Some(language) = &req.language {
13631 form = form.text("language_code", language.clone());
13632 }
13633
13634 let resp = self
13635 .remote_backend
13636 .client
13637 .post(format!(
13638 "{}/v1/speech-to-text",
13639 endpoint.trim_end_matches('/')
13640 ))
13641 .header("xi-api-key", api_key)
13642 .multipart(form)
13643 .send()
13644 .await
13645 .map_err(|e| {
13646 self.remote_backend
13647 .request_error("ElevenLabs STT request failed", &e)
13648 })?;
13649 let status = resp.status();
13650 let body = resp.text().await.map_err(|e| {
13651 InferenceError::InferenceFailed(format!("read ElevenLabs STT body: {e}"))
13652 })?;
13653 if !status.is_success() {
13654 return Err(InferenceError::InferenceFailed(format!(
13655 "ElevenLabs STT returned {status}: {body}"
13656 )));
13657 }
13658 let payload: serde_json::Value = serde_json::from_str(&body).map_err(|e| {
13659 InferenceError::InferenceFailed(format!("parse ElevenLabs STT response: {e}"))
13660 })?;
13661 let text = payload
13662 .get("text")
13663 .and_then(|v| v.as_str())
13664 .map(str::to_string)
13665 .ok_or_else(|| {
13666 InferenceError::InferenceFailed("ElevenLabs STT response missing text".into())
13667 })?;
13668
13669 Ok(TranscribeResult {
13670 text,
13671 model_used: Some(schema.name.clone()),
13672 language: payload
13673 .get("language_code")
13674 .and_then(|v| v.as_str())
13675 .map(str::to_string),
13676 words: Vec::new(),
13677 routing_explanation: None,
13678 })
13679 }
13680
13681 async fn synthesize_elevenlabs(
13682 &self,
13683 schema: &ModelSchema,
13684 req: &SynthesizeRequest,
13685 ) -> Result<SynthesizeResult, InferenceError> {
13686 let requested = req.requested_advanced_controls();
13690 if !requested.is_empty() {
13691 if req.strict_capabilities {
13692 return Err(InferenceError::InferenceFailed(format!(
13693 "ElevenLabs backend does not support Qwen3-TTS advanced controls \
13694 {requested:?}; route to a Qwen3-TTS model or set strict_capabilities = false"
13695 )));
13696 }
13697 tracing::warn!(
13698 model = %schema.name,
13699 fields = ?requested,
13700 "Qwen3-TTS advanced controls ignored by ElevenLabs backend"
13701 );
13702 }
13703 let (endpoint, api_key) = elevenlabs_auth(
13704 schema,
13705 crate::tasks::synthesize::resolve_provider_credential_for_request,
13706 )?;
13707 let voice_id = req
13708 .voice
13709 .clone()
13710 .unwrap_or_else(|| "JBFqnCBsd6RMkjVDRZzb".to_string());
13711 let output_format = elevenlabs_output_format(&req.format);
13712 let url = format!(
13713 "{}/v1/text-to-speech/{}?output_format={}",
13714 endpoint.trim_end_matches('/'),
13715 voice_id,
13716 output_format
13717 );
13718
13719 let mut body = serde_json::json!({
13720 "text": req.text,
13721 "model_id": schema.name,
13722 });
13723 if let Some(language) = &req.language {
13724 body["language_code"] = serde_json::Value::String(language.clone());
13725 }
13726
13727 let resp = self
13728 .remote_backend
13729 .client
13730 .post(url)
13731 .header("xi-api-key", api_key)
13732 .header("Content-Type", "application/json")
13733 .json(&body)
13734 .send()
13735 .await
13736 .map_err(|e| {
13737 self.remote_backend
13738 .request_error("ElevenLabs TTS request failed", &e)
13739 })?;
13740 let status = resp.status();
13741 let audio = resp.bytes().await.map_err(|e| {
13742 InferenceError::InferenceFailed(format!("read ElevenLabs TTS body: {e}"))
13743 })?;
13744 if !status.is_success() {
13745 let err_body = String::from_utf8_lossy(&audio);
13746 return Err(InferenceError::InferenceFailed(format!(
13747 "ElevenLabs TTS returned {status}: {err_body}"
13748 )));
13749 }
13750
13751 let final_path = requested_or_temp_output(req.output_path.as_deref(), &req.format)?;
13752 ensure_parent_dir(&final_path)?;
13753 tokio::fs::write(&final_path, &audio).await?;
13754
13755 Ok(SynthesizeResult {
13756 audio_path: final_path.display().to_string(),
13757 media_type: media_type_for_format(&req.format),
13758 model_used: Some(schema.name.clone()),
13759 voice_used: Some(voice_id),
13760 routing_explanation: None,
13761 })
13762 }
13763}
13764
13765#[derive(Default)]
13766struct ProviderAccumulator {
13767 configured: bool,
13768 local_models: usize,
13769 remote_models: usize,
13770 available_models: usize,
13771 capabilities: std::collections::HashSet<ModelCapability>,
13772}
13773
13774struct CommandOutput {
13779 stdout: String,
13780 stderr: String,
13781}
13782
13783#[derive(Debug, Clone)]
13784struct SpeechRuntime {
13785 root: PathBuf,
13786 python: PathBuf,
13787 stt_program: PathBuf,
13788 tts_program: PathBuf,
13789}
13790
13791impl SpeechRuntime {
13792 fn new(root: PathBuf) -> Self {
13793 let python = managed_venv::interpreter(&root);
13794 let stt_program = managed_venv::venv_program(&root, "mlx_audio.stt.generate");
13795 let tts_program = managed_venv::venv_program(&root, "mlx_audio.tts.generate");
13796 Self {
13797 root,
13798 python,
13799 stt_program,
13800 tts_program,
13801 }
13802 }
13803
13804 fn is_ready(&self) -> bool {
13805 managed_venv::interpreter_healthy(&self.root)
13810 && self.stt_program.exists()
13811 && self.tts_program.exists()
13812 }
13813
13814 fn command_for(&self, subcommand: &str) -> Result<&Path, InferenceError> {
13815 match subcommand {
13816 "stt.generate" => Ok(&self.stt_program),
13817 "tts.generate" => Ok(&self.tts_program),
13818 _ => Err(InferenceError::InferenceFailed(format!(
13819 "unknown speech subcommand: {subcommand}"
13820 ))),
13821 }
13822 }
13823}
13824
13825async fn run_mlx_audio_command(
13826 runtime: &SpeechRuntime,
13827 subcommand: &str,
13828 args: &[String],
13829 detached_lease: resource_policy::DetachedLocalLease,
13830) -> Result<CommandOutput, InferenceError> {
13831 run_mlx_audio_command_with_env(runtime, subcommand, args, &[], detached_lease).await
13832}
13833
13834async fn run_admitted_blocking<F, T>(
13838 detached_lease: resource_policy::DetachedLocalLease,
13839 operation: F,
13840) -> Result<T, tokio::task::JoinError>
13841where
13842 F: FnOnce() -> T + Send + 'static,
13843 T: Send + 'static,
13844{
13845 tokio::task::spawn_blocking(move || {
13846 let _detached_lease = detached_lease;
13847 operation()
13848 })
13849 .await
13850}
13851
13852struct DetachedSpeechProcess {
13853 child: Option<tokio::process::Child>,
13854 lease: Option<resource_policy::DetachedLocalLease>,
13855}
13856
13857impl Drop for DetachedSpeechProcess {
13858 fn drop(&mut self) {
13859 let Some(mut child) = self.child.take() else {
13860 return;
13861 };
13862 let lease = self.lease.take();
13863 let _ = child.start_kill();
13864 if tokio::runtime::Handle::try_current().is_ok() {
13865 tokio::spawn(async move {
13866 let _ = child.wait().await;
13867 drop(lease);
13868 });
13869 } else {
13870 std::mem::forget((child, lease));
13873 }
13874 }
13875}
13876
13877async fn run_mlx_audio_command_with_env(
13878 runtime: &SpeechRuntime,
13879 subcommand: &str,
13880 args: &[String],
13881 envs: &[(String, String)],
13882 detached_lease: resource_policy::DetachedLocalLease,
13883) -> Result<CommandOutput, InferenceError> {
13884 let program = runtime.command_for(subcommand)?;
13885 let mut command = Command::new(program);
13886 command.args(args);
13887 crate::hf_cache::apply_to(&mut command);
13888 for (key, value) in envs {
13889 command.env(key, value);
13890 }
13891 command
13892 .stdout(std::process::Stdio::piped())
13893 .stderr(std::process::Stdio::piped())
13894 .kill_on_drop(true);
13895 let child = command
13896 .spawn()
13897 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
13898 let mut owned = DetachedSpeechProcess {
13899 child: Some(child),
13900 lease: Some(detached_lease),
13901 };
13902 let mut stdout = owned
13903 .child
13904 .as_mut()
13905 .expect("speech child owned")
13906 .stdout
13907 .take()
13908 .ok_or_else(|| {
13909 InferenceError::InferenceFailed(format!("{} stdout unavailable", program.display()))
13910 })?;
13911 let mut stderr = owned
13912 .child
13913 .as_mut()
13914 .expect("speech child owned")
13915 .stderr
13916 .take()
13917 .ok_or_else(|| {
13918 InferenceError::InferenceFailed(format!("{} stderr unavailable", program.display()))
13919 })?;
13920 let stdout_reader = tokio::spawn(async move {
13921 let mut bytes = Vec::new();
13922 stdout.read_to_end(&mut bytes).await.map(|_| bytes)
13923 });
13924 let stderr_reader = tokio::spawn(async move {
13925 let mut bytes = Vec::new();
13926 stderr.read_to_end(&mut bytes).await.map(|_| bytes)
13927 });
13928 let status = owned
13929 .child
13930 .as_mut()
13931 .expect("speech child owned")
13932 .wait()
13933 .await
13934 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
13935 owned.child.take();
13936 owned.lease.take();
13937 let stdout = stdout_reader
13938 .await
13939 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout join: {err}")))?
13940 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout: {err}")))?;
13941 let stderr = stderr_reader
13942 .await
13943 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr join: {err}")))?
13944 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr: {err}")))?;
13945
13946 if status.success() {
13947 Ok(CommandOutput {
13948 stdout: String::from_utf8_lossy(&stdout).to_string(),
13949 stderr: String::from_utf8_lossy(&stderr).to_string(),
13950 })
13951 } else {
13952 Err(InferenceError::InferenceFailed(format!(
13953 "{} exited with {}: {}",
13954 program.display(),
13955 status,
13956 String::from_utf8_lossy(&stderr)
13957 )))
13958 }
13959}
13960
13961async fn bootstrap_speech_runtime(runtime: &SpeechRuntime) -> Result<(), InferenceError> {
13962 let python = select_speech_python()?;
13963
13964 let outcome = managed_venv::ensure_venv(&runtime.root, &python)
13970 .await
13971 .map_err(|err| InferenceError::InferenceFailed(err.to_string()))?;
13972 if outcome == managed_venv::VenvOutcome::Recreated {
13973 tracing::warn!(
13974 root = %runtime.root.display(),
13975 "managed speech runtime had an unusable interpreter; rebuilt it \
13976 (installed packages were discarded and are being reinstalled)"
13977 );
13978 }
13979
13980 run_command(
13981 "uv",
13982 &[
13983 "pip".to_string(),
13984 "install".to_string(),
13985 "--python".to_string(),
13986 runtime.python.display().to_string(),
13987 speech_runtime_mlx_audio_spec(),
13988 "misaki[en]".to_string(),
13989 speech_runtime_spacy_model_spec(),
13990 ],
13991 )
13992 .await?;
13993
13994 Ok(())
13995}
13996
13997async fn run_command(program: &str, args: &[String]) -> Result<(), InferenceError> {
13998 let output = Command::new(program)
13999 .args(args)
14000 .output()
14001 .await
14002 .map_err(|err| InferenceError::InferenceFailed(format!("{program}: {err}")))?;
14003
14004 if output.status.success() {
14005 Ok(())
14006 } else {
14007 Err(InferenceError::InferenceFailed(format!(
14008 "{} exited with {}: {}",
14009 program,
14010 output.status,
14011 String::from_utf8_lossy(&output.stderr)
14012 )))
14013 }
14014}
14015
14016fn select_speech_python() -> Result<String, InferenceError> {
14017 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14018 if !path.trim().is_empty() {
14019 return Ok(path);
14020 }
14021 }
14022
14023 for candidate in ["python3.13", "python3.12", "python3.11"] {
14024 if command_in_path(candidate) {
14025 return Ok(candidate.to_string());
14026 }
14027 }
14028
14029 Ok(SPEECH_RUNTIME_FALLBACK_PYTHON.to_string())
14039}
14040
14041const SPEECH_RUNTIME_FALLBACK_PYTHON: &str = "3.12";
14047
14048fn detect_speech_python() -> Option<String> {
14049 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14050 if !path.trim().is_empty() {
14051 return Some(path);
14052 }
14053 }
14054
14055 ["python3.13", "python3.12", "python3.11"]
14056 .into_iter()
14057 .find(|candidate| command_in_path(candidate))
14058 .map(str::to_string)
14059}
14060
14061fn speech_runtime_root_from_models_dir(_models_dir: &Path) -> PathBuf {
14062 if let Ok(path) = std::env::var("CAR_SPEECH_RUNTIME_DIR") {
14063 if !path.trim().is_empty() {
14064 return PathBuf::from(path);
14065 }
14066 }
14067
14068 std::env::var_os("HOME")
14069 .or_else(|| std::env::var_os("USERPROFILE"))
14070 .map(PathBuf::from)
14071 .unwrap_or_else(|| PathBuf::from("."))
14072 .join(".car")
14073 .join("speech-runtime")
14074}
14075
14076fn command_in_path(name: &str) -> bool {
14077 std::env::var_os("PATH")
14078 .map(|paths| {
14079 std::env::split_paths(&paths).any(|dir| {
14080 let path = dir.join(name);
14081 path.exists() && path.is_file()
14082 })
14083 })
14084 .unwrap_or(false)
14085}
14086
14087fn speech_model_cached(schema: &ModelSchema) -> bool {
14088 match &schema.source {
14089 ModelSource::Mlx { hf_repo, .. } => huggingface_repo_has_snapshot(hf_repo),
14090 ModelSource::WhisperCpp { model } => car_whisper::model_cached(model),
14091 ModelSource::WindowsSpeech {} => cfg!(target_os = "windows"),
14093 ModelSource::Proprietary { .. } => false,
14096 _ => false,
14097 }
14098}
14099
14100fn model_source_configured(schema: &ModelSchema, credential: &dyn Fn(&str) -> bool) -> bool {
14120 match &schema.source {
14121 ModelSource::RemoteApi {
14122 protocol: ApiProtocol::OpenRouter,
14123 ..
14124 } => crate::openrouter::credential_source().is_some(),
14125 ModelSource::RemoteApi {
14126 api_key_env,
14127 api_key_envs,
14128 ..
14129 } => credential(api_key_env) || api_key_envs.iter().any(|env_var| credential(env_var)),
14130 ModelSource::Proprietary { auth, .. } => match auth {
14131 ProprietaryAuth::ApiKeyEnv { env_var } => credential(env_var),
14132 ProprietaryAuth::BearerTokenEnv { env_var } => credential(env_var),
14133 ProprietaryAuth::OAuth2Pkce { .. } => matches!(
14134 car_auth::credential_authority_hint().state,
14135 car_auth::CredentialAuthorityState::Configured
14136 ),
14137 ProprietaryAuth::ChatGptSubscription {} => {
14138 credential(car_auth::CODEX_AUTH_RECORD_KEY)
14139 || crate::registry::passive_codex_oauth_available()
14140 }
14141 },
14142 ModelSource::VllmMlx { .. } => {
14143 std::env::var("VLLM_MLX_ENDPOINT").is_ok() || schema.available
14144 }
14145 ModelSource::Ollama { .. } => schema.available,
14146 ModelSource::CodexCli { .. } => crate::backend::codex_cli::is_available(),
14147 ModelSource::Mlx { .. }
14148 | ModelSource::ManagedVllmMlx { .. }
14149 | ModelSource::Local { .. }
14150 | ModelSource::WhisperCpp { .. } => true,
14151 ModelSource::WindowsSpeech {} => schema.available,
14153 ModelSource::AppleFoundationModels { .. } => schema.available,
14154 ModelSource::Delegated { .. } => true,
14159 }
14160}
14161
14162fn scoreboard_price_model(schema: &ModelSchema) -> Option<crate::scoreboard::PriceModel> {
14163 if schema.is_subscription_billed() {
14164 return None;
14165 }
14166 match (schema.cost.input_per_mtok, schema.cost.output_per_mtok) {
14167 (Some(input_per_mtok), Some(output_per_mtok)) => Some(crate::scoreboard::PriceModel {
14171 input_per_mtok,
14172 output_per_mtok,
14173 cache: schema.cache_rates(),
14174 is_estimate: !schema.cost.pricing_tiers.is_empty(),
14175 }),
14176 _ => None,
14177 }
14178}
14179
14180fn all_model_capabilities() -> [ModelCapability; 13] {
14181 [
14182 ModelCapability::Generate,
14183 ModelCapability::Embed,
14184 ModelCapability::Classify,
14185 ModelCapability::Code,
14186 ModelCapability::Reasoning,
14187 ModelCapability::Summarize,
14188 ModelCapability::ToolUse,
14189 ModelCapability::MultiToolCall,
14190 ModelCapability::Vision,
14191 ModelCapability::SpeechToText,
14192 ModelCapability::TextToSpeech,
14193 ModelCapability::ImageGeneration,
14194 ModelCapability::VideoGeneration,
14195 ]
14196}
14197
14198fn sort_capabilities(mut capabilities: Vec<ModelCapability>) -> Vec<ModelCapability> {
14199 capabilities.sort_by_key(|capability| {
14200 all_model_capabilities()
14201 .iter()
14202 .position(|candidate| candidate == capability)
14203 .unwrap_or(usize::MAX)
14204 });
14205 capabilities
14206}
14207
14208fn speech_model_source_label(schema: &ModelSchema) -> String {
14209 match &schema.source {
14210 ModelSource::Mlx { hf_repo, .. } => format!("mlx:{hf_repo}"),
14211 ModelSource::ManagedVllmMlx { hf_repo, .. } => {
14212 format!("managed-vllm-mlx:{hf_repo}")
14213 }
14214 ModelSource::WhisperCpp { model } => format!("whisper:{model}"),
14215 ModelSource::WindowsSpeech {} => "windows-speech".to_string(),
14216 ModelSource::Proprietary {
14217 provider, endpoint, ..
14218 } => format!("proprietary:{provider}:{endpoint}"),
14219 ModelSource::RemoteApi { endpoint, .. } => format!("remote:{endpoint}"),
14220 ModelSource::CodexCli { model } => format!("codex-cli:{model}"),
14221 ModelSource::Local { hf_repo, .. } => format!("local:{hf_repo}"),
14222 ModelSource::VllmMlx {
14223 endpoint,
14224 model_name,
14225 } => format!("vllm-mlx:{endpoint}:{model_name}"),
14226 ModelSource::Ollama { model_tag, host } => format!("ollama:{host}:{model_tag}"),
14227 ModelSource::AppleFoundationModels { use_case } => {
14228 format!(
14229 "apple-foundation:{}",
14230 use_case.as_deref().unwrap_or("default")
14231 )
14232 }
14233 ModelSource::Delegated { hint } => {
14234 format!("delegated:{}", hint.as_deref().unwrap_or("(none)"))
14235 }
14236 }
14237}
14238
14239fn rerank_prompt(instruction: &str, query: &str, document: &str) -> String {
14247 const SYSTEM: &str = "Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".";
14248 format!(
14249 "<|im_start|>system\n{SYSTEM}<|im_end|>\n\
14250 <|im_start|>user\n<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {document}<|im_end|>\n\
14251 <|im_start|>assistant\n<think>\n\n</think>\n\n"
14252 )
14253}
14254
14255fn parse_api_returned_status(err: &str) -> Option<u16> {
14279 let lower = err.to_ascii_lowercase();
14280 let idx = lower.find("api returned ")?;
14281 let digits: String = lower[idx + "api returned ".len()..]
14282 .chars()
14283 .take_while(|c| c.is_ascii_digit())
14284 .take(3)
14285 .collect();
14286 digits.parse().ok()
14287}
14288
14289pub const NO_BACKEND_RECOVERY_MARKER: &str = "no inference backend is available.";
14302
14303const AUTO_GRACE_FILE: &str = "concierge-auto-since";
14306const AUTO_GRACE_SECS: u64 = 7 * 24 * 60 * 60;
14308
14309#[doc(hidden)]
14313pub const LOCAL_TOOL_MODEL_PULL_ID: &str =
14314 if cfg!(all(
14317 target_os = "macos",
14318 target_arch = "aarch64",
14319 not(car_skip_mlx)
14320 )) {
14321 "mlx/qwen3-4b:4bit"
14322 } else {
14323 "qwen/qwen3-4b:q4_k_m"
14324 };
14325
14326#[doc(hidden)]
14333pub fn no_backend_recovery_hint(underlying: &str) -> Option<String> {
14334 let is_no_backend = underlying.contains("no credential")
14335 || underlying.contains("model not found")
14336 || underlying.contains("no models available")
14337 || underlying.contains("no inference runner");
14338 if !is_no_backend {
14339 return None;
14340 }
14341 Some(format!(
14347 "{NO_BACKEND_RECOVERY_MARKER} To use Parslee's hosted models, run:\n \
14348 car auth login\n\
14349 Or, for a model you name yourself with --model, install a local \
14350 tool-capable one:\n \
14351 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
14352 (underlying error: {underlying})"
14353 ))
14354}
14355
14356pub fn is_auth_rejection_message(underlying: &str) -> bool {
14374 let l = underlying.to_ascii_lowercase();
14375 l.contains("org lookup failed")
14376 || l.contains("authentication required")
14377 || (l.contains("401") && l.contains("unauthorized"))
14378 || (l.contains("403") && l.contains("forbidden"))
14379 || l.contains("invalid_grant")
14380 || l.contains("token expired")
14381}
14382
14383pub fn is_auth_failure_message(message: &str) -> bool {
14390 let lower = message.to_ascii_lowercase();
14391 AUTH_FAILURE_MESSAGE_MARKERS
14392 .iter()
14393 .any(|marker| lower.contains(marker))
14394 || is_auth_rejection_message(message)
14395}
14396
14397fn record_auth_dead_lane(slot: &mut Option<String>, candidate: &str, error: &str) {
14406 if slot.is_none() && is_auth_rejection_message(error) {
14407 *slot = Some(candidate.to_string());
14408 }
14409}
14410
14411fn is_local_last_resort(appended_id: Option<&str>, candidate_id: &str) -> bool {
14416 appended_id == Some(candidate_id)
14417}
14418
14419fn record_generation_span(
14435 model: &str,
14436 latency_ms: u64,
14437 ttft_ms: Option<u64>,
14438 usage: Option<&TokenUsage>,
14439) {
14440 let span = tracing::Span::current();
14441 span.record("model", model);
14442 span.record("gen_ai.response.model", model);
14443 span.record("latency_ms", latency_ms);
14444 if let Some(ttft) = ttft_ms {
14445 span.record("ttft_ms", ttft);
14446 }
14447 if let Some(u) = usage {
14448 span.record("prompt_tokens", u.prompt_tokens);
14449 span.record("completion_tokens", u.completion_tokens);
14450 span.record("gen_ai.usage.input_tokens", u.prompt_tokens);
14451 span.record("gen_ai.usage.output_tokens", u.completion_tokens);
14452 }
14453}
14454
14455fn report_local_last_resort_served(
14456 appended_id: Option<&str>,
14457 candidate_id: &str,
14458 resolved_id: &str,
14459) -> bool {
14460 let served = is_local_last_resort(appended_id, candidate_id);
14461 if served {
14462 tracing::warn!(
14463 local_model = %resolved_id,
14464 "on-device last-resort model served inference turn"
14465 );
14466 }
14467 served
14468}
14469
14470fn auth_expired_recovery_hint(underlying: &str) -> Option<String> {
14480 if !is_auth_rejection_message(underlying) {
14481 return None;
14482 }
14483 Some(format!(
14484 "your Parslee session has expired or was rejected — re-authenticate:\n \
14485 car auth login\n\
14486 Or, for a model you name yourself with --model, install a local \
14487 tool-capable one:\n \
14488 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
14489 (underlying error: {underlying})"
14490 ))
14491}
14492
14493fn score_from_rerank_output(text: &str, model_name: &str) -> f32 {
14494 let normalized: String = text
14499 .to_ascii_lowercase()
14500 .chars()
14501 .map(|c| if c.is_ascii_alphanumeric() { c } else { ' ' })
14502 .collect();
14503 for tok in normalized.split_ascii_whitespace().take(5) {
14504 match tok {
14505 "yes" => return 1.0,
14506 "no" => return 0.0,
14507 _ => continue,
14508 }
14509 }
14510 tracing::warn!(
14511 model = %model_name,
14512 output = %text,
14513 "rerank: first tokens contain neither `yes` nor `no`; returning neutral 0.5"
14514 );
14515 0.5
14516}
14517
14518fn default_speech_voice(schema: &ModelSchema) -> Option<String> {
14519 if schema.provider == "elevenlabs" {
14520 Some("JBFqnCBsd6RMkjVDRZzb".to_string())
14521 } else if schema.name == "Kokoro-82M-6bit" || schema.name == "Kokoro-82M-bf16" {
14522 Some("af_heart".to_string())
14523 } else if schema.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
14524 Some("Chelsie".to_string())
14525 } else {
14526 None
14527 }
14528}
14529
14530#[allow(dead_code)] fn huggingface_repo_has_snapshot(repo_id: &str) -> bool {
14532 crate::registry::latest_huggingface_repo_snapshot(repo_id).is_some()
14533}
14534
14535#[allow(dead_code)] fn count_files_recursive(root: &Path) -> usize {
14537 let Ok(entries) = std::fs::read_dir(root) else {
14538 return 0;
14539 };
14540 entries
14541 .filter_map(Result::ok)
14542 .map(|entry| entry.path())
14543 .map(|path| {
14544 if path.is_dir() {
14545 count_files_recursive(&path)
14546 } else if path.is_file() {
14547 1
14548 } else {
14549 0
14550 }
14551 })
14552 .sum()
14553}
14554
14555async fn download_hf_repo_snapshot(repo_id: &str) -> Result<(PathBuf, usize), InferenceError> {
14556 let api = crate::hf_cache::api()
14557 .with_progress(false)
14558 .build()
14559 .map_err(|e| InferenceError::DownloadFailed(format!("init hf api: {e}")))?;
14560 let repo = api.model(repo_id.to_string());
14561 let info = repo
14562 .info()
14563 .await
14564 .map_err(|e| InferenceError::DownloadFailed(format!("{repo_id}: {e}")))?;
14565
14566 let snapshot_path = crate::hf_cache::repo_dir(repo_id)
14567 .join("snapshots")
14568 .join(&info.sha);
14569 let mut downloaded = 0usize;
14570 for sibling in &info.siblings {
14571 let local_path = snapshot_path.join(&sibling.rfilename);
14572 if crate::download::cache_file_usable(&local_path) {
14580 downloaded += 1;
14581 continue;
14582 }
14583 let _ = std::fs::remove_file(&local_path);
14588 repo.download(&sibling.rfilename).await.map_err(|e| {
14589 InferenceError::DownloadFailed(format!("{repo_id}/{}: {e}", sibling.rfilename))
14590 })?;
14591 crate::retire::note_fetched(repo_id, &sibling.rfilename, Some(info.sha.clone()));
14592 downloaded += 1;
14593 }
14594
14595 Ok((snapshot_path, downloaded))
14596}
14597
14598fn temp_work_dir(prefix: &str) -> Result<PathBuf, InferenceError> {
14599 let unique = SystemTime::now()
14600 .duration_since(UNIX_EPOCH)
14601 .map_err(|e| InferenceError::InferenceFailed(format!("clock error: {e}")))?
14602 .as_nanos();
14603 let dir = std::env::temp_dir().join(format!("car-inference-{prefix}-{unique}"));
14604 std::fs::create_dir_all(&dir)?;
14605 Ok(dir)
14606}
14607
14608fn ensure_parent_dir(path: &Path) -> Result<(), InferenceError> {
14609 if let Some(parent) = path.parent() {
14610 std::fs::create_dir_all(parent)?;
14611 }
14612 Ok(())
14613}
14614
14615fn requested_or_temp_output(
14616 output_path: Option<&str>,
14617 format: &str,
14618) -> Result<PathBuf, InferenceError> {
14619 if let Some(path) = output_path {
14620 return Ok(PathBuf::from(path));
14621 }
14622 let dir = temp_work_dir("audio-out")?;
14623 Ok(dir.join(format!("speech.{format}")))
14624}
14625
14626#[allow(dead_code)] fn requested_or_temp_media_output(
14628 output_path: Option<&str>,
14629 format: &str,
14630 stem: &str,
14631) -> Result<PathBuf, InferenceError> {
14632 if let Some(path) = output_path {
14633 return Ok(PathBuf::from(path));
14634 }
14635 let dir = temp_work_dir(&format!("{stem}-out"))?;
14636 Ok(dir.join(format!("{stem}.{format}")))
14637}
14638
14639fn materialize_audio_output(
14640 produced: &Path,
14641 requested: Option<&str>,
14642 format: &str,
14643) -> Result<PathBuf, InferenceError> {
14644 if let Some(path) = requested {
14645 let dest = PathBuf::from(path);
14646 ensure_parent_dir(&dest)?;
14647 std::fs::copy(produced, &dest)?;
14648 Ok(dest)
14649 } else {
14650 let dest = requested_or_temp_output(None, format)?;
14651 ensure_parent_dir(&dest)?;
14652 std::fs::copy(produced, &dest)?;
14653 Ok(dest)
14654 }
14655}
14656
14657#[cfg(target_os = "windows")]
14663fn winrt_synthesize_wav(text: &str, voice: &str, rate: f64) -> Result<Vec<u8>, InferenceError> {
14664 use windows::core::HSTRING;
14665 use windows::Media::SpeechSynthesis::SpeechSynthesizer;
14666 use windows::Storage::Streams::DataReader;
14667
14668 let err = |m: String| InferenceError::InferenceFailed(m);
14669 let synth =
14670 SpeechSynthesizer::new().map_err(|e| err(format!("SpeechSynthesizer::new: {e}")))?;
14671 if let Ok(opts) = synth.Options() {
14672 let _ = opts.SetSpeakingRate(rate.clamp(0.5, 6.0));
14673 }
14674 if !voice.is_empty() {
14675 if let Ok(all) = SpeechSynthesizer::AllVoices() {
14676 let want = voice.to_lowercase();
14677 let count = all.Size().unwrap_or(0);
14678 for i in 0..count {
14679 if let Ok(info) = all.GetAt(i) {
14680 if let Ok(name) = info.DisplayName() {
14681 if name.to_string_lossy().to_lowercase().contains(&want) {
14682 let _ = synth.SetVoice(&info);
14683 break;
14684 }
14685 }
14686 }
14687 }
14688 }
14689 }
14690 let stream = synth
14691 .SynthesizeTextToStreamAsync(&HSTRING::from(text))
14692 .map_err(|e| err(format!("SynthesizeTextToStreamAsync: {e}")))?
14693 .get()
14694 .map_err(|e| err(format!("synthesize await: {e}")))?;
14695 let size = stream
14696 .Size()
14697 .map_err(|e| err(format!("stream size: {e}")))?;
14698 let input = stream
14699 .GetInputStreamAt(0)
14700 .map_err(|e| err(format!("input stream: {e}")))?;
14701 let reader =
14702 DataReader::CreateDataReader(&input).map_err(|e| err(format!("data reader: {e}")))?;
14703 reader
14704 .LoadAsync(size as u32)
14705 .map_err(|e| err(format!("load async: {e}")))?
14706 .get()
14707 .map_err(|e| err(format!("load await: {e}")))?;
14708 let mut buf = vec![0u8; size as usize];
14709 reader
14710 .ReadBytes(&mut buf)
14711 .map_err(|e| err(format!("read bytes: {e}")))?;
14712 Ok(buf)
14713}
14714
14715#[allow(dead_code)] fn read_transcription_result(output_prefix: &Path) -> Result<Option<String>, InferenceError> {
14717 let candidates = [
14718 output_prefix.with_extension("json"),
14719 output_prefix.to_path_buf(),
14720 ];
14721
14722 for path in candidates {
14723 if path.exists() {
14724 let contents = std::fs::read_to_string(path)?;
14725 if let Some(text) = extract_text_from_payload(&contents) {
14726 return Ok(Some(text));
14727 }
14728 }
14729 }
14730
14731 Ok(None)
14732}
14733
14734#[allow(dead_code)] fn extract_text_from_payload(payload: &str) -> Option<String> {
14736 let value: serde_json::Value = serde_json::from_str(payload).ok()?;
14737 if let Some(text) = value.get("text").and_then(|v| v.as_str()) {
14738 return Some(text.to_string());
14739 }
14740 if let Some(transcripts) = value.get("transcripts").and_then(|v| v.as_array()) {
14741 let joined = transcripts
14742 .iter()
14743 .filter_map(|item| item.get("text").and_then(|v| v.as_str()))
14744 .collect::<Vec<_>>()
14745 .join("\n");
14746 if !joined.is_empty() {
14747 return Some(joined);
14748 }
14749 }
14750 if let Some(items) = value.as_array() {
14751 let joined = items
14752 .iter()
14753 .filter_map(|item| {
14754 item.get("text")
14755 .or_else(|| item.get("Content"))
14756 .and_then(|v| v.as_str())
14757 })
14758 .collect::<Vec<_>>()
14759 .join(" ");
14760 if !joined.is_empty() {
14761 return Some(joined);
14762 }
14763 }
14764 None
14765}
14766
14767#[allow(dead_code)] fn find_audio_file(output_dir: &Path) -> Result<Option<PathBuf>, InferenceError> {
14769 let mut audio_files = Vec::new();
14770 collect_audio_files(output_dir, &mut audio_files)?;
14771 audio_files.sort();
14772 Ok(audio_files.into_iter().next())
14773}
14774
14775#[allow(dead_code)] fn collect_audio_files(dir: &Path, audio_files: &mut Vec<PathBuf>) -> Result<(), InferenceError> {
14777 for entry in std::fs::read_dir(dir)? {
14778 let path = entry?.path();
14779 if path.is_dir() {
14780 collect_audio_files(&path, audio_files)?;
14781 } else if matches!(
14782 path.extension().and_then(|ext| ext.to_str()),
14783 Some("wav" | "mp3" | "flac" | "pcm" | "m4a")
14784 ) {
14785 audio_files.push(path);
14786 }
14787 }
14788 Ok(())
14789}
14790
14791fn media_type_for_format(format: &str) -> String {
14792 match format.to_ascii_lowercase().as_str() {
14793 "mp3" => "audio/mpeg".to_string(),
14794 "flac" => "audio/flac".to_string(),
14795 "pcm" => "audio/L16".to_string(),
14796 "m4a" => "audio/mp4".to_string(),
14797 _ => "audio/wav".to_string(),
14798 }
14799}
14800
14801fn kokoro_lang_code(language: Option<&str>) -> &'static str {
14802 match language.unwrap_or("en").to_ascii_lowercase().as_str() {
14803 "en-gb" | "british" | "british english" => "b",
14804 "ja" | "japanese" => "j",
14805 "zh" | "zh-cn" | "mandarin" | "chinese" => "z",
14806 "es" | "spanish" => "e",
14807 "fr" | "french" => "f",
14808 _ => "a",
14809 }
14810}
14811
14812#[allow(dead_code)] fn normalize_lang_code(language: &str) -> String {
14814 match language.to_ascii_lowercase().as_str() {
14815 "english" | "en-us" | "en_us" => "en".to_string(),
14816 "spanish" => "es".to_string(),
14817 "french" => "fr".to_string(),
14818 "japanese" => "ja".to_string(),
14819 "chinese" | "mandarin" => "zh".to_string(),
14820 other => match other {
14821 "en" | "es" | "fr" | "ja" | "zh" => other.to_string(),
14822 _ => "en".to_string(),
14823 },
14824 }
14825}
14826
14827fn elevenlabs_auth(
14828 schema: &ModelSchema,
14829 resolve_credential: fn(&str) -> Option<String>,
14830) -> Result<(String, String), InferenceError> {
14831 match &schema.source {
14832 ModelSource::Proprietary {
14833 endpoint,
14834 auth: schema::ProprietaryAuth::ApiKeyEnv { env_var },
14835 ..
14836 } => {
14837 let key = resolve_credential(env_var).ok_or_else(|| {
14838 InferenceError::InferenceFailed(format!(
14839 "missing API key {env_var}; set the environment variable or \
14840 store it with `car secrets put {env_var}`"
14841 ))
14842 })?;
14843 Ok((endpoint.clone(), key))
14844 }
14845 _ => Err(InferenceError::InferenceFailed(format!(
14846 "model {} is not an ElevenLabs proprietary model",
14847 schema.id
14848 ))),
14849 }
14850}
14851
14852fn elevenlabs_output_format(format: &str) -> &'static str {
14853 match format.to_ascii_lowercase().as_str() {
14854 "mp3" => "mp3_44100_128",
14855 "pcm" => "pcm_16000",
14856 _ => "wav_44100",
14857 }
14858}
14859
14860fn quality_seeds(
14864 state_models_dir: &Path,
14865) -> std::collections::HashMap<String, outcome::QualitySeed> {
14866 let mut seeds = std::collections::HashMap::new();
14867 for path in benchmark_priors_paths(state_models_dir) {
14868 let Ok(priors) = routing_ext::load_benchmark_priors(&path) else {
14869 continue;
14870 };
14871 for (model_id, prior) in priors {
14872 seeds.insert(
14873 model_id,
14874 outcome::QualitySeed {
14875 overall: Some(prior.overall_score),
14876 tasks: prior.task_scores,
14877 },
14878 );
14879 }
14880 }
14881 seeds
14882}
14883
14884fn benchmark_priors_paths(state_models_dir: &Path) -> Vec<PathBuf> {
14889 let mut paths = Vec::new();
14890
14891 let direct = state_models_dir.join("benchmark_priors.json");
14892 if !paths.contains(&direct) {
14893 paths.push(direct);
14894 }
14895
14896 if let Some(parent) = state_models_dir.parent() {
14897 let parent_path = parent.join("benchmark_priors.json");
14898 if !paths.contains(&parent_path) {
14899 paths.push(parent_path);
14900 }
14901 }
14902
14903 if let Some(path) = std::env::var_os("CAR_BENCHMARK_PRIORS_PATH") {
14904 let path = PathBuf::from(path);
14905 if !paths.contains(&path) {
14906 paths.push(path);
14907 }
14908 }
14909
14910 paths
14911}
14912
14913fn load_benchmark_prior_health(
14914 state_models_dir: &Path,
14915 schemas: &[ModelSchema],
14916) -> Vec<ModelBenchmarkPriorHealth> {
14917 let mut priors = std::collections::BTreeMap::new();
14918 for path in benchmark_priors_paths(state_models_dir) {
14919 let Ok(loaded) = routing_ext::load_benchmark_priors(&path) else {
14920 continue;
14921 };
14922 for (model_id, prior) in loaded {
14923 let model_name = schemas
14924 .iter()
14925 .find(|schema| schema.id == model_id)
14926 .map(|schema| schema.name.clone());
14927 priors.insert(
14928 model_id.clone(),
14929 ModelBenchmarkPriorHealth {
14930 model_id,
14931 model_name,
14932 overall_score: prior.overall_score,
14933 overall_latency_ms: prior.overall_latency_ms,
14934 task_scores: prior.task_scores,
14935 task_latency_ms: prior.task_latency_ms,
14936 source_path: path.clone(),
14937 },
14938 );
14939 }
14940 }
14941
14942 priors.into_values().collect()
14943}
14944
14945fn kokoro_runtime_fallback_enabled() -> bool {
14946 std::env::var("CAR_SPEECH_KOKORO_FALLBACK")
14947 .ok()
14948 .map(|value| {
14949 !matches!(
14950 value.trim().to_ascii_lowercase().as_str(),
14951 "0" | "false" | "off"
14952 )
14953 })
14954 .unwrap_or(true)
14955}
14956
14957fn speech_runtime_mlx_audio_spec() -> String {
14958 std::env::var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC")
14959 .ok()
14960 .filter(|value| !value.trim().is_empty())
14961 .unwrap_or_else(|| "mlx-audio==0.4.2".to_string())
14962}
14963
14964fn speech_runtime_spacy_model_spec() -> String {
14965 std::env::var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC")
14966 .ok()
14967 .filter(|value| !value.trim().is_empty())
14968 .unwrap_or_else(|| {
14969 "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl".to_string()
14970 })
14971}
14972
14973#[cfg(test)]
14974pub(crate) fn run_in_isolated_test_process(test_name: &str, sentinel: &str) -> bool {
14975 static CHILD_PROCESS_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
14976
14977 if std::env::var_os(sentinel).is_some() {
14978 return true;
14979 }
14980
14981 let _child_process = CHILD_PROCESS_MUTEX
14982 .lock()
14983 .unwrap_or_else(std::sync::PoisonError::into_inner);
14984 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
14985 command
14986 .arg("--exact")
14987 .arg(test_name)
14988 .arg("--nocapture")
14989 .arg("--test-threads=1")
14990 .env(sentinel, "1");
14991 for name in [
14992 car_home::ENV_VAR,
14993 "CAR_SECRETS_FILE_DIR",
14994 "CAR_TEST_NATIVE_KEYCHAIN",
14995 "CAR_AUTH_LOCK_PATH",
14996 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
14997 car_auth::PARSLEE_API_BASE_KEY,
14998 crate::openrouter::API_KEY_ENV,
14999 "OPENAI_API_KEY",
15000 "ANTHROPIC_API_KEY",
15001 "GOOGLE_API_KEY",
15002 "ELEVENLABS_API_KEY",
15003 "SSL_CERT_FILE",
15004 "SSL_CERT_DIR",
15005 ] {
15006 command.env_remove(name);
15007 }
15008 const CHILD_BUDGET: std::time::Duration = std::time::Duration::from_secs(300);
15015 let mut stdout = tempfile::tempfile().expect("create isolated-test stdout capture");
15016 let mut stderr = tempfile::tempfile().expect("create isolated-test stderr capture");
15017 command
15018 .stdin(std::process::Stdio::null())
15019 .stdout(stdout.try_clone().expect("clone stdout capture"))
15020 .stderr(stderr.try_clone().expect("clone stderr capture"));
15021 #[cfg(unix)]
15022 {
15023 use std::os::unix::process::CommandExt;
15024 command.process_group(0);
15025 }
15026
15027 let mut child = command.spawn().expect("spawn isolated inference test");
15028 let deadline = std::time::Instant::now() + CHILD_BUDGET;
15029 let mut timed_out = false;
15030 let status = loop {
15031 match child.try_wait().expect("poll isolated inference test") {
15032 Some(status) => break status,
15033 None if std::time::Instant::now() < deadline => {
15034 std::thread::sleep(std::time::Duration::from_millis(50));
15035 }
15036 None => {
15037 timed_out = true;
15038 #[cfg(unix)]
15039 unsafe {
15040 libc::kill(-(child.id() as i32), libc::SIGKILL);
15041 }
15042 let _ = child.kill();
15043 break child
15044 .wait()
15045 .expect("reap timed-out isolated inference test");
15046 }
15047 }
15048 };
15049
15050 use std::io::{Read, Seek};
15051 stdout.rewind().expect("rewind isolated-test stdout");
15052 stderr.rewind().expect("rewind isolated-test stderr");
15053 let mut stdout_bytes = Vec::new();
15054 let mut stderr_bytes = Vec::new();
15055 stdout
15056 .read_to_end(&mut stdout_bytes)
15057 .expect("read isolated-test stdout");
15058 stderr
15059 .read_to_end(&mut stderr_bytes)
15060 .expect("read isolated-test stderr");
15061 assert!(
15062 !timed_out,
15063 "isolated {test_name} exceeded {}s and was killed\nstdout:\n{}\nstderr:\n{}",
15064 CHILD_BUDGET.as_secs(),
15065 String::from_utf8_lossy(&stdout_bytes),
15066 String::from_utf8_lossy(&stderr_bytes),
15067 );
15068 assert!(
15069 status.success(),
15070 "isolated {test_name} failed\nstdout:\n{}\nstderr:\n{}",
15071 String::from_utf8_lossy(&stdout_bytes),
15072 String::from_utf8_lossy(&stderr_bytes),
15073 );
15074 false
15075}
15076
15077fn pinned_by(agents: &[String]) -> String {
15079 let quoted: Vec<String> = agents.iter().map(|a| format!("\"{a}\"")).collect();
15080 match quoted.as_slice() {
15081 [one] => format!("pinned by the agent {one}"),
15082 _ => format!("pinned by the agents {}", quoted.join(", ")),
15083 }
15084}
15085
15086#[cfg(test)]
15087mod tests {
15088
15089 #[test]
15099 fn every_successful_generation_records_its_span() {
15100 let source = include_str!("lib.rs");
15101 let start = source
15102 .find("async fn generate_tracked_inner")
15103 .expect("the tracked-inference entry point");
15104 let open = source[start..].find('{').expect("function body") + start;
15105 let mut depth = 0usize;
15106 let mut end = open;
15107 for (offset, ch) in source[open..].char_indices() {
15108 match ch {
15109 '{' => depth += 1,
15110 '}' => {
15111 depth -= 1;
15112 if depth == 0 {
15113 end = open + offset;
15114 break;
15115 }
15116 }
15117 _ => {}
15118 }
15119 }
15120 let body = &source[start..end];
15121 let returns = body.matches("return Ok(InferenceResult {").count();
15122 let records = body.matches("record_generation_span(").count();
15123 assert!(
15124 returns >= 3,
15125 "control: the function still has several success returns, got {returns}"
15126 );
15127 assert_eq!(
15128 returns, records,
15129 "every successful return must record its generation span: \
15130 {returns} returns, {records} recorder calls"
15131 );
15132 }
15133 use super::*;
15134
15135 #[test]
15136 fn classify_defaults_to_jev_only_while_it_is_available() {
15137 let catalog: Vec<ModelSchema> =
15138 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
15139 let mut jev = catalog
15140 .iter()
15141 .find(|s| s.id == "parslee/jev")
15142 .unwrap()
15143 .clone();
15144 let local = catalog
15145 .iter()
15146 .find(|s| s.id == "mlx/qwen3-0.6b:6bit")
15147 .unwrap()
15148 .clone();
15149 jev.available = false;
15150 assert_eq!(
15151 default_system_one_classifier([&jev, &local].into_iter()),
15152 None,
15153 "signed out: keep the local default"
15154 );
15155 jev.available = true;
15156 assert_eq!(
15157 default_system_one_classifier([&local, &jev].into_iter()).as_deref(),
15158 Some("parslee/jev")
15159 );
15160 let chat = catalog
15162 .iter()
15163 .find(|s| s.id == "parslee/fast")
15164 .unwrap()
15165 .clone();
15166 assert_eq!(default_system_one_classifier([&chat].into_iter()), None);
15167 }
15168
15169 #[tokio::test]
15170 async fn generate_pinned_to_a_system_one_row_is_refused_streaming_included() {
15171 let dir = tempfile::tempdir().unwrap();
15172 let engine = InferenceEngine::new(test_config(dir.path().to_path_buf()));
15173 let req = || GenerateRequest {
15174 prompt: "hello".into(),
15175 model: Some("parslee/jev".into()),
15176 ..Default::default()
15177 };
15178 let refused = |e: &InferenceError| {
15179 matches!(
15180 e,
15181 InferenceError::UnsupportedMode {
15182 backend: "parslee-system-one",
15183 ..
15184 }
15185 )
15186 };
15187 match engine.generate_tracked(req()).await {
15188 Err(e) => assert!(refused(&e), "{e}"),
15189 Ok(_) => panic!("generate on a System One row must be refused"),
15190 }
15191 match engine.generate_tracked_stream(req()).await {
15192 Err(e) => assert!(refused(&e), "{e}"),
15193 Ok(_) => panic!("streaming generate on a System One row must be refused"),
15194 }
15195 }
15196 use std::ffi::OsString;
15197 use tempfile::TempDir;
15198
15199 #[test]
15206 fn local_models_keep_the_callers_default_output_budget() {
15207 use crate::tasks::generate::DEFAULT_MAX_TOKENS;
15208
15209 let catalog = crate::registry::builtin_catalog();
15210 let local = catalog
15211 .iter()
15212 .find(|s| s.id == "mlx/qwen3-8b:4bit")
15213 .expect("mlx/qwen3-8b:4bit is a builtin catalog entry");
15214
15215 assert!(local.is_local(), "mlx/* is decoded in-process");
15216 assert_eq!(
15217 local.effective_max_output(),
15218 32_768,
15219 "the budget that made the reported turn ~24 minutes long"
15220 );
15221 assert_eq!(
15222 resolved_max_tokens(DEFAULT_MAX_TOKENS, local),
15223 DEFAULT_MAX_TOKENS,
15224 "a local model must keep the budget the caller actually asked for"
15225 );
15226
15227 let mut remote = catalog
15230 .iter()
15231 .find(|s| !s.is_local())
15232 .expect("the builtin catalog ships remote models")
15233 .clone();
15234 remote.context_length = local.context_length;
15235 remote.max_output_tokens = local.max_output_tokens;
15236 assert!(!remote.is_local());
15237 assert_eq!(
15238 resolved_max_tokens(DEFAULT_MAX_TOKENS, &remote),
15239 32_768,
15240 "remote models still get their advertised output budget"
15241 );
15242
15243 assert_eq!(resolved_max_tokens(512, local), 512);
15245 assert_eq!(resolved_max_tokens(512, &remote), 512);
15246
15247 let codex = catalog
15248 .iter()
15249 .find(|s| s.id == "openai/gpt-5.6-sol:high")
15250 .expect("subscription-backed Codex row is builtin");
15251 assert_eq!(
15252 resolved_max_tokens(DEFAULT_MAX_TOKENS, codex),
15253 DEFAULT_MAX_TOKENS,
15254 "an approximate instruction must not widen the default to 128K tokens"
15255 );
15256
15257 if let Some(vllm) = catalog.iter().find(|s| s.is_vllm_mlx()) {
15263 assert!(vllm.is_local(), "vLLM-MLX runs on this machine");
15264 assert!(
15265 !vllm.decodes_in_process(),
15266 "but CAR does not decode it token by token"
15267 );
15268 assert_eq!(
15269 resolved_max_tokens(DEFAULT_MAX_TOKENS, vllm),
15270 vllm.effective_max_output(),
15271 );
15272 }
15273 }
15274
15275 #[test]
15278 fn a_ceiling_stop_fails_instead_of_retrying() {
15279 use EmptyPassAction::*;
15280 const CEILING: Option<&str> = Some(LOCAL_DECODE_TIMEOUT_STOP_REASON);
15281
15282 assert_eq!(
15285 classify_empty_pass(true, CEILING, "", true),
15286 FailDecodeCeiling
15287 );
15288 assert_eq!(
15289 classify_empty_pass(false, CEILING, "", true),
15290 FailDecodeCeiling
15291 );
15292 assert_eq!(
15294 classify_empty_pass(true, CEILING, " \n ", true),
15295 FailDecodeCeiling
15296 );
15297
15298 assert_eq!(
15301 classify_empty_pass(true, Some("length"), "", true),
15302 RetryWithoutThinking
15303 );
15304 assert_eq!(
15305 classify_empty_pass(true, None, "", true),
15306 RetryWithoutThinking
15307 );
15308
15309 assert_eq!(classify_empty_pass(false, Some("stop"), "", true), Accept);
15312
15313 assert_eq!(classify_empty_pass(true, CEILING, "partial", true), Accept);
15316 assert_eq!(classify_empty_pass(true, CEILING, "", false), Accept);
15317
15318 assert_ne!(LOCAL_DECODE_TIMEOUT_STOP_REASON, "timeout");
15322 assert_eq!(
15323 classify_empty_pass(true, Some("timeout"), "", true),
15324 RetryWithoutThinking
15325 );
15326 }
15327
15328 #[test]
15332 fn decode_deadline_and_heartbeat_predicates() {
15333 use std::time::Duration;
15334 let limit = Duration::from_secs(300);
15335
15336 assert!(!deadline_exceeded(Duration::from_secs(86_400), None));
15338
15339 assert!(!deadline_exceeded(Duration::from_secs(299), Some(limit)));
15340 assert!(deadline_exceeded(limit, Some(limit)));
15343 assert!(deadline_exceeded(Duration::from_secs(301), Some(limit)));
15344
15345 let every = Duration::from_secs(10);
15346 assert!(!heartbeat_due(
15348 Duration::from_secs(9),
15349 Duration::ZERO,
15350 every
15351 ));
15352 assert!(heartbeat_due(
15353 Duration::from_secs(10),
15354 Duration::ZERO,
15355 every
15356 ));
15357 assert!(!heartbeat_due(
15359 Duration::from_secs(11),
15360 Duration::from_secs(10),
15361 every
15362 ));
15363 assert!(heartbeat_due(
15364 Duration::from_secs(20),
15365 Duration::from_secs(10),
15366 every
15367 ));
15368 assert!(!heartbeat_due(
15370 Duration::from_secs(5),
15371 Duration::from_secs(10),
15372 every
15373 ));
15374 }
15375
15376 #[test]
15377 fn decode_timeout_parses_with_a_safe_fallback() {
15378 let default = std::time::Duration::from_secs(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS);
15379 assert_eq!(parse_decode_timeout(None), Some(default));
15380 assert_eq!(
15381 parse_decode_timeout(Some(" 45 ")),
15382 Some(std::time::Duration::from_secs(45))
15383 );
15384 assert_eq!(
15385 parse_decode_timeout(Some("0")),
15386 None,
15387 "0 disables the ceiling"
15388 );
15389 assert_eq!(
15390 parse_decode_timeout(Some("banana")),
15391 Some(default),
15392 "garbage must fall back to the default, never silently disable it"
15393 );
15394 }
15395
15396 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15401 #[test]
15402 fn local_decode_stops_at_the_wall_clock_ceiling() {
15403 struct NeverStops;
15406
15407 impl crate::backend::local::TextDecoder for NeverStops {
15408 fn encode(&self, _text: &str) -> Result<Vec<u32>, InferenceError> {
15409 Ok(vec![1, 2, 3])
15410 }
15411 fn decode(&self, tokens: &[u32]) -> Result<String, InferenceError> {
15412 Ok("x".repeat(tokens.len()))
15413 }
15414 fn forward(
15415 &mut self,
15416 _tokens: &[u32],
15417 _pos: usize,
15418 ) -> Result<Vec<f32>, InferenceError> {
15419 std::thread::sleep(std::time::Duration::from_millis(5));
15420 let mut logits = vec![0.0f32; 16];
15421 logits[7] = 10.0;
15422 Ok(logits)
15423 }
15424 fn eos_ids(&self) -> Vec<u32> {
15425 vec![0]
15426 }
15427 fn context_length(&self) -> usize {
15428 4096
15429 }
15430 fn clear_kv_cache(&mut self) {}
15431 }
15432
15433 let params = GenerateParams {
15436 max_tokens: 100_000,
15437 temperature: 0.0,
15438 ..Default::default()
15439 };
15440 let ceiling = std::time::Duration::from_millis(300);
15441
15442 let mut backend = NeverStops;
15443 let started = std::time::Instant::now();
15444 let generated = match InferenceEngine::drive_generation_with_timeout(
15445 &mut backend,
15446 "anything",
15447 ¶ms,
15448 Some(ceiling),
15449 ) {
15450 Ok(generated) => generated,
15451 Err(e) => panic!(
15453 "a decode cut short by the ceiling returns its partial text, not an error: {}",
15454 e.into_inner()
15455 ),
15456 };
15457 let elapsed = started.elapsed();
15458
15459 assert_eq!(
15460 generated.stop_reason.as_deref(),
15461 Some(LOCAL_DECODE_TIMEOUT_STOP_REASON),
15462 "the caller must be able to tell a deadline stop from a clean finish"
15463 );
15464 assert!(
15465 InferenceResult {
15466 text: generated.text.clone(),
15467 bounding_boxes: Vec::new(),
15468 tool_calls: vec![],
15469 trace_id: String::new(),
15470 model_used: String::new(),
15471 model_identity: Default::default(),
15472 latency_ms: 0,
15473 time_to_first_token_ms: None,
15474 usage: None,
15475 provider_output_items: Vec::new(),
15476 thinking: Vec::new(),
15477 stop_reason: generated.stop_reason.clone(),
15478 auth_fallback_from: None,
15479 local_last_resort: false,
15480 fallback_from: Vec::new(),
15481 }
15482 .was_truncated(),
15483 "a ceiling stop is a cut-short answer, not a complete one"
15484 );
15485 assert!(
15486 generated.completion_tokens > 0,
15487 "the partial response is kept, not discarded"
15488 );
15489 assert!(
15490 generated.completion_tokens < params.max_tokens,
15491 "the loop stopped on the clock, not by exhausting the budget"
15492 );
15493 assert!(
15494 elapsed < std::time::Duration::from_secs(30),
15495 "bounded in wall clock; took {elapsed:?}"
15496 );
15497 }
15498
15499 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
15500
15501 impl RestoredEnvironment {
15502 fn capture(names: &[&'static str]) -> Self {
15503 Self(
15504 names
15505 .iter()
15506 .map(|name| (*name, std::env::var_os(name)))
15507 .collect(),
15508 )
15509 }
15510 }
15511
15512 impl Drop for RestoredEnvironment {
15513 fn drop(&mut self) {
15514 for (name, value) in &self.0 {
15515 unsafe {
15516 match value {
15517 Some(value) => std::env::set_var(name, value),
15518 None => std::env::remove_var(name),
15519 }
15520 }
15521 }
15522 }
15523 }
15524
15525 struct FixtureLocalOffload {
15526 emit_done: bool,
15527 }
15528
15529 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15534 struct ExactPinCaptureOffload {
15535 dispatched_models: std::sync::Mutex<Vec<String>>,
15536 }
15537
15538 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15539 impl ExactPinCaptureOffload {
15540 fn new() -> Self {
15541 Self {
15542 dispatched_models: std::sync::Mutex::new(Vec::new()),
15543 }
15544 }
15545
15546 fn dispatched_models(&self) -> Vec<String> {
15547 self.dispatched_models.lock().unwrap().clone()
15548 }
15549 }
15550
15551 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15552 #[async_trait::async_trait]
15553 impl crate::offload::LocalGenerationOffload for ExactPinCaptureOffload {
15554 async fn generate(
15555 &self,
15556 request: GenerateRequest,
15557 ) -> Result<InferenceResult, InferenceError> {
15558 let model_id = request.model.expect("resolved worker model");
15559 self.dispatched_models
15560 .lock()
15561 .unwrap()
15562 .push(model_id.clone());
15563 Ok(InferenceResult {
15564 text: "exact pin".into(),
15565 tool_calls: vec![],
15566 bounding_boxes: vec![],
15567 trace_id: "worker-trace".into(),
15568 model_used: model_id,
15569 model_identity: Default::default(),
15570 latency_ms: 0,
15571 time_to_first_token_ms: None,
15572 usage: None,
15573 provider_output_items: vec![],
15574 thinking: vec![],
15575 stop_reason: Some("stop".into()),
15576 auth_fallback_from: None,
15577 local_last_resort: false,
15578 fallback_from: Vec::new(),
15579 })
15580 }
15581
15582 async fn stream(
15583 &self,
15584 request: GenerateRequest,
15585 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15586 let model_id = request.model.expect("resolved worker model");
15587 self.dispatched_models.lock().unwrap().push(model_id);
15588 let (tx, rx) = tokio::sync::mpsc::channel(2);
15589 tx.send(StreamEvent::Done {
15590 text: "exact pin".into(),
15591 tool_calls: vec![],
15592 })
15593 .await
15594 .unwrap();
15595 Ok(rx)
15596 }
15597 }
15598
15599 struct ReleaseRetryOffload {
15600 calls: std::sync::Mutex<Vec<String>>,
15601 model_id: String,
15602 resident: bool,
15603 release_acknowledged: bool,
15604 }
15605
15606 struct SequencedRetryProbe {
15607 calls: Arc<std::sync::atomic::AtomicUsize>,
15608 first_available_mb: u64,
15609 subsequent_available_mb: u64,
15610 }
15611
15612 impl crate::resource_policy::LiveMemoryProbe for SequencedRetryProbe {
15613 fn available_memory_mb(
15614 &self,
15615 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
15616 let call = self
15617 .calls
15618 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
15619 Ok(Some(if call == 0 {
15620 self.first_available_mb
15621 } else {
15622 self.subsequent_available_mb
15623 }))
15624 }
15625 }
15626
15627 #[async_trait::async_trait]
15628 impl crate::offload::LocalGenerationOffload for ReleaseRetryOffload {
15629 async fn generate(
15630 &self,
15631 _request: GenerateRequest,
15632 ) -> Result<InferenceResult, InferenceError> {
15633 unreachable!("release-and-retry fixture")
15634 }
15635
15636 async fn stream(
15637 &self,
15638 _request: GenerateRequest,
15639 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15640 unreachable!("release-and-retry fixture")
15641 }
15642
15643 async fn resident_models(&self) -> Vec<String> {
15644 self.resident
15645 .then(|| self.model_id.clone())
15646 .into_iter()
15647 .collect()
15648 }
15649
15650 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
15651 Some(format!("worker:retry-fixture:{model_id}"))
15652 }
15653
15654 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15655 self.calls
15656 .lock()
15657 .unwrap()
15658 .push(format!("release:{model_id}"));
15659 Ok(self.release_acknowledged)
15660 }
15661 }
15662
15663 struct RetiringReleaseOffload {
15664 calls: std::sync::Mutex<Vec<String>>,
15665 model_id: String,
15666 allocation_id: String,
15667 resident: std::sync::atomic::AtomicBool,
15668 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
15669 }
15670
15671 #[async_trait::async_trait]
15672 impl crate::offload::LocalGenerationOffload for RetiringReleaseOffload {
15673 async fn generate(
15674 &self,
15675 _request: GenerateRequest,
15676 ) -> Result<InferenceResult, InferenceError> {
15677 unreachable!("retiring-release fixture")
15678 }
15679
15680 async fn stream(
15681 &self,
15682 _request: GenerateRequest,
15683 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15684 unreachable!("retiring-release fixture")
15685 }
15686
15687 async fn resident_models(&self) -> Vec<String> {
15688 self.resident
15689 .load(std::sync::atomic::Ordering::Acquire)
15690 .then(|| self.model_id.clone())
15691 .into_iter()
15692 .collect()
15693 }
15694
15695 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15696 Some(self.allocation_id.clone())
15697 }
15698
15699 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15700 self.calls
15701 .lock()
15702 .unwrap()
15703 .push(format!("release:{model_id}"));
15704 self.coordinator
15705 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
15706 self.coordinator
15707 .finish_teardown_allocation(model_id, &self.allocation_id);
15708 self.resident
15709 .store(false, std::sync::atomic::Ordering::Release);
15710 Ok(true)
15711 }
15712 }
15713
15714 #[tokio::test]
15715 async fn release_and_retry_releases_resident_worker_after_live_memory_refusal() {
15716 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15717 let root = tempfile::tempdir().unwrap();
15718 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15719 let schema = crate::registry::builtin_catalog()
15720 .into_iter()
15721 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15722 .unwrap();
15723 let allocation_id = format!("worker:retry-fixture:{}", schema.id);
15724 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15725 let coordinator = Arc::new(
15726 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15727 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15728 metal_mac_for_fit(32),
15729 Arc::new(SequencedRetryProbe {
15730 calls: probe_calls.clone(),
15731 first_available_mb: 1_000,
15732 subsequent_available_mb: 1_048_576,
15733 }),
15734 ),
15735 );
15736 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
15737 engine.local_admission = coordinator.clone();
15738 let offload = Arc::new(RetiringReleaseOffload {
15739 calls: std::sync::Mutex::new(Vec::new()),
15740 model_id: schema.id.clone(),
15741 allocation_id,
15742 resident: std::sync::atomic::AtomicBool::new(true),
15743 coordinator,
15744 });
15745 crate::offload::set_local_offload(Some(offload.clone()));
15746
15747 let result = engine
15748 .reserve_local_request_with_worker_retry(&schema, 9_000)
15749 .await;
15750 crate::offload::set_local_offload(None);
15751
15752 assert!(
15753 result.is_ok(),
15754 "release followed by a second probe should admit"
15755 );
15756 assert_eq!(
15757 *offload.calls.lock().unwrap(),
15758 vec![format!("release:{}", schema.id)]
15759 );
15760 assert_eq!(
15761 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
15762 2,
15763 "admission must probe exactly once before and once after release"
15764 );
15765 }
15766
15767 #[tokio::test]
15768 async fn release_and_retry_ignores_sibling_allocation_after_exact_worker_retirement() {
15769 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15770 let root = tempfile::tempdir().unwrap();
15771 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15772 let schema = crate::registry::builtin_catalog()
15773 .into_iter()
15774 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15775 .unwrap();
15776 let worker_allocation_id = format!("worker:sibling-test:{}", schema.id);
15777 let sibling_allocation_id = format!("cache:sibling-test:{}", schema.id);
15778 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15779 let coordinator = Arc::new(
15780 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15781 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15782 metal_mac_for_fit(32),
15783 Arc::new(SequencedRetryProbe {
15784 calls: probe_calls.clone(),
15785 first_available_mb: 1_000,
15786 subsequent_available_mb: 1_048_576,
15787 }),
15788 ),
15789 );
15790 coordinator.mark_resident_allocation(&schema.id, &worker_allocation_id, 4_800);
15791 coordinator.mark_resident_allocation(&schema.id, &sibling_allocation_id, 500);
15792 engine.local_admission = coordinator.clone();
15793 let offload = Arc::new(RetiringReleaseOffload {
15794 calls: std::sync::Mutex::new(Vec::new()),
15795 model_id: schema.id.clone(),
15796 allocation_id: worker_allocation_id.clone(),
15797 resident: std::sync::atomic::AtomicBool::new(true),
15798 coordinator: coordinator.clone(),
15799 });
15800 crate::offload::set_local_offload(Some(offload.clone()));
15801
15802 let result = engine
15803 .reserve_local_request_with_worker_retry(&schema, 9_000)
15804 .await;
15805 crate::offload::set_local_offload(None);
15806
15807 assert!(
15808 result.is_ok(),
15809 "an unrelated resident allocation must not block the worker retry"
15810 );
15811 assert_eq!(
15812 coordinator.resident_allocation_ids(&schema.id),
15813 vec![sibling_allocation_id]
15814 );
15815 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 2);
15816 assert_eq!(
15817 *offload.calls.lock().unwrap(),
15818 vec![format!("release:{}", schema.id)]
15819 );
15820 }
15821
15822 struct UnretiredReleaseOffload {
15823 calls: std::sync::Mutex<Vec<String>>,
15824 model_id: String,
15825 allocation_id: String,
15826 resident: std::sync::atomic::AtomicBool,
15827 }
15828
15829 #[async_trait::async_trait]
15830 impl crate::offload::LocalGenerationOffload for UnretiredReleaseOffload {
15831 async fn generate(
15832 &self,
15833 _request: GenerateRequest,
15834 ) -> Result<InferenceResult, InferenceError> {
15835 unreachable!("unretired-release fixture")
15836 }
15837
15838 async fn stream(
15839 &self,
15840 _request: GenerateRequest,
15841 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15842 unreachable!("unretired-release fixture")
15843 }
15844
15845 async fn resident_models(&self) -> Vec<String> {
15846 self.resident
15847 .load(std::sync::atomic::Ordering::Acquire)
15848 .then(|| self.model_id.clone())
15849 .into_iter()
15850 .collect()
15851 }
15852
15853 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15854 Some(self.allocation_id.clone())
15855 }
15856
15857 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15858 self.calls
15859 .lock()
15860 .unwrap()
15861 .push(format!("release:{model_id}"));
15862 self.resident
15863 .store(false, std::sync::atomic::Ordering::Release);
15864 Ok(true)
15865 }
15866 }
15867
15868 #[tokio::test]
15869 async fn release_and_retry_rejects_ack_without_accounting_retirement() {
15870 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15871 let warnings_before =
15872 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
15873 let root = tempfile::tempdir().unwrap();
15874 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15875 let schema = crate::registry::builtin_catalog()
15876 .into_iter()
15877 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15878 .unwrap();
15879 let allocation_id = format!("worker:phantom:{}", schema.id);
15880 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15881 let coordinator = Arc::new(
15882 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15883 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15884 metal_mac_for_fit(32),
15885 Arc::new(SequencedRetryProbe {
15886 calls: probe_calls.clone(),
15887 first_available_mb: 1_000,
15888 subsequent_available_mb: 8_000,
15889 }),
15890 ),
15891 );
15892 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
15893 engine.local_admission = coordinator;
15894 let offload = Arc::new(UnretiredReleaseOffload {
15895 calls: std::sync::Mutex::new(Vec::new()),
15896 model_id: schema.id.clone(),
15897 allocation_id,
15898 resident: std::sync::atomic::AtomicBool::new(true),
15899 });
15900 crate::offload::set_local_offload(Some(offload.clone()));
15901
15902 let result = engine
15903 .reserve_local_request_with_worker_retry(&schema, 9_000)
15904 .await;
15905 crate::offload::set_local_offload(None);
15906
15907 assert!(matches!(
15908 result,
15909 Err(InferenceError::LocalResourceBlocked { preflight, .. })
15910 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
15911 ));
15912 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
15913 assert_eq!(
15914 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
15915 warnings_before + 1,
15916 "the refused retry must emit the stale-residency warning"
15917 );
15918 assert!(engine.local_admission.is_resident(&schema.id));
15919 assert!(offload.resident_models().await.is_empty());
15920 assert_eq!(
15921 *offload.calls.lock().unwrap(),
15922 vec![format!("release:{}", schema.id)]
15923 );
15924 }
15925
15926 struct PendingTeardownAckOffload {
15927 calls: std::sync::Mutex<Vec<String>>,
15928 model_id: String,
15929 allocation_id: String,
15930 resident: std::sync::atomic::AtomicBool,
15931 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
15932 }
15933
15934 #[async_trait::async_trait]
15935 impl crate::offload::LocalGenerationOffload for PendingTeardownAckOffload {
15936 async fn generate(
15937 &self,
15938 _request: GenerateRequest,
15939 ) -> Result<InferenceResult, InferenceError> {
15940 unreachable!("pending-teardown ACK fixture")
15941 }
15942
15943 async fn stream(
15944 &self,
15945 _request: GenerateRequest,
15946 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15947 unreachable!("pending-teardown ACK fixture")
15948 }
15949
15950 async fn resident_models(&self) -> Vec<String> {
15951 self.resident
15952 .load(std::sync::atomic::Ordering::Acquire)
15953 .then(|| self.model_id.clone())
15954 .into_iter()
15955 .collect()
15956 }
15957
15958 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15959 Some(self.allocation_id.clone())
15960 }
15961
15962 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15963 self.calls
15964 .lock()
15965 .unwrap()
15966 .push(format!("release:{model_id}"));
15967 self.coordinator
15968 .mark_teardown_pending_allocation_with_charge(
15969 model_id,
15970 &self.allocation_id,
15971 4_800 * 1024 * 1024,
15972 );
15973 self.resident
15974 .store(false, std::sync::atomic::Ordering::Release);
15975 Ok(true)
15976 }
15977 }
15978
15979 #[tokio::test]
15980 async fn release_and_retry_rejects_ack_with_exact_worker_pending_teardown() {
15981 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15982 let warnings_before =
15983 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
15984 let root = tempfile::tempdir().unwrap();
15985 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15986 let schema = crate::registry::builtin_catalog()
15987 .into_iter()
15988 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15989 .unwrap();
15990 let allocation_id = format!("worker:pending-test:{}", schema.id);
15991 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15992 let coordinator = Arc::new(
15993 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15994 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15995 metal_mac_for_fit(32),
15996 Arc::new(SequencedRetryProbe {
15997 calls: probe_calls.clone(),
15998 first_available_mb: 1_000,
15999 subsequent_available_mb: 8_000,
16000 }),
16001 ),
16002 );
16003 engine.local_admission = coordinator.clone();
16004 let offload = Arc::new(PendingTeardownAckOffload {
16005 calls: std::sync::Mutex::new(Vec::new()),
16006 model_id: schema.id.clone(),
16007 allocation_id: allocation_id.clone(),
16008 resident: std::sync::atomic::AtomicBool::new(true),
16009 coordinator: coordinator.clone(),
16010 });
16011 crate::offload::set_local_offload(Some(offload.clone()));
16012
16013 let result = engine
16014 .reserve_local_request_with_worker_retry(&schema, 9_000)
16015 .await;
16016 crate::offload::set_local_offload(None);
16017
16018 assert!(matches!(
16019 result,
16020 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16021 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16022 ));
16023 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16024 assert_eq!(
16025 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
16026 warnings_before + 1,
16027 "the pending exact allocation must emit the fail-closed warning"
16028 );
16029 assert_eq!(
16030 coordinator.resident_allocation_ids(&schema.id),
16031 vec![allocation_id]
16032 );
16033 assert!(offload.resident_models().await.is_empty());
16034 assert_eq!(
16035 *offload.calls.lock().unwrap(),
16036 vec![format!("release:{}", schema.id)]
16037 );
16038 }
16039
16040 struct MissingAllocationIdOffload {
16041 calls: std::sync::Mutex<Vec<String>>,
16042 model_id: String,
16043 }
16044
16045 #[async_trait::async_trait]
16046 impl crate::offload::LocalGenerationOffload for MissingAllocationIdOffload {
16047 async fn generate(
16048 &self,
16049 _request: GenerateRequest,
16050 ) -> Result<InferenceResult, InferenceError> {
16051 unreachable!("missing allocation-id fixture")
16052 }
16053
16054 async fn stream(
16055 &self,
16056 _request: GenerateRequest,
16057 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16058 unreachable!("missing allocation-id fixture")
16059 }
16060
16061 async fn resident_models(&self) -> Vec<String> {
16062 vec![self.model_id.clone()]
16063 }
16064
16065 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16066 self.calls
16067 .lock()
16068 .unwrap()
16069 .push(format!("release:{model_id}"));
16070 Ok(true)
16071 }
16072 }
16073
16074 #[tokio::test]
16075 async fn release_and_retry_without_scoped_allocation_id_fails_closed() {
16076 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16077 let root = tempfile::tempdir().unwrap();
16078 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16079 let schema = crate::registry::builtin_catalog()
16080 .into_iter()
16081 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16082 .unwrap();
16083 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16084 engine.local_admission = Arc::new(
16085 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16086 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16087 metal_mac_for_fit(32),
16088 Arc::new(SequencedRetryProbe {
16089 calls: probe_calls.clone(),
16090 first_available_mb: 1_000,
16091 subsequent_available_mb: 8_000,
16092 }),
16093 ),
16094 );
16095 let offload = Arc::new(MissingAllocationIdOffload {
16096 calls: std::sync::Mutex::new(Vec::new()),
16097 model_id: schema.id.clone(),
16098 });
16099 crate::offload::set_local_offload(Some(offload.clone()));
16100
16101 let result = engine
16102 .reserve_local_request_with_worker_retry(&schema, 9_000)
16103 .await;
16104 crate::offload::set_local_offload(None);
16105
16106 assert!(matches!(
16107 result,
16108 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16109 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16110 ));
16111 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16112 assert!(offload.calls.lock().unwrap().is_empty());
16113 }
16114
16115 struct ReplacingResidentOffload {
16116 calls: std::sync::Mutex<Vec<String>>,
16117 model_id: String,
16118 allocation_id: String,
16119 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16120 }
16121
16122 #[async_trait::async_trait]
16123 impl crate::offload::LocalGenerationOffload for ReplacingResidentOffload {
16124 async fn generate(
16125 &self,
16126 _request: GenerateRequest,
16127 ) -> Result<InferenceResult, InferenceError> {
16128 unreachable!("replacement-resident fixture")
16129 }
16130
16131 async fn stream(
16132 &self,
16133 _request: GenerateRequest,
16134 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16135 unreachable!("replacement-resident fixture")
16136 }
16137
16138 async fn resident_models(&self) -> Vec<String> {
16139 vec![self.model_id.clone()]
16140 }
16141
16142 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16143 Some(self.allocation_id.clone())
16144 }
16145
16146 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16147 self.calls
16148 .lock()
16149 .unwrap()
16150 .push(format!("release:{model_id}"));
16151 self.coordinator
16155 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
16156 self.coordinator
16157 .finish_teardown_allocation(model_id, &self.allocation_id);
16158 self.coordinator
16159 .mark_resident_allocation(model_id, &self.allocation_id, 4_800);
16160 Ok(true)
16161 }
16162 }
16163
16164 #[tokio::test]
16165 async fn release_and_retry_same_id_replacement_blocks_retry_without_erasing_accounting() {
16166 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16167 let root = tempfile::tempdir().unwrap();
16168 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16169 let schema = crate::registry::builtin_catalog()
16170 .into_iter()
16171 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16172 .unwrap();
16173 let allocation_id = format!("worker:reused-scope:{}", schema.id);
16174 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16175 let coordinator = Arc::new(
16176 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16177 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16178 metal_mac_for_fit(32),
16179 Arc::new(SequencedRetryProbe {
16180 calls: probe_calls.clone(),
16181 first_available_mb: 1_000,
16182 subsequent_available_mb: 8_000,
16183 }),
16184 ),
16185 );
16186 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
16187 engine.local_admission = coordinator.clone();
16188 let offload = Arc::new(ReplacingResidentOffload {
16189 calls: std::sync::Mutex::new(Vec::new()),
16190 model_id: schema.id.clone(),
16191 allocation_id,
16192 coordinator,
16193 });
16194 crate::offload::set_local_offload(Some(offload.clone()));
16195
16196 let result = engine
16197 .reserve_local_request_with_worker_retry(&schema, 9_000)
16198 .await;
16199 crate::offload::set_local_offload(None);
16200
16201 assert!(matches!(
16202 result,
16203 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16204 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16205 ));
16206 assert!(engine.local_admission.is_resident(&schema.id));
16207 assert_eq!(
16208 *offload.calls.lock().unwrap(),
16209 vec![format!("release:{}", schema.id)]
16210 );
16211 assert_eq!(
16212 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
16213 1,
16214 "replacement residency must stop the retry before a second probe"
16215 );
16216 }
16217
16218 #[tokio::test]
16219 async fn release_and_retry_nonresident_worker_returns_original_refusal() {
16220 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16221 let root = tempfile::tempdir().unwrap();
16222 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16223 let schema = crate::registry::builtin_catalog()
16224 .into_iter()
16225 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16226 .unwrap();
16227 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16228 engine.local_admission = Arc::new(
16229 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16230 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16231 metal_mac_for_fit(32),
16232 Arc::new(SequencedRetryProbe {
16233 calls: probe_calls.clone(),
16234 first_available_mb: 1_000,
16235 subsequent_available_mb: 1_048_576,
16236 }),
16237 ),
16238 );
16239 let offload = Arc::new(ReleaseRetryOffload {
16240 calls: std::sync::Mutex::new(Vec::new()),
16241 model_id: schema.id.clone(),
16242 resident: false,
16243 release_acknowledged: true,
16244 });
16245 crate::offload::set_local_offload(Some(offload.clone()));
16246
16247 let result = engine
16248 .reserve_local_request_with_worker_retry(&schema, 9_000)
16249 .await;
16250 crate::offload::set_local_offload(None);
16251
16252 assert!(matches!(
16253 result,
16254 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16255 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16256 ));
16257 assert!(offload.calls.lock().unwrap().is_empty());
16258 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16259 }
16260
16261 #[tokio::test]
16262 async fn release_and_retry_unacknowledged_release_does_not_reprobe() {
16263 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16264 let root = tempfile::tempdir().unwrap();
16265 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16266 let schema = crate::registry::builtin_catalog()
16267 .into_iter()
16268 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16269 .unwrap();
16270 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16271 engine.local_admission = Arc::new(
16272 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16273 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16274 metal_mac_for_fit(32),
16275 Arc::new(SequencedRetryProbe {
16276 calls: probe_calls.clone(),
16277 first_available_mb: 1_000,
16278 subsequent_available_mb: 1_048_576,
16279 }),
16280 ),
16281 );
16282 let offload = Arc::new(ReleaseRetryOffload {
16283 calls: std::sync::Mutex::new(Vec::new()),
16284 model_id: schema.id.clone(),
16285 resident: true,
16286 release_acknowledged: false,
16287 });
16288 crate::offload::set_local_offload(Some(offload.clone()));
16289
16290 let result = engine
16291 .reserve_local_request_with_worker_retry(&schema, 9_000)
16292 .await;
16293 crate::offload::set_local_offload(None);
16294
16295 assert!(matches!(
16296 result,
16297 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16298 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16299 ));
16300 assert_eq!(
16301 *offload.calls.lock().unwrap(),
16302 vec![format!("release:{}", schema.id)]
16303 );
16304 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16305 }
16306
16307 #[tokio::test]
16308 async fn release_and_retry_allowed_first_verdict_never_releases() {
16309 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16310 let root = tempfile::tempdir().unwrap();
16311 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16312 let schema = crate::registry::builtin_catalog()
16313 .into_iter()
16314 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16315 .unwrap();
16316 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16317 engine.local_admission = Arc::new(
16318 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16319 crate::resource_policy::ResourcePolicy::local_focused(),
16320 metal_mac_for_fit(64),
16321 Arc::new(SequencedRetryProbe {
16322 calls: probe_calls.clone(),
16323 first_available_mb: 1_048_576,
16324 subsequent_available_mb: 1_048_576,
16325 }),
16326 ),
16327 );
16328 let offload = Arc::new(ReleaseRetryOffload {
16329 calls: std::sync::Mutex::new(Vec::new()),
16330 model_id: schema.id.clone(),
16331 resident: true,
16332 release_acknowledged: true,
16333 });
16334 crate::offload::set_local_offload(Some(offload.clone()));
16335
16336 let result = engine
16337 .reserve_local_request_with_worker_retry(&schema, 9_000)
16338 .await;
16339 crate::offload::set_local_offload(None);
16340
16341 assert!(result.is_ok());
16342 assert!(offload.calls.lock().unwrap().is_empty());
16343 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16344 }
16345
16346 struct ReconcileOffload {
16347 calls: std::sync::Mutex<Vec<String>>,
16348 release_acknowledged: bool,
16349 }
16350
16351 #[async_trait::async_trait]
16352 impl crate::offload::LocalGenerationOffload for ReconcileOffload {
16353 async fn generate(
16354 &self,
16355 _request: GenerateRequest,
16356 ) -> Result<InferenceResult, InferenceError> {
16357 unreachable!("residency reconciliation fixture")
16358 }
16359
16360 async fn stream(
16361 &self,
16362 _request: GenerateRequest,
16363 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16364 unreachable!("residency reconciliation fixture")
16365 }
16366
16367 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
16368 self.calls
16369 .lock()
16370 .unwrap()
16371 .push(format!("allocation:{model_id}"));
16372 Some(format!("worker:{model_id}:generation-7"))
16373 }
16374
16375 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16376 self.calls
16377 .lock()
16378 .unwrap()
16379 .push(format!("release:{model_id}"));
16380 Ok(self.release_acknowledged)
16381 }
16382 }
16383
16384 fn reconciliation_reservation() -> (
16385 Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16386 crate::resource_policy::LocalLoadReservation,
16387 ) {
16388 struct FixedProbe;
16389 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
16390 fn available_memory_mb(
16391 &self,
16392 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
16393 Ok(Some(24_000))
16394 }
16395 }
16396 let coordinator = Arc::new(
16397 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16398 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16399 crate::hardware::HardwareInfo {
16400 total_ram_mb: 32 * 1024,
16401 ..crate::hardware::HardwareInfo::detect()
16402 },
16403 Arc::new(FixedProbe),
16404 ),
16405 );
16406 let reservation = coordinator
16407 .reserve_measured_host_allocation(
16408 "expected/model",
16409 "expected/model#request",
16410 1024 * 1024,
16411 0,
16412 )
16413 .unwrap();
16414 (coordinator, reservation)
16415 }
16416
16417 #[tokio::test]
16418 async fn worker_residency_reconciliation_releases_mismatched_ack_before_returning_error() {
16419 let offload = ReconcileOffload {
16420 calls: std::sync::Mutex::new(Vec::new()),
16421 release_acknowledged: true,
16422 };
16423 let (_coordinator, mut reservation) = reconciliation_reservation();
16424 let error = InferenceEngine::reconcile_worker_residency(
16425 &offload,
16426 "expected/model",
16427 &crate::offload::LocalWorkerResidency {
16428 model_id: "wrong/model".into(),
16429 measured_weights_bytes: 2 * 1024 * 1024,
16430 },
16431 backend_cache::BackendRetention::Resident,
16432 &mut reservation,
16433 )
16434 .await
16435 .unwrap_err();
16436 assert!(error.to_string().contains("wrong/model"));
16437 assert_eq!(
16438 *offload.calls.lock().unwrap(),
16439 vec!["release:wrong/model".to_string()]
16440 );
16441 }
16442
16443 #[tokio::test]
16444 async fn worker_residency_reconciliation_publishes_exact_owner_for_matching_ack() {
16445 let offload = ReconcileOffload {
16446 calls: std::sync::Mutex::new(Vec::new()),
16447 release_acknowledged: false,
16448 };
16449 let (coordinator, mut reservation) = reconciliation_reservation();
16450 InferenceEngine::reconcile_worker_residency(
16451 &offload,
16452 "expected/model",
16453 &crate::offload::LocalWorkerResidency {
16454 model_id: "expected/model".into(),
16455 measured_weights_bytes: 2 * 1024 * 1024,
16456 },
16457 backend_cache::BackendRetention::Resident,
16458 &mut reservation,
16459 )
16460 .await
16461 .unwrap();
16462 drop(reservation);
16463 assert_eq!(
16464 *offload.calls.lock().unwrap(),
16465 vec!["allocation:expected/model".to_string()]
16466 );
16467 assert!(coordinator.is_resident("expected/model"));
16468 }
16469
16470 #[async_trait::async_trait]
16471 impl crate::offload::LocalGenerationOffload for FixtureLocalOffload {
16472 async fn generate(
16473 &self,
16474 _request: GenerateRequest,
16475 ) -> Result<InferenceResult, InferenceError> {
16476 unreachable!("streaming fixture")
16477 }
16478
16479 async fn stream(
16480 &self,
16481 _request: GenerateRequest,
16482 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16483 unreachable!("admission-aware streaming fixture")
16484 }
16485
16486 async fn stream_admitted(
16487 &self,
16488 request: GenerateRequest,
16489 admission: crate::offload::LocalWorkerAdmission,
16490 ) -> Result<crate::offload::LocalOffloadStream, InferenceError> {
16491 let (tx, rx) = tokio::sync::mpsc::channel(4);
16492 let emit_done = self.emit_done;
16493 tokio::spawn(async move {
16494 let _ = tx.send(StreamEvent::TextDelta("local answer".into())).await;
16495 if emit_done {
16496 let _ = tx
16497 .send(StreamEvent::Done {
16498 text: "local answer".into(),
16499 tool_calls: vec![],
16500 })
16501 .await;
16502 }
16503 });
16504 Ok(crate::offload::LocalOffloadStream {
16505 events: rx,
16506 residency: crate::offload::LocalWorkerResidency {
16507 model_id: request.model.unwrap_or_else(|| "fixture/local".into()),
16508 measured_weights_bytes: admission.measured_weights_bytes,
16509 },
16510 retention: backend_cache::BackendRetention::Resident,
16511 })
16512 }
16513 }
16514
16515 fn pin_test_local_admission_memory(engine: &mut InferenceEngine) {
16523 let policy = engine.local_admission.policy();
16524 let mut hardware = HardwareInfo::detect();
16525 hardware.total_ram_mb = hardware.total_ram_mb.max(16 * 1024);
16526 hardware.max_model_mb = hardware.max_model_mb.max(16 * 1024);
16527 engine.local_admission = Arc::new(resource_policy::LocalAdmissionCoordinator::with_probe(
16528 policy,
16529 hardware,
16530 Arc::new(resource_policy::FixedLiveMemoryProbe::known(1_048_576)),
16531 ));
16532 }
16533
16534 fn install_small_local_fixture(engine: &InferenceEngine) -> String {
16535 let schema = engine
16536 .unified_registry()
16537 .find_by_name("Qwen3-0.6B")
16538 .expect("small built-in local model")
16539 .clone();
16540 let model_dir = engine.config.models_dir.join(&schema.name);
16541 std::fs::create_dir_all(&model_dir).unwrap();
16542 std::fs::write(model_dir.join("model.gguf"), b"fixture").unwrap();
16543 std::fs::write(model_dir.join("tokenizer.json"), b"{}").unwrap();
16544 schema.id
16545 }
16546
16547 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16548 fn install_exact_pin_equivalent_fixture(engine: &InferenceEngine) -> String {
16549 let gguf = engine
16550 .unified_registry()
16551 .get("qwen/qwen3-0.6b:q8_0")
16552 .expect("GGUF fixture row")
16553 .clone();
16554 let mlx = engine
16555 .unified_registry()
16556 .get("mlx/qwen3-0.6b:6bit")
16557 .expect("MLX equivalent fixture row")
16558 .clone();
16559
16560 let gguf_dir = engine.config.models_dir.join(&gguf.name);
16561 std::fs::create_dir_all(&gguf_dir).unwrap();
16562 std::fs::write(gguf_dir.join("model.gguf"), b"fixture").unwrap();
16563 std::fs::write(gguf_dir.join("tokenizer.json"), b"{}").unwrap();
16564
16565 let mlx_dir = engine.config.models_dir.join(&mlx.name);
16566 std::fs::create_dir_all(&mlx_dir).unwrap();
16567 std::fs::write(mlx_dir.join("config.json"), b"{}").unwrap();
16568 std::fs::write(mlx_dir.join("model.safetensors"), b"fixture").unwrap();
16569
16570 gguf.id
16571 }
16572
16573 fn remote_stream_fixture_schema(
16574 id: &str,
16575 endpoint: String,
16576 protocol: schema::ApiProtocol,
16577 api_key_env: &str,
16578 ) -> ModelSchema {
16579 ModelSchema {
16580 id: id.into(),
16581 name: "gemini-test".into(),
16582 provider: "test".into(),
16583 family: "test".into(),
16584 version: "1".into(),
16585 capabilities: vec![ModelCapability::Generate],
16586 context_length: 128_000,
16587 max_output_tokens: Some(8_192),
16588 param_count: String::new(),
16589 quantization: None,
16590 performance: Default::default(),
16591 cost: Default::default(),
16592 source: ModelSource::RemoteApi {
16593 endpoint,
16594 api_key_env: api_key_env.into(),
16595 api_key_envs: vec![],
16596 api_version: None,
16597 protocol,
16598 },
16599 tags: vec!["test".into()],
16600 supported_params: vec![],
16601 public_benchmarks: vec![],
16602 trust_tier: TrustTier::Community,
16603 deprecated: false,
16604 available: true,
16605 weights_ready: true,
16606 }
16607 }
16608
16609 async fn assert_remote_model_identity_contract(
16610 protocol: schema::ApiProtocol,
16611 model_id: &str,
16612 provider_model: &str,
16613 api_key_env: &str,
16614 ) {
16615 use wiremock::matchers::{method, path};
16616 use wiremock::{Mock, MockServer, ResponseTemplate};
16617
16618 let server = MockServer::start().await;
16619 let (endpoint_path, response) = match protocol {
16620 schema::ApiProtocol::OpenAiCompat => (
16621 "/v1/chat/completions",
16622 serde_json::json!({
16623 "choices": [{
16624 "message": {"content": "openai exact"},
16625 "finish_reason": "stop"
16626 }],
16627 "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}
16628 }),
16629 ),
16630 schema::ApiProtocol::Anthropic => (
16631 "/v1/messages",
16632 serde_json::json!({
16633 "content": [{"type": "text", "text": "anthropic exact"}],
16634 "stop_reason": "end_turn",
16635 "usage": {"input_tokens": 2, "output_tokens": 2}
16636 }),
16637 ),
16638 _ => unreachable!("identity regression covers the newsroom's two remote providers"),
16639 };
16640 Mock::given(method("POST"))
16641 .and(path(endpoint_path))
16642 .respond_with(ResponseTemplate::new(200).set_body_json(response))
16643 .mount(&server)
16644 .await;
16645 unsafe { std::env::set_var(api_key_env, "fixture") };
16646
16647 let tmp = TempDir::new().unwrap();
16648 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
16649 let mut schema =
16650 remote_stream_fixture_schema(model_id, server.uri(), protocol, api_key_env);
16651 schema.name = provider_model.into();
16652 engine.register_model(schema);
16653
16654 let mut exact_request = GenerateRequest {
16655 prompt: "return the exact identity".into(),
16656 ..Default::default()
16657 };
16658 pin_exact_model_id(&mut exact_request, model_id.into()).unwrap();
16659 let exact = engine.generate_tracked(exact_request).await.unwrap();
16660 assert_eq!(exact.model_used, model_id);
16661 assert_eq!(
16662 exact.model_identity.requested_model_id.as_deref(),
16663 Some(model_id)
16664 );
16665 assert_eq!(exact.model_identity.resolved_model_id, model_id);
16666 let exact_envelope = serde_json::to_value(&exact).unwrap();
16667 assert_eq!(exact_envelope["model_used"], model_id);
16668 assert_eq!(exact_envelope["requested_model_id"], model_id);
16669 assert_eq!(exact_envelope["resolved_model_id"], model_id);
16670 assert!(exact_envelope["row_digest"].is_string());
16671 assert!(exact_envelope["catalog_revision"].is_string());
16672
16673 let loose = engine
16674 .generate_tracked(GenerateRequest {
16675 prompt: "keep legacy display-name routing".into(),
16676 model: Some(provider_model.into()),
16677 params: GenerateParams {
16678 strict_model: true,
16679 ..Default::default()
16680 },
16681 ..Default::default()
16682 })
16683 .await
16684 .unwrap();
16685 assert_eq!(loose.model_used, provider_model);
16686 assert_eq!(loose.model_identity.requested_model_id, None);
16687 assert_eq!(loose.model_identity.resolved_model_id, model_id);
16688
16689 let requests = server.received_requests().await.unwrap();
16690 assert_eq!(requests.len(), 2);
16691 for request in requests {
16692 let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
16693 assert_eq!(body["model"], provider_model);
16694 }
16695
16696 unsafe { std::env::remove_var(api_key_env) };
16697 }
16698
16699 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16705 #[tokio::test]
16706 async fn mlx_device_lock_is_singleton_and_serializes() {
16707 use std::sync::atomic::{AtomicUsize, Ordering};
16708 use std::sync::Arc;
16709 assert!(
16711 Arc::ptr_eq(
16712 &InferenceEngine::mlx_device_lock(),
16713 &InferenceEngine::mlx_device_lock()
16714 ),
16715 "device lock must be a process-wide singleton"
16716 );
16717 let inside = Arc::new(AtomicUsize::new(0));
16719 let peak = Arc::new(AtomicUsize::new(0));
16720 let mut handles = Vec::new();
16721 for _ in 0..8 {
16722 let inside = inside.clone();
16723 let peak = peak.clone();
16724 handles.push(tokio::spawn(async move {
16725 let _g = InferenceEngine::mlx_device_lock().lock_owned().await;
16726 let n = inside.fetch_add(1, Ordering::SeqCst) + 1;
16727 peak.fetch_max(n, Ordering::SeqCst);
16728 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
16729 inside.fetch_sub(1, Ordering::SeqCst);
16730 }));
16731 }
16732 for h in handles {
16733 h.await.unwrap();
16734 }
16735 assert_eq!(
16736 peak.load(Ordering::SeqCst),
16737 1,
16738 "at most one MLX device holder at a time"
16739 );
16740 }
16741
16742 #[test]
16747 fn auto_thinking_budget_gates_code_complex_and_capability() {
16748 assert_eq!(auto_thinking_budget(true, false, true), Some(24_000));
16750 assert_eq!(auto_thinking_budget(false, true, true), Some(8_000));
16752 assert_eq!(auto_thinking_budget(true, true, true), Some(24_000));
16754 assert_eq!(auto_thinking_budget(true, true, false), None);
16756 assert_eq!(auto_thinking_budget(false, false, true), None);
16758 }
16759
16760 #[test]
16761 fn is_explicit_code_intent_keys_on_caller_intent_not_keyword_classifier() {
16762 use crate::intent::{IntentHint, TaskHint};
16763 let code = IntentHint {
16765 task: Some(TaskHint::Code),
16766 ..Default::default()
16767 };
16768 assert!(is_explicit_code_intent(Some(&code)));
16769 assert!(!is_explicit_code_intent(None));
16774 let reasoning = IntentHint {
16776 task: Some(TaskHint::Reasoning),
16777 ..Default::default()
16778 };
16779 assert!(!is_explicit_code_intent(Some(&reasoning)));
16780 let unset = IntentHint {
16782 task: None,
16783 ..Default::default()
16784 };
16785 assert!(!is_explicit_code_intent(Some(&unset)));
16786 }
16787
16788 #[test]
16789 fn strict_model_suppresses_the_local_last_resort_append() {
16790 assert!(should_append_local_last_resort(false, false));
16792 assert!(!should_append_local_last_resort(false, true));
16795 assert!(!should_append_local_last_resort(true, false));
16798 assert!(!should_append_local_last_resort(true, true));
16799 }
16800
16801 #[test]
16802 fn last_resort_fallback_never_returns_an_unrunnable_apple_foundation() {
16803 let tmp = TempDir::new().unwrap();
16818 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
16819 if let Some(name) = engine.first_installed_local_model(false) {
16820 let runnable = engine
16821 .unified_registry()
16822 .find_by_name(&name)
16823 .map(|s| !s.is_foundation_models() || s.available)
16824 .unwrap_or(false);
16825 assert!(
16826 runnable,
16827 "last-resort fallback returned a non-runnable model: {name}"
16828 );
16829 }
16830 }
16831
16832 #[test]
16833 fn empty_tool_catalog_is_no_tools() {
16834 let mut req = GenerateRequest {
16841 prompt: "p".into(),
16842 ..Default::default()
16843 };
16844 assert!(!InferenceEngine::request_has_tools(&req));
16845 req.tools = Some(vec![]);
16846 assert!(!InferenceEngine::request_has_tools(&req));
16847 req.tools = Some(vec![serde_json::json!({
16848 "name": "t", "description": "d", "parameters": {"type": "object"}
16849 })]);
16850 assert!(InferenceEngine::request_has_tools(&req));
16851 }
16852
16853 #[test]
16854 fn top_k_keeps_only_k_highest() {
16855 let mut probs = vec![0.1, 0.4, 0.2, 0.25, 0.05];
16857 InferenceEngine::apply_top_k_top_p(&mut probs, 2, 1.0);
16858 assert!(probs[0] == 0.0 && probs[2] == 0.0 && probs[4] == 0.0);
16860 assert!(probs[1] > 0.0 && probs[3] > 0.0);
16861 let sum: f32 = probs.iter().sum();
16862 assert!((sum - 1.0).abs() < 1e-5, "renormalized to 1.0, got {sum}");
16863 }
16864
16865 #[test]
16866 fn top_k_zero_is_a_noop() {
16867 let mut probs = vec![0.1, 0.4, 0.2, 0.3];
16868 let before = probs.clone();
16869 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 1.0);
16870 assert_eq!(probs, before);
16871 }
16872
16873 #[test]
16874 fn top_p_nucleus_truncates_tail() {
16875 let mut probs = vec![0.6, 0.3, 0.07, 0.03];
16876 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 0.9);
16877 assert!(probs[2] == 0.0 && probs[3] == 0.0);
16879 assert!(probs[0] > 0.0 && probs[1] > 0.0);
16880 }
16881
16882 #[test]
16883 fn truncate_at_stop_excludes_stop_sequence() {
16884 let stops = vec!["<|end|>".to_string(), "STOP".to_string()];
16885 assert_eq!(
16886 tasks::generate::truncate_at_stop("hello world<|end|>extra", &stops),
16887 "hello world"
16888 );
16889 assert_eq!(
16891 tasks::generate::truncate_at_stop("aSTOPb<|end|>c", &stops),
16892 "a"
16893 );
16894 assert_eq!(
16896 tasks::generate::truncate_at_stop("clean output", &stops),
16897 "clean output"
16898 );
16899 assert_eq!(
16901 tasks::generate::truncate_at_stop("text", &["".to_string()]),
16902 "text"
16903 );
16904 }
16905
16906 #[test]
16907 fn no_backend_hint_fires_on_missing_backend_phrases() {
16908 for phrase in [
16910 "no credential for proprietary provider 'parslee'",
16911 "model not found",
16912 "no models available",
16913 "model declares ModelSource::Delegated but no inference runner is registered",
16914 ] {
16915 let hint = no_backend_recovery_hint(phrase)
16916 .unwrap_or_else(|| panic!("expected a hint for {phrase:?}"));
16917 assert!(hint.contains("car models pull"));
16918 assert!(hint.contains("car auth login"));
16921 assert!(hint.contains(phrase));
16923 }
16924 }
16925
16926 #[test]
16927 fn no_backend_hint_opens_with_the_exported_marker() {
16928 let hint = no_backend_recovery_hint(
16932 "model declares ModelSource::Delegated but no inference runner is registered",
16933 )
16934 .expect("a delegated model with no runner is a no-backend case");
16935 assert!(
16936 hint.starts_with(NO_BACKEND_RECOVERY_MARKER),
16937 "hint must open with the exported marker: {hint}"
16938 );
16939 assert!(hint.contains(NO_BACKEND_RECOVERY_MARKER), "{hint}");
16940 }
16941
16942 #[test]
16943 fn route_failure_context_keeps_the_no_backend_marker_behind_the_credential_summary() {
16944 let credential = parslee_signed_out_route_failure();
16948 let underlying = InferenceError::InferenceFailed(
16949 "model declares ModelSource::Delegated but no inference runner is registered".into(),
16950 );
16951 let augmented = apply_route_failure_context(underlying, Some(&credential));
16952 let InferenceError::InferenceFailed(message) = &augmented else {
16953 panic!("a no-backend exhaustion stays InferenceFailed: {augmented:?}");
16954 };
16955 assert!(
16956 message.starts_with(&credential.summary),
16957 "credential context is prepended: {message}"
16958 );
16959 assert!(
16960 message.contains(NO_BACKEND_RECOVERY_MARKER),
16961 "the marker survives the prepend: {message}"
16962 );
16963 assert!(
16964 !message.starts_with(NO_BACKEND_RECOVERY_MARKER),
16965 "the marker is mid-message here, which is why consumers use contains: {message}"
16966 );
16967 }
16968
16969 #[test]
16977 fn credential_failure_is_matchable_as_data() {
16978 let expired = InferenceError::CredentialUnavailable {
16979 provider: "parslee".into(),
16980 model: "parslee/reasoning".into(),
16981 reason: CredentialFailure::Expired {
16982 expires_at: 1_754_257_929,
16983 },
16984 detail: "the Parslee token expired at unix 1754257929 and could not be refreshed"
16985 .into(),
16986 };
16987 let InferenceError::CredentialUnavailable { reason, .. } = &expired else {
16988 panic!("expected CredentialUnavailable");
16989 };
16990 assert_eq!(
16991 *reason,
16992 CredentialFailure::Expired {
16993 expires_at: 1_754_257_929
16994 },
16995 "a consumer must be able to branch on the expiry without parsing prose"
16996 );
16997 assert_ne!(
17001 CredentialFailure::SignedOut,
17002 CredentialFailure::StoreUnreadable
17003 );
17004 assert_ne!(
17005 CredentialFailure::SignedOut,
17006 CredentialFailure::Expired { expires_at: 0 }
17007 );
17008 assert_ne!(
17009 CredentialFailure::StoreUnreadable,
17010 CredentialFailure::RaceRetryable
17011 );
17012 }
17013
17014 #[test]
17024 fn typed_credential_error_still_satisfies_the_substring_classifiers() {
17025 for reason in [
17026 CredentialFailure::Expired { expires_at: 1 },
17027 CredentialFailure::SignedOut,
17028 CredentialFailure::StoreUnreadable,
17029 CredentialFailure::RaceRetryable,
17030 CredentialFailure::EnvVarMissing {
17031 env_var: "OPENAI_API_KEY".into(),
17032 },
17033 ] {
17034 let rendered = InferenceError::CredentialUnavailable {
17035 provider: "parslee".into(),
17036 model: "parslee/reasoning".into(),
17037 reason: reason.clone(),
17038 detail: "detail text".into(),
17039 }
17040 .to_string();
17041 assert!(
17043 rendered
17044 .to_ascii_lowercase()
17045 .contains("no credential for proprietary"),
17046 "classifier substring lost for {reason:?}: {rendered}"
17047 );
17048 assert!(rendered.contains("parslee/reasoning"), "{rendered}");
17050 assert!(rendered.contains("detail text"), "{rendered}");
17052 }
17053 }
17054
17055 #[test]
17056 fn auth_expired_hint_fires_on_auth_rejection_but_not_transient() {
17057 for phrase in [
17059 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17060 "HTTP 403: forbidden",
17061 "invalid_grant: The refresh token is invalid or expired",
17062 "token expired",
17063 ] {
17064 let hint = auth_expired_recovery_hint(phrase)
17065 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
17066 assert!(hint.contains("car auth login"));
17067 assert!(hint.contains(phrase));
17068 }
17069 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
17071 assert!(auth_expired_recovery_hint("request timed out").is_none());
17072 }
17073
17074 #[test]
17092 fn auth_expired_hint_never_carries_the_no_backend_marker() {
17093 for phrase in [
17096 "Parslee org lookup failed: the account service could not be reached",
17097 "Authentication required",
17098 "HTTP 401 Unauthorized",
17099 "HTTP 403 Forbidden",
17100 "invalid_grant: the refresh token is invalid or expired",
17101 "token expired",
17102 ] {
17103 let hint = auth_expired_recovery_hint(phrase)
17104 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
17105 assert!(
17106 !hint.contains(NO_BACKEND_RECOVERY_MARKER),
17107 "the auth-expired hint now carries the no-backend marker, so \
17108 car infer would auto-pull for {phrase:?}: {hint}"
17109 );
17110 assert!(hint.contains("car models pull"), "{hint}");
17114 }
17115 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
17118 }
17119
17120 #[test]
17125 fn auth_rejection_classifier_matches_the_real_expired_token_error() {
17126 assert!(is_auth_rejection_message(
17127 "inference failed: Parslee org lookup failed: HTTP 401 Unauthorized: \
17128 Authentication required"
17129 ));
17130 for transient in [
17133 "API returned 503: service unavailable",
17134 "request timed out",
17135 "connection reset by peer",
17136 ] {
17137 assert!(
17138 !is_auth_rejection_message(transient),
17139 "transient wrongly classified as an auth rejection: {transient:?}"
17140 );
17141 }
17142 }
17143
17144 #[test]
17152 fn typed_errors_classify_by_their_structure_not_their_prose() {
17153 use FallbackReason as R;
17154 let cases: Vec<(InferenceError, R)> = vec![
17155 (
17158 InferenceError::ProviderAccount {
17159 provider: "openai".into(),
17160 status: 401,
17161 message: "provider rejected the API key — check the configured credential"
17162 .into(),
17163 },
17164 R::CredentialRejected,
17165 ),
17166 (
17168 InferenceError::ProviderAccount {
17169 provider: "openrouter".into(),
17170 status: 402,
17171 message: "OpenRouter account is out of credits".into(),
17172 },
17173 R::QuotaExhausted,
17175 ),
17176 (
17177 InferenceError::CredentialUnavailable {
17178 provider: "parslee".into(),
17179 model: "parslee/reasoning".into(),
17180 reason: CredentialFailure::Expired { expires_at: 0 },
17181 detail: "session expired".into(),
17182 },
17183 R::CredentialRejected,
17184 ),
17185 (
17186 InferenceError::CredentialUnavailable {
17187 provider: "openai".into(),
17188 model: "openai/gpt-5.6".into(),
17189 reason: CredentialFailure::EnvVarMissing {
17190 env_var: "OPENAI_API_KEY".into(),
17191 },
17192 detail: "not set".into(),
17193 },
17194 R::CredentialAbsent,
17195 ),
17196 (
17199 InferenceError::CredentialUnavailable {
17200 provider: "parslee".into(),
17201 model: "parslee/reasoning".into(),
17202 reason: CredentialFailure::StoreUnreadable,
17203 detail: "the bounded Keychain helper timed out".into(),
17204 },
17205 R::Failed,
17206 ),
17207 (
17208 InferenceError::Transient {
17209 status: Some(429),
17210 message: "slow down".into(),
17211 },
17212 R::RateLimited,
17213 ),
17214 (
17219 InferenceError::Transient {
17220 status: None,
17221 message: "connection reset by peer".into(),
17222 },
17223 R::Failed,
17224 ),
17225 (
17226 InferenceError::Transient {
17227 status: Some(503),
17228 message: "upstream down".into(),
17229 },
17230 R::Failed,
17231 ),
17232 ];
17233 for (err, want) in cases {
17234 assert_eq!(classify_fallback_reason(&err), want, "{err}");
17235 }
17236 }
17237
17238 #[test]
17246 fn a_quoted_status_in_a_provider_body_does_not_decide_the_bucket() {
17247 use FallbackReason as R;
17248 for (msg, want) in [
17249 (
17250 "API returned 400 Bad Request: your last request 401'd upstream and was unauthorized",
17251 R::Failed,
17252 ),
17253 ("API returned 400 Bad Request: timeout param invalid", R::Failed),
17254 ("API returned 429 Too Many Requests: slow down", R::RateLimited),
17255 ("API returned 401 Unauthorized: bad key", R::CredentialRejected),
17256 ] {
17257 assert_eq!(
17258 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
17259 want,
17260 "{msg}"
17261 );
17262 }
17263 }
17264
17265 #[test]
17269 fn a_parslee_org_lookup_failure_is_classified_by_its_status() {
17270 use FallbackReason as R;
17271 for (msg, want) in [
17272 (
17273 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17274 R::CredentialRejected,
17275 ),
17276 (
17277 "Parslee org lookup failed: HTTP 429 Too Many Requests: slow down",
17278 R::RateLimited,
17279 ),
17280 (
17281 "Parslee org lookup failed: HTTP 500 Internal Server Error: boom",
17282 R::Failed,
17283 ),
17284 ] {
17285 assert_eq!(
17286 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
17287 want,
17288 "{msg}"
17289 );
17290 }
17291 }
17292
17293 #[test]
17297 fn every_skipped_lane_is_recorded_in_order() {
17298 let mut hops = Vec::new();
17299 for (cand, err) in [
17300 (
17301 "lane-one",
17302 InferenceError::Transient {
17303 status: Some(429),
17304 message: "x".into(),
17305 },
17306 ),
17307 (
17310 "lane-two",
17311 InferenceError::Transient {
17312 status: None,
17313 message: "connection refused".into(),
17314 },
17315 ),
17316 (
17317 "lane-three",
17318 InferenceError::ProviderAccount {
17319 provider: "openai".into(),
17320 status: 401,
17321 message: "rejected".into(),
17322 },
17323 ),
17324 ] {
17325 record_fallback_from(&mut hops, cand, &err);
17326 }
17327 assert_eq!(
17328 hops.iter()
17329 .map(|h| h.candidate.as_str())
17330 .collect::<Vec<_>>(),
17331 ["lane-one", "lane-two", "lane-three"]
17332 );
17333 assert_eq!(
17334 hops.iter().map(|h| h.reason).collect::<Vec<_>>(),
17335 [
17336 FallbackReason::RateLimited,
17337 FallbackReason::Failed,
17338 FallbackReason::CredentialRejected
17339 ]
17340 );
17341 }
17342
17343 #[test]
17351 fn a_refused_api_key_is_journaled_but_does_not_claim_sign_in_fixes_it() {
17352 let refused = InferenceError::ProviderAccount {
17353 provider: "openai".into(),
17354 status: 401,
17355 message: "provider rejected the API key — check the configured credential".into(),
17356 };
17357 let mut hops = Vec::new();
17358 record_fallback_from(&mut hops, "openai/gpt-5.6", &refused);
17359 assert_eq!(hops[0].reason, FallbackReason::CredentialRejected);
17360
17361 let mut auth = None;
17363 record_auth_dead_lane(&mut auth, "openai/gpt-5.6", &refused.to_string());
17364 assert_eq!(auth, None, "car auth login does not fix a bad OpenAI key");
17365 }
17366
17367 #[test]
17368 fn auth_dead_lane_records_first_rejected_candidate_only() {
17369 let mut slot: Option<String> = None;
17371 record_auth_dead_lane(
17372 &mut slot,
17373 "parslee/reasoning",
17374 "API returned 503: unavailable",
17375 );
17376 record_auth_dead_lane(&mut slot, "openai/gpt-5.6", "request timed out");
17377 assert_eq!(slot, None);
17378
17379 record_auth_dead_lane(
17381 &mut slot,
17382 "parslee/reasoning",
17383 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17384 );
17385 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
17386
17387 record_auth_dead_lane(&mut slot, "anthropic/claude", "HTTP 403: forbidden");
17390 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
17391 }
17392
17393 #[test]
17394 fn configured_provider_with_expired_token_is_named_before_a_local_oom() {
17395 let expired = InferenceError::CredentialUnavailable {
17396 provider: "parslee".into(),
17397 model: "parslee/reasoning".into(),
17398 reason: CredentialFailure::Expired { expires_at: 42 },
17399 detail: "access token expired".into(),
17400 };
17401 let mut credential = None;
17402 record_route_credential_failure(&mut credential, "parslee/reasoning", &expired, false);
17403
17404 let error = apply_route_failure_context(
17405 InferenceError::InferenceFailed(
17406 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17407 .into(),
17408 ),
17409 credential.as_ref(),
17410 )
17411 .to_string();
17412
17413 let auth_pos = error
17414 .find("Parslee login expired")
17415 .expect("expired login must be named");
17416 let remedy_pos = error
17417 .find("car auth login")
17418 .expect("credential remedy must be named");
17419 let oom_pos = error
17420 .find("9059 MB")
17421 .expect("fallback error must remain as secondary detail");
17422 assert!(auth_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
17423 assert!(is_auth_failure_message(&error), "{error}");
17424 }
17425
17426 #[test]
17427 fn absent_login_is_named_before_a_local_oom() {
17428 let failure = parslee_signed_out_route_failure();
17429 let error = apply_route_failure_context(
17430 InferenceError::InferenceFailed("local fallback needs 9059 MB".into()),
17431 Some(&failure),
17432 )
17433 .to_string();
17434
17435 let absent_pos = error
17436 .find("Parslee login is absent")
17437 .expect("missing login must be named");
17438 let remedy_pos = error
17439 .find("car auth login")
17440 .expect("credential remedy must be named");
17441 let oom_pos = error
17442 .find("9059 MB")
17443 .expect("fallback OOM must remain as secondary detail");
17444 assert!(absent_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
17445 }
17446
17447 #[test]
17448 fn genuine_local_oom_is_not_reclassified_as_auth() {
17449 let oom = InferenceError::InferenceFailed(
17450 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17451 .into(),
17452 );
17453 let error = apply_route_failure_context(oom, None);
17454 assert!(matches!(error, InferenceError::InferenceFailed(ref message)
17455 if message.starts_with("This model needs about 9059 MB")));
17456 }
17457
17458 #[test]
17459 fn unconfigured_provider_is_not_surfaced_over_the_terminal_failure() {
17460 let mut credential = None;
17461 record_route_credential_failure(
17462 &mut credential,
17463 "openai/gpt-5.6",
17464 &InferenceError::CredentialUnavailable {
17465 provider: "openai".into(),
17466 model: "openai/gpt-5.6".into(),
17467 reason: CredentialFailure::EnvVarMissing {
17468 env_var: "OPENAI_API_KEY".into(),
17469 },
17470 detail: "set OPENAI_API_KEY".into(),
17471 },
17472 false,
17473 );
17474
17475 let terminal = InferenceError::InferenceFailed(
17476 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17477 .into(),
17478 );
17479 let error = apply_route_failure_context(terminal, credential.as_ref()).to_string();
17480 assert!(
17481 credential.is_none(),
17482 "an unconfigured fallback is ambient noise"
17483 );
17484 assert!(error.contains("9059 MB"), "{error}");
17485 assert!(!error.contains("OPENAI_API_KEY"), "{error}");
17486 assert!(!is_auth_failure_message(&error), "{error}");
17487 }
17488
17489 fn workspace_required() -> InferenceError {
17493 InferenceError::WorkspaceRequired {
17494 provider: "Parslee".into(),
17495 detail: "finish setting up at https://parslee.ai, then try again".into(),
17496 }
17497 }
17498
17499 #[test]
17508 fn workspace_required_reads_as_configuration_not_sign_in() {
17509 let rendered = workspace_required().to_string();
17510 for marker in AUTH_FAILURE_MESSAGE_MARKERS {
17511 assert!(
17512 !rendered
17513 .to_ascii_lowercase()
17514 .contains(&marker.to_ascii_lowercase()),
17515 "`{marker}` must not appear in the no-workspace text: {rendered}"
17516 );
17517 }
17518 assert!(!is_auth_failure_message(&rendered), "{rendered}");
17519 assert!(!is_auth_rejection_message(&rendered), "{rendered}");
17520 assert!(rendered.contains("https://parslee.ai"), "{rendered}");
17521 }
17522
17523 #[test]
17530 fn workspace_required_survives_a_mixed_chain() {
17531 let mut credential = None;
17532 record_route_credential_failure(
17533 &mut credential,
17534 "openai/gpt-5.6",
17535 &InferenceError::CredentialUnavailable {
17536 provider: "openai".into(),
17537 model: "openai/gpt-5.6".into(),
17538 reason: CredentialFailure::SignedOut,
17539 detail: "no account is signed in".into(),
17540 },
17541 true,
17542 );
17543 assert!(
17544 credential.is_some(),
17545 "the fixture must actually record a credential failure"
17546 );
17547
17548 let final_error = apply_route_failure_context(workspace_required(), credential.as_ref());
17549 assert!(
17550 matches!(final_error, InferenceError::WorkspaceRequired { .. }),
17551 "the type must survive the route context: {final_error}"
17552 );
17553 assert!(!is_auth_failure_message(&final_error.to_string()));
17554 assert!(
17555 error_ends_fallback_chain(&final_error),
17556 "no later candidate on that account can succeed"
17557 );
17558 }
17559
17560 #[test]
17567 fn repeated_workspace_required_leaves_model_health_untouched() {
17568 assert!(
17569 !error_counts_against_circuit_breaker(&workspace_required()),
17570 "the breaker must not count a workspace gap"
17571 );
17572
17573 let mut tracker = OutcomeTracker::new();
17574 for _ in 0..5 {
17575 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
17576 record_dispatch_failure(&mut tracker, &trace, &workspace_required());
17577 }
17578 let failed = tracker
17579 .profile("parslee/advisor")
17580 .map(|p| p.fail_count)
17581 .unwrap_or(0);
17582 assert_eq!(
17583 failed, 0,
17584 "five workspace refusals must not bench the model"
17585 );
17586
17587 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
17588 record_dispatch_failure(
17589 &mut tracker,
17590 &trace,
17591 &InferenceError::InferenceFailed("decoder fell over".into()),
17592 );
17593 assert_eq!(
17594 tracker
17595 .profile("parslee/advisor")
17596 .map(|p| p.fail_count)
17597 .unwrap_or(0),
17598 1,
17599 "a genuine failure must still count, or the test above proves nothing"
17600 );
17601 }
17602
17603 #[test]
17604 fn explicitly_requested_missing_credential_uses_the_shared_auth_table() {
17605 let mut credential = None;
17606 record_route_credential_failure(
17607 &mut credential,
17608 "openai/gpt-5.6",
17609 &InferenceError::CredentialUnavailable {
17610 provider: "openai".into(),
17611 model: "openai/gpt-5.6".into(),
17612 reason: CredentialFailure::EnvVarMissing {
17613 env_var: "OPENAI_API_KEY".into(),
17614 },
17615 detail: "set OPENAI_API_KEY".into(),
17616 },
17617 true,
17618 );
17619 let summary = credential
17620 .expect("an explicitly requested provider must surface its missing key")
17621 .summary;
17622 assert!(summary.contains(AUTH_ENV_MISSING_MARKER), "{summary}");
17623 assert!(is_auth_failure_message(&summary), "{summary}");
17624 }
17625
17626 #[test]
17627 fn latest_actionable_credential_failure_wins() {
17628 let mut credential = None;
17629 record_route_credential_failure(
17630 &mut credential,
17631 "parslee/reasoning",
17632 &InferenceError::ProviderAccount {
17633 provider: "parslee".into(),
17634 status: 401,
17635 message: "Unauthorized".into(),
17636 },
17637 false,
17638 );
17639 record_route_credential_failure(
17640 &mut credential,
17641 "anthropic/claude",
17642 &InferenceError::CredentialUnavailable {
17643 provider: "anthropic".into(),
17644 model: "anthropic/claude".into(),
17645 reason: CredentialFailure::Expired { expires_at: 43 },
17646 detail: "configured token expired".into(),
17647 },
17648 false,
17649 );
17650
17651 assert_eq!(
17652 credential
17653 .expect("latest actionable credential cause must be retained")
17654 .summary,
17655 "anthropic login expired for `anthropic/claude` — run `car auth login`"
17656 );
17657 }
17658
17659 #[test]
17660 fn store_unreadable_summary_uses_the_shared_auth_table() {
17661 let summary = route_credential_failure(
17662 "parslee/reasoning",
17663 &InferenceError::CredentialUnavailable {
17664 provider: "parslee".into(),
17665 model: "parslee/reasoning".into(),
17666 reason: CredentialFailure::StoreUnreadable,
17667 detail: "keychain helper timed out".into(),
17668 },
17669 false,
17670 )
17671 .expect("an unreadable configured credential store is actionable");
17672 assert!(summary.contains(AUTH_STORE_UNREADABLE_MARKER), "{summary}");
17673 assert!(is_auth_failure_message(&summary), "{summary}");
17674 }
17675
17676 fn chain_gate_fixture_schema(id: &str, provider: &str, source: ModelSource) -> ModelSchema {
17677 ModelSchema {
17678 id: id.into(),
17679 name: id.into(),
17680 provider: provider.into(),
17681 family: "test".into(),
17682 version: "1".into(),
17683 capabilities: vec![ModelCapability::Generate],
17684 context_length: 32_768,
17685 max_output_tokens: Some(4_096),
17686 param_count: String::new(),
17687 quantization: None,
17688 performance: Default::default(),
17689 cost: Default::default(),
17690 source,
17691 tags: vec![],
17692 supported_params: vec![],
17693 public_benchmarks: vec![],
17694 trust_tier: TrustTier::Community,
17695 deprecated: false,
17696 available: true,
17697 weights_ready: true,
17698 }
17699 }
17700
17701 #[test]
17702 fn signed_out_pre_seed_is_gated_on_a_parslee_route_in_the_chain() {
17703 let local = chain_gate_fixture_schema(
17704 "qwen/qwen3-4b:q4_k_m",
17705 "qwen",
17706 ModelSource::Mlx {
17707 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
17708 hf_weight_file: None,
17709 },
17710 );
17711 let ollama = chain_gate_fixture_schema(
17712 "ollama/llama3",
17713 "ollama",
17714 ModelSource::Ollama {
17715 model_tag: "llama3".into(),
17716 host: "http://localhost:11434".into(),
17717 },
17718 );
17719 let parslee = chain_gate_fixture_schema(
17720 "parslee/reasoning",
17721 "parslee",
17722 ModelSource::Proprietary {
17723 provider: "parslee".into(),
17724 endpoint: "https://api.parslee.ai".into(),
17725 auth: ProprietaryAuth::OAuth2Pkce {
17726 authority: "https://login.example".into(),
17727 client_id: "client".into(),
17728 scopes: vec![],
17729 },
17730 protocol: Default::default(),
17731 },
17732 );
17733 let cloud = chain_gate_fixture_schema(
17734 "openai/gpt-5.6",
17735 "openai",
17736 ModelSource::RemoteApi {
17737 endpoint: "https://api.openai.com".into(),
17738 api_key_env: "OPENAI_API_KEY".into(),
17739 api_key_envs: vec![],
17740 api_version: None,
17741 protocol: schema::ApiProtocol::OpenAiCompat,
17742 },
17743 );
17744 let schemas: std::collections::HashMap<&str, &ModelSchema> = [
17745 ("qwen/qwen3-4b:q4_k_m", &local),
17746 ("ollama/llama3", &ollama),
17747 ("parslee/reasoning", &parslee),
17748 ("openai/gpt-5.6", &cloud),
17749 ]
17750 .into_iter()
17751 .collect();
17752 let resolve = |m: &str| schemas.get(m).copied();
17753
17754 let local_only = vec![
17756 "qwen/qwen3-4b:q4_k_m".to_string(),
17757 "ollama/llama3".to_string(),
17758 ];
17759 assert!(!chain_includes_parslee_route(resolve, &local_only));
17760
17761 let with_parslee = vec![
17763 "parslee/reasoning".to_string(),
17764 "qwen/qwen3-4b:q4_k_m".to_string(),
17765 ];
17766 assert!(chain_includes_parslee_route(resolve, &with_parslee));
17767 let with_cloud = vec![
17768 "openai/gpt-5.6".to_string(),
17769 "qwen/qwen3-4b:q4_k_m".to_string(),
17770 ];
17771 assert!(!chain_includes_parslee_route(resolve, &with_cloud));
17775 let mut credential = chain_includes_parslee_route(resolve, &with_cloud)
17776 .then(parslee_signed_out_route_failure);
17777 let outage = InferenceError::Transient {
17778 status: Some(500),
17779 message: "OpenAI HTTP 500 Internal Server Error".into(),
17780 };
17781 record_route_credential_failure(&mut credential, &with_cloud[0], &outage, false);
17782 let terminal = apply_route_failure_context(outage, credential.as_ref());
17783 assert!(matches!(terminal, InferenceError::Transient { .. }));
17784 assert!(!is_auth_failure_message(&terminal.to_string()));
17785 let oom = InferenceError::InferenceFailed("local model out of memory".into());
17786 record_route_credential_failure(&mut credential, &with_cloud[1], &oom, false);
17787 let terminal = apply_route_failure_context(oom, credential.as_ref()).to_string();
17788 assert!(!terminal.contains("Parslee"), "{terminal}");
17789 assert!(!is_auth_failure_message(&terminal), "{terminal}");
17790
17791 record_route_credential_failure(
17793 &mut credential,
17794 &with_cloud[0],
17795 &InferenceError::ProviderAccount {
17796 provider: "openai".into(),
17797 status: 401,
17798 message: "invalid API key".into(),
17799 },
17800 false,
17801 );
17802 let terminal = apply_route_failure_context(
17803 InferenceError::InferenceFailed("local model out of memory".into()),
17804 credential.as_ref(),
17805 )
17806 .to_string();
17807 assert!(terminal.contains("openai"), "{terminal}");
17808 assert!(!terminal.contains("Parslee"), "{terminal}");
17809 assert!(is_auth_failure_message(&terminal), "{terminal}");
17810
17811 let unknown = vec!["missing/model".to_string()];
17813 assert!(!chain_includes_parslee_route(resolve, &unknown));
17814 }
17815
17816 #[test]
17820 fn local_only_terminal_failures_are_not_relabeled_as_auth() {
17821 let cases = [
17822 InferenceError::InferenceFailed(
17824 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17825 .into(),
17826 ),
17827 InferenceError::Transient {
17829 status: Some(500),
17830 message: "HTTP 500 Internal Server Error".into(),
17831 },
17832 InferenceError::InferenceFailed("llama runner process has terminated".into()),
17834 ];
17835 for case in cases {
17836 let error = apply_route_failure_context(case, None).to_string();
17837 assert!(!error.contains("Parslee login is absent"), "{error}");
17838 assert!(!is_auth_failure_message(&error), "{error}");
17839 }
17840
17841 let error = apply_route_failure_context(
17845 InferenceError::ModelNotFound("qwen/qwen3-4b:q4_k_m".into()),
17846 None,
17847 )
17848 .to_string();
17849 assert!(!error.contains("Parslee login is absent"), "{error}");
17850 assert!(error.contains("car models pull"), "{error}");
17851 }
17852
17853 #[test]
17856 fn credential_context_preserves_the_typed_terminal_variant() {
17857 let credential = parslee_signed_out_route_failure();
17858
17859 let transient = apply_route_failure_context(
17860 InferenceError::Transient {
17861 status: Some(500),
17862 message: "HTTP 500 Internal Server Error".into(),
17863 },
17864 Some(&credential),
17865 );
17866 match &transient {
17867 InferenceError::Transient { status, message } => {
17868 assert_eq!(*status, Some(500));
17869 assert!(message.starts_with("Parslee login is absent"), "{message}");
17870 assert!(message.contains("HTTP 500"), "{message}");
17871 }
17872 other => panic!("Transient must stay Transient, got {other:?}"),
17873 }
17874
17875 let account = apply_route_failure_context(
17876 InferenceError::ProviderAccount {
17877 provider: "openai".into(),
17878 status: 402,
17879 message: "insufficient credits".into(),
17880 },
17881 Some(&credential),
17882 );
17883 assert!(
17884 matches!(
17885 &account,
17886 InferenceError::ProviderAccount {
17887 provider,
17888 status: 402,
17889 ..
17890 } if provider == "openai"
17891 ),
17892 "ProviderAccount must stay ProviderAccount, got {account:?}"
17893 );
17894
17895 let unavailable = apply_route_failure_context(
17896 InferenceError::CredentialUnavailable {
17897 provider: "parslee".into(),
17898 model: "parslee/reasoning".into(),
17899 reason: CredentialFailure::Expired { expires_at: 42 },
17900 detail: "access token expired".into(),
17901 },
17902 Some(&credential),
17903 );
17904 match &unavailable {
17905 InferenceError::CredentialUnavailable { reason, detail, .. } => {
17906 assert_eq!(*reason, CredentialFailure::Expired { expires_at: 42 });
17907 assert!(detail.contains("Parslee login is absent"), "{detail}");
17908 }
17909 other => panic!("CredentialUnavailable must keep its reason data, got {other:?}"),
17910 }
17911 assert!(is_auth_failure_message(&unavailable.to_string()));
17912 }
17913
17914 #[test]
17917 fn non_parslee_rejection_summaries_match_the_shared_classifier() {
17918 let rejected = route_credential_failure(
17919 "openai/gpt-5.6",
17920 &InferenceError::ProviderAccount {
17921 provider: "openai".into(),
17922 status: 403,
17923 message: "key revoked".into(),
17924 },
17925 false,
17926 )
17927 .expect("a 403 from a configured provider is actionable");
17928 assert!(rejected.contains("credential was rejected"), "{rejected}");
17929 assert!(is_auth_failure_message(&rejected), "{rejected}");
17930
17931 let generic = route_credential_failure(
17932 "openai/gpt-5.6",
17933 &InferenceError::InferenceFailed("upstream said: token expired".into()),
17934 false,
17935 )
17936 .expect("an auth-rejection message from a non-Parslee route is actionable");
17937 assert!(generic.contains("repair its provider login"), "{generic}");
17938 assert!(is_auth_failure_message(&generic), "{generic}");
17939 }
17940
17941 #[test]
17946 fn the_recovery_hints_lead_with_parslee_sign_in() {
17947 for hint in [
17948 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
17949 .expect("a missing credential is a no-backend case"),
17950 auth_expired_recovery_hint(
17951 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17952 )
17953 .expect("a rejected session is an auth case"),
17954 ] {
17955 let login = hint
17956 .find("car auth login")
17957 .expect("sign-in must be offered");
17958 let pull = hint
17959 .find("car models pull")
17960 .expect("the local path must still be offered");
17961 assert!(login < pull, "sign-in must be read first:\n{hint}");
17962 }
17963 assert!(
17966 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
17967 .expect("hint")
17968 .starts_with(NO_BACKEND_RECOVERY_MARKER),
17969 "the marker opening is load-bearing"
17970 );
17971 }
17972
17973 #[test]
17976 fn fresh_install_exhaustion_keeps_the_models_pull_guidance() {
17977 let credential = parslee_signed_out_route_failure();
17978 let error = apply_route_failure_context(
17979 InferenceError::InferenceFailed("no models available for generate".into()),
17980 Some(&credential),
17981 )
17982 .to_string();
17983 assert!(error.contains("Parslee login is absent"), "{error}");
17984 assert!(
17985 error.contains(&format!("car models pull {LOCAL_TOOL_MODEL_PULL_ID}")),
17986 "{error}"
17987 );
17988 assert!(error.contains("car auth login"), "{error}");
17989 }
17990
17991 #[test]
17994 fn auth_fallback_from_round_trips_and_defaults_to_none() {
17995 let mut result: InferenceResult = serde_json::from_value(serde_json::json!({
17996 "text": "hi",
17997 "tool_calls": [],
17998 "trace_id": "t",
17999 "model_used": "openai/gpt-5.6",
18000 "latency_ms": 1,
18001 }))
18002 .expect("a payload without the field still deserializes");
18003 assert_eq!(result.auth_fallback_from, None);
18004 let json = serde_json::to_value(&result).unwrap();
18005 assert!(json.get("auth_fallback_from").is_none());
18006
18007 result.auth_fallback_from = Some("parslee/reasoning".to_string());
18008 let json = serde_json::to_value(&result).unwrap();
18009 assert_eq!(json["auth_fallback_from"], "parslee/reasoning");
18010 let back: InferenceResult = serde_json::from_value(json).unwrap();
18011 assert_eq!(
18012 back.auth_fallback_from.as_deref(),
18013 Some("parslee/reasoning")
18014 );
18015 }
18016
18017 #[test]
18022 fn local_last_resort_turn_carries_model_id_and_flag() {
18023 let ordinary: InferenceResult = serde_json::from_value(serde_json::json!({
18024 "text": "hi",
18025 "tool_calls": [],
18026 "trace_id": "ordinary",
18027 "model_used": "anthropic/claude-haiku-4-5:latest",
18028 "latency_ms": 1,
18029 }))
18030 .expect("older payloads default the marker");
18031 assert!(!ordinary.local_last_resort);
18032
18033 let fallback: InferenceResult = serde_json::from_value(serde_json::json!({
18034 "text": "offline answer",
18035 "tool_calls": [],
18036 "trace_id": "fallback",
18037 "model_used": "Qwen3 4B MLX",
18038 "resolved_model_id": "mlx/qwen3-4b:4bit",
18039 "latency_ms": 1,
18040 "local_last_resort": true,
18041 }))
18042 .expect("fallback attribution payload");
18043 assert_eq!(fallback.served_model_id(), "mlx/qwen3-4b:4bit");
18044 assert!(fallback.local_last_resort);
18045
18046 assert!(!is_local_last_resort(None, "mlx/qwen3-4b:4bit"));
18047 assert!(report_local_last_resort_served(
18048 Some("mlx/qwen3-4b:4bit"),
18049 "mlx/qwen3-4b:4bit",
18050 "mlx/qwen3-4b:4bit"
18051 ));
18052 assert!(!is_local_last_resort(
18053 Some("mlx/qwen3-4b:4bit"),
18054 "anthropic/claude-haiku-4-5:latest"
18055 ));
18056 }
18057
18058 #[test]
18059 fn no_backend_hint_passes_through_transient_errors() {
18060 for phrase in [
18063 "API returned 401 Unauthorized",
18064 "API returned 429 Too Many Requests",
18065 "API returned 500 Internal Server Error",
18066 "connection refused",
18067 "request timed out",
18068 "parse response: unexpected end of input",
18069 ] {
18070 assert!(
18071 no_backend_recovery_hint(phrase).is_none(),
18072 "transient error wrongly classified as no-backend: {phrase:?}"
18073 );
18074 }
18075 }
18076
18077 static ENV_MUTEX: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
18080
18081 #[cfg(unix)]
18082 #[tokio::test]
18083 async fn exact_codex_subscription_row_dispatches_without_an_openai_key() {
18084 use std::os::unix::fs::PermissionsExt;
18085
18086 let _environment = ENV_MUTEX.lock().await;
18087 let _restore = RestoredEnvironment::capture(&["CAR_CODEX_BIN", "OPENAI_API_KEY"]);
18088 let tmp = TempDir::new().unwrap();
18089 let fixture = tmp.path().join("codex-fixture.sh");
18090 std::fs::write(
18091 &fixture,
18092 r#"#!/bin/sh
18093if [ -n "${OPENAI_API_KEY-}" ]; then
18094 echo 'OPENAI_API_KEY leaked' >&2
18095 exit 91
18096fi
18097cat >/dev/null
18098printf '%s\n' '{"type":"turn.started"}'
18099printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"fixture newsroom answer"}}'
18100printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":17,"output_tokens":5}}'
18101"#,
18102 )
18103 .unwrap();
18104 std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o700)).unwrap();
18105 unsafe {
18106 std::env::set_var("CAR_CODEX_BIN", &fixture);
18107 std::env::set_var("OPENAI_API_KEY", "must-not-reach-codex");
18108 }
18109
18110 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
18111 let mut request = GenerateRequest {
18112 prompt: "write a brief".into(),
18113 params: GenerateParams {
18114 max_tokens: 1056,
18115 strict_model: true,
18116 ..Default::default()
18117 },
18118 ..Default::default()
18119 };
18120 pin_exact_model_id(&mut request, "openai/gpt-5.6-sol:high".into()).unwrap();
18121 let result = engine.generate_tracked(request).await.unwrap();
18122
18123 assert_eq!(result.model_used, "openai/gpt-5.6-sol:high");
18124 assert_eq!(result.text, "fixture newsroom answer");
18125 assert_eq!(result.usage.as_ref().unwrap().total_tokens, 22);
18126 assert_eq!(result.stop_reason, None, "Codex reports no finish reason");
18127 assert!(result.tool_calls.is_empty());
18128 }
18129
18130 #[tokio::test]
18131 async fn isolated_test_child_removes_parent_auth_lock_override() {
18132 const CHILD_SENTINEL: &str = "CAR_AUTH_LOCK_SANITIZER_CHILD";
18133 if std::env::var_os(CHILD_SENTINEL).is_some() {
18134 assert!(
18135 std::env::var_os("CAR_AUTH_LOCK_PATH").is_none(),
18136 "an isolated credential test must not inherit the invoking process's auth lock"
18137 );
18138 return;
18139 }
18140
18141 let _environment = ENV_MUTEX.lock().await;
18142 let _restore = RestoredEnvironment::capture(&["CAR_AUTH_LOCK_PATH"]);
18143 unsafe {
18144 std::env::set_var(
18145 "CAR_AUTH_LOCK_PATH",
18146 "/sentinel/must-not-reach-isolated-child.lock",
18147 );
18148 }
18149 assert!(!crate::run_in_isolated_test_process(
18150 "tests::isolated_test_child_removes_parent_auth_lock_override",
18151 CHILD_SENTINEL,
18152 ));
18153 }
18154
18155 fn test_config(models_dir: PathBuf) -> InferenceConfig {
18156 let state_root = models_dir
18160 .parent()
18161 .map(Path::to_path_buf)
18162 .unwrap_or_else(|| models_dir.clone());
18163 InferenceConfig {
18164 models_dir,
18165 state_root,
18166 device: None,
18167 generation_model: "Qwen3-0.6B".into(),
18168 preferred_generation_model: None,
18169 embedding_model: "Qwen3-Embedding-0.6B".into(),
18170 preferred_embedding_model: None,
18171 classification_model: "Qwen3-0.6B".into(),
18172 preferred_classification_model: None,
18173 }
18174 }
18175
18176 fn metal_mac_for_fit(ram_gb: u64) -> HardwareInfo {
18177 HardwareInfo {
18178 os: "macos".into(),
18179 arch: "aarch64".into(),
18180 cpu_cores: 8,
18181 total_ram_mb: ram_gb * 1024,
18182 gpu_backend: crate::hardware::GpuBackend::Metal,
18183 gpu_memory_mb: None,
18184 gpu_devices: vec![],
18185 recommended_model: String::new(),
18186 recommended_context: 8_192,
18187 max_model_mb: 0,
18188 }
18189 }
18190
18191 #[test]
18198 fn unified_rows_carry_fit_per_machine_and_keep_deprecated_rows() {
18199 if !crate::run_in_isolated_test_process(
18200 "tests::unified_rows_carry_fit_per_machine_and_keep_deprecated_rows",
18201 "CAR_UNIFIED_ROWS_FIT_TEST_CHILD",
18202 ) {
18203 return;
18204 }
18205
18206 let _credential_scope = crate::openrouter::test_credential_scope();
18213 crate::openrouter::set_test_credential(Some("unified-rows-fit-test-key"));
18214 let secret_activity_before = car_secrets::secret_store_activity();
18215 let root = tempfile::tempdir().unwrap();
18216 let config = test_config(root.path().join("weights"));
18217 let mut engine = InferenceEngine::new(config);
18218 let mut retired = engine
18219 .list_schemas()
18220 .into_iter()
18221 .find(|schema| schema.id == "mlx/qwen3-4b:4bit")
18222 .expect("builtin 4B MLX row");
18223 retired.id = "test/retired-4b:4bit".into();
18224 retired.name = "Retired 4B".into();
18225 retired.deprecated = true;
18226 engine.register_model(retired);
18227 let policy = resource_policy::ResourcePolicy::everyday();
18228
18229 let at_8 = engine.list_models_unified_for(&metal_mac_for_fit(8), &policy);
18230 let at_32 = engine.list_models_unified_for(&metal_mac_for_fit(32), &policy);
18231 let ids = |rows: &[ModelInfo]| rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
18232 assert_eq!(
18233 ids(&at_8),
18234 ids(&at_32),
18235 "the machine never removes or reorders a row"
18236 );
18237 let row = |rows: &[ModelInfo], id: &str| {
18238 rows.iter()
18239 .find(|row| row.id == id)
18240 .unwrap_or_else(|| panic!("{id} missing"))
18241 .clone()
18242 };
18243
18244 let eight_b_small = row(&at_8, "mlx/qwen3-8b:4bit");
18245 let eight_b_large = row(&at_32, "mlx/qwen3-8b:4bit");
18246 assert_eq!(eight_b_small.fit, ModelFitStatus::TooBig);
18247 assert_eq!(eight_b_large.fit, ModelFitStatus::Fits);
18248 assert!(eight_b_small.platform_compatible && eight_b_large.platform_compatible);
18249 assert_eq!(
18250 eight_b_small.estimated_peak_mb, eight_b_large.estimated_peak_mb,
18251 "the estimate is the model's; only the budget differs"
18252 );
18253 assert!(eight_b_small.estimated_peak_mb.is_some_and(|mb| mb > 4_800));
18254 assert_eq!(eight_b_small.family.as_deref(), Some("qwen3"));
18255 assert!(eight_b_small.version.is_some());
18256
18257 let retired = row(&at_8, "test/retired-4b:4bit");
18258 assert!(retired.deprecated, "deprecated rows stay listed, flagged");
18259 assert!(!eight_b_small.deprecated);
18260
18261 let mut saw_remote = false;
18262 for remote in at_8.iter().filter(|row| !row.is_local) {
18263 saw_remote = true;
18264 assert_eq!(remote.fit, ModelFitStatus::Fits, "{}", remote.id);
18265 assert!(remote.platform_compatible, "{}", remote.id);
18266 assert_eq!(remote.estimated_peak_mb, None, "{}", remote.id);
18267 assert_eq!(
18268 remote.family, None,
18269 "{}: no upstream identifier here",
18270 remote.id
18271 );
18272 assert_eq!(remote.version, None, "{}", remote.id);
18273 }
18274 assert!(saw_remote);
18275 assert_eq!(
18276 car_secrets::secret_store_activity(),
18277 secret_activity_before,
18278 "catalog snapshots must use the injected credential without consulting host secrets"
18279 );
18280
18281 for (small, large) in at_8.iter().zip(&at_32) {
18284 let strip = |row: &ModelInfo| {
18285 let mut value = serde_json::to_value(row).unwrap();
18286 let object = value.as_object_mut().unwrap();
18287 for key in ["fit", "estimated_peak_mb", "platform_compatible"] {
18288 object.remove(key);
18289 }
18290 value
18291 };
18292 assert_eq!(strip(small), strip(large), "{}", small.id);
18293 }
18294 }
18295
18296 #[tokio::test]
18297 async fn explicit_model_blocked_by_zero_budget_does_not_substitute_or_download() {
18298 let root = tempfile::tempdir().unwrap();
18299 let models_dir = root.path().join("weights");
18300 let config = test_config(models_dir.clone());
18301 let repository =
18302 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
18303 crate::resource_policy::ResourcePolicyRepository::save(
18304 &repository,
18305 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
18306 )
18307 .unwrap();
18308 let engine = InferenceEngine::new(config);
18309
18310 let error = engine
18311 .generate_tracked(GenerateRequest {
18312 prompt: "hello".into(),
18313 model: Some("mlx/qwen3-4b:4bit".into()),
18314 ..Default::default()
18315 })
18316 .await
18317 .unwrap_err();
18318
18319 assert!(matches!(
18320 error,
18321 InferenceError::LocalResourceBlocked {
18322 preflight: crate::resource_policy::LocalLoadPreflight {
18323 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
18324 ..
18325 },
18326 ..
18327 }
18328 ));
18329 assert!(
18330 !models_dir.exists() || std::fs::read_dir(models_dir).unwrap().next().is_none(),
18331 "admission must happen before download/load"
18332 );
18333 }
18334
18335 #[tokio::test]
18336 async fn external_vllm_mlx_bypasses_local_admission_while_managed_artifact_does_not() {
18337 let schema = crate::vllm_mlx::to_model_schema(
18338 &crate::vllm_mlx::DiscoveredModel {
18339 id: "mlx-community/Qwen3-4B-4bit".into(),
18340 owned_by: None,
18341 },
18342 "http://localhost:8000",
18343 );
18344 assert!(!InferenceEngine::requires_local_admission(&schema));
18345
18346 let root = tempfile::tempdir().unwrap();
18347 let config = test_config(root.path().join("weights"));
18348 let repository =
18349 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
18350 crate::resource_policy::ResourcePolicyRepository::save(
18351 &repository,
18352 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
18353 )
18354 .unwrap();
18355 let engine = InferenceEngine::new(config);
18356 let (external, reservation) = engine
18357 .vllm_live_schema(schema.clone(), None, 0)
18358 .await
18359 .unwrap();
18360 assert!(matches!(
18361 external.source,
18362 ModelSource::VllmMlx { ref endpoint, .. } if endpoint == "http://localhost:8000"
18363 ));
18364 assert!(reservation.is_none());
18365
18366 let mut managed = schema;
18367 managed.source = ModelSource::ManagedVllmMlx {
18368 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
18369 hf_weight_file: None,
18370 };
18371 assert!(InferenceEngine::requires_local_admission(&managed));
18372 let error = engine
18373 .reserve_local_request(&managed, 0)
18374 .expect_err("zero-GB policy must reject before vllm runtime/download/spawn");
18375 assert!(matches!(
18376 error,
18377 InferenceError::LocalResourceBlocked {
18378 preflight: crate::resource_policy::LocalLoadPreflight {
18379 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
18380 ..
18381 },
18382 ..
18383 }
18384 ));
18385 }
18386
18387 #[cfg(unix)]
18388 async fn assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(
18389 streaming: bool,
18390 cancel_first: bool,
18391 ) {
18392 struct FixedProbe;
18393 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
18394 fn available_memory_mb(
18395 &self,
18396 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
18397 Ok(Some(24_000))
18398 }
18399 }
18400
18401 let Some(python) = crate::vllm_runtime::test_python_interpreter() else {
18402 panic!("a real Python interpreter is required for the managed-vllm dispatch fixture");
18403 };
18404 let root = tempfile::tempdir().unwrap();
18405 let script = root.path().join("fake-vllm-mlx");
18406 let spawned = root.path().join("spawned");
18407 let release = root.path().join("release");
18408 std::fs::write(
18409 &script,
18410 format!(
18411 "#!{}\n\
18412 import http.server, os, sys, time\n\
18413 import socket as _s; _s.getfqdn = lambda n='': n\n\
18414 marker = {:?}\n\
18415 release = {:?}\n\
18416 open(marker, 'w').close()\n\
18417 while not os.path.exists(release): time.sleep(0.01)\n\
18418 port = int(sys.argv[sys.argv.index('--port') + 1])\n\
18419 class H(http.server.BaseHTTPRequestHandler):\n\
18420 \x20 def do_GET(self):\n\
18421 \x20 self.send_response(200); self.end_headers(); self.wfile.write(b'ok')\n\
18422 \x20 def do_POST(self):\n\
18423 \x20 length = int(self.headers.get('content-length', '0'))\n\
18424 \x20 request = self.rfile.read(length).replace(b' ', b'')\n\
18425 \x20 if b'\"stream\":true' in request:\n\
18426 \x20 body = b'data: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"round13-stream\"}},\"finish_reason\":null}}]}}\\n\\ndata: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"stop\"}}]}}\\n\\ndata: [DONE]\\n\\n'\n\
18427 \x20 content_type = 'text/event-stream'\n\
18428 \x20 else:\n\
18429 \x20 body = b'{{\"id\":\"fixture\",\"object\":\"chat.completion\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"message\":{{\"role\":\"assistant\",\"content\":\"round13-ok\"}},\"finish_reason\":\"stop\"}}],\"usage\":{{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}}}'\n\
18430 \x20 content_type = 'application/json'\n\
18431 \x20 self.send_response(200); self.send_header('content-type', content_type); self.send_header('content-length', str(len(body))); self.end_headers(); self.wfile.write(body)\n\
18432 \x20 def log_message(self, *args): pass\n\
18433 http.server.HTTPServer(('127.0.0.1', port), H).serve_forever()\n",
18434 python.display(),
18435 spawned.display().to_string(),
18436 release.display().to_string(),
18437 ),
18438 )
18439 .unwrap();
18440 use std::os::unix::fs::PermissionsExt;
18441 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
18442
18443 let models_dir = root.path().join("models");
18444 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
18445 let mut schema = crate::registry::builtin_catalog()
18446 .into_iter()
18447 .find(ModelSchema::is_car_managed_vllm_mlx)
18448 .expect("managed vllm fixture schema");
18449 schema.id = "vllm-mlx/round12-singleflight".into();
18450 schema.name = "round12-singleflight".into();
18451 schema.cost.size_mb = Some(1);
18452 schema.cost.ram_mb = Some(1);
18453 schema.source = ModelSource::ManagedVllmMlx {
18454 hf_repo: "fixture/round12-singleflight".into(),
18455 hf_weight_file: None,
18456 };
18457 let model_dir = models_dir.join(&schema.name);
18458 std::fs::create_dir_all(&model_dir).unwrap();
18459 std::fs::write(model_dir.join("config.json"), b"{}").unwrap();
18460 std::fs::write(model_dir.join("model.safetensors"), b"fixture").unwrap();
18461 engine
18462 .unified_registry_mut()
18463 .register_project_model(schema.clone());
18464
18465 let coordinator = Arc::new(
18466 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
18467 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
18472 crate::hardware::HardwareInfo::detect(),
18473 Arc::new(FixedProbe),
18474 ),
18475 );
18476 engine.local_admission = coordinator.clone();
18477 engine.vllm_pool = Arc::new(crate::vllm_pool::VllmServerPool::with_test_runtime(
18478 std::time::Duration::from_secs(300),
18479 coordinator,
18480 script,
18481 ));
18482 let engine = Arc::new(engine);
18483 let model_id = schema.id.clone();
18484 let mut request = GenerateRequest {
18485 prompt: "hello from round 13".into(),
18486 model: Some(model_id.clone()),
18487 ..Default::default()
18488 };
18489 request.params.strict_model = true;
18494 let (estimated_input, _, _) = engine.estimated_tokens(&request, Some(&model_id));
18495 let expected_footprint = estimated_input.saturating_add(request.params.max_tokens);
18496 let expected_estimate = engine
18497 .local_model_preflight(&model_id, expected_footprint)
18498 .unwrap()
18499 .estimate;
18500 let expected_request_overhead_mb = expected_estimate
18501 .runtime_overhead_mb
18502 .saturating_add(expected_estimate.context_overhead_mb)
18503 .saturating_add(expected_estimate.transient_margin_mb);
18504
18505 async fn execute(
18506 engine: Arc<InferenceEngine>,
18507 request: GenerateRequest,
18508 streaming: bool,
18509 direct_vllm: bool,
18510 ) -> Result<String, InferenceError> {
18511 if direct_vllm {
18512 let model_id = request.model.as_deref().expect("explicit fixture model");
18513 let schema = engine
18514 .unified_registry()
18515 .get(model_id)
18516 .cloned()
18517 .expect("fixture schema");
18518 let (schema, _) = engine.vllm_live_schema(schema, None, 0).await?;
18519 return match schema.source {
18520 ModelSource::VllmMlx { endpoint, .. } => Ok(endpoint),
18521 source => Err(InferenceError::InferenceFailed(format!(
18522 "fixture did not resolve to a vllm endpoint: {source:?}"
18523 ))),
18524 };
18525 }
18526 if !streaming {
18527 return engine
18528 .generate_tracked(request)
18529 .await
18530 .map(|result| result.text);
18531 }
18532 let mut tracked = engine.generate_tracked_stream(request).await?;
18533 let mut accumulator = crate::stream::StreamAccumulator::default();
18534 while let Some(event) = tracked.events.recv().await {
18535 let done = matches!(event, crate::stream::StreamEvent::Done { .. });
18536 accumulator.push(&event);
18537 if done {
18538 break;
18539 }
18540 }
18541 Ok(accumulator.finish().0)
18542 }
18543
18544 let first_engine = engine.clone();
18545 let first_request = request.clone();
18546 let mut first = tokio::spawn(async move {
18547 execute(first_engine, first_request, streaming, cancel_first).await
18548 });
18549 tokio::time::timeout(std::time::Duration::from_secs(5), async {
18550 while !spawned.exists() {
18551 if first.is_finished() {
18552 let result = (&mut first).await;
18553 panic!("first dispatch ended before spawn: {result:?}");
18554 }
18555 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
18556 }
18557 })
18558 .await
18559 .expect("first request must spawn before publication");
18560 if !cancel_first {
18561 let during_startup = engine.local_model_preflight(&model_id, 0).unwrap();
18562 assert_eq!(
18563 during_startup.active_reservations_mb, expected_request_overhead_mb,
18564 "the cold-to-pending transfer must retain all request context/KV overhead"
18565 );
18566 assert_eq!(
18567 during_startup.resident_model_mb, expected_estimate.weights_mb,
18568 "the pending allocation must replace the exact cold-weight charge"
18569 );
18570 assert_eq!(
18571 during_startup
18572 .active_reservations_mb
18573 .saturating_add(during_startup.resident_model_mb),
18574 expected_estimate.estimated_peak_mb,
18575 "active request overhead plus pending weights must preserve the full admitted footprint"
18576 );
18577 }
18578
18579 let second_engine = engine.clone();
18580 let mut second =
18581 tokio::spawn(
18582 async move { execute(second_engine, request, streaming, cancel_first).await },
18583 );
18584 assert!(
18585 tokio::time::timeout(std::time::Duration::from_millis(100), &mut second)
18586 .await
18587 .is_err(),
18588 "request 2 must wait for request 1 to publish instead of failing on startup state"
18589 );
18590
18591 let first_result = if cancel_first {
18592 std::fs::remove_file(&spawned).unwrap();
18593 first.abort();
18594 let _ = first.await;
18595 tokio::time::timeout(std::time::Duration::from_secs(5), async {
18596 while !spawned.exists() {
18597 if second.is_finished() {
18598 let result = (&mut second).await;
18599 panic!("waiter ended before replacement spawn: {result:?}");
18600 }
18601 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
18602 }
18603 })
18604 .await
18605 .expect("waiter must continue after cancelled owner teardown");
18606 None
18607 } else {
18608 std::fs::write(&release, b"release").unwrap();
18609 Some(
18610 tokio::time::timeout(std::time::Duration::from_secs(5), first)
18611 .await
18612 .expect("first dispatch completes")
18613 .unwrap()
18614 .unwrap(),
18615 )
18616 };
18617 std::fs::write(&release, b"release").unwrap();
18618 let second_result = tokio::time::timeout(std::time::Duration::from_secs(5), second)
18619 .await
18620 .expect("second dispatch completes")
18621 .unwrap()
18622 .unwrap();
18623 let expected = if cancel_first {
18624 None
18625 } else if streaming {
18626 Some("round13-stream")
18627 } else {
18628 Some("round13-ok")
18629 };
18630 if let (Some(first_result), Some(expected)) = (first_result, expected) {
18631 assert_eq!(first_result, expected);
18632 }
18633 if let Some(expected) = expected {
18634 assert_eq!(second_result, expected);
18635 } else {
18636 assert!(second_result.starts_with("http://127.0.0.1:"));
18637 }
18638 assert_eq!(engine.vllm_pool.len().await, 1);
18639 assert!(engine
18640 .vllm_pool
18641 .release_model_if_present(&model_id)
18642 .await
18643 .unwrap());
18644 }
18645
18646 #[cfg(unix)]
18647 #[tokio::test]
18648 async fn concurrent_managed_vllm_generate_waits_before_outer_reservation() {
18649 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, false).await;
18650 }
18651
18652 #[cfg(unix)]
18653 #[tokio::test]
18654 async fn concurrent_managed_vllm_stream_waits_before_outer_reservation() {
18655 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(true, false).await;
18656 }
18657
18658 #[cfg(unix)]
18659 #[tokio::test]
18660 async fn managed_vllm_waiter_continues_after_startup_owner_cancellation() {
18661 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, true).await;
18662 }
18663
18664 #[tokio::test]
18665 async fn same_state_root_engines_share_runtime_components() {
18666 const CHILD_ENV: &str = "CAR_TWO_ENGINE_RUNTIME_TEST_CHILD";
18667 if std::env::var_os(CHILD_ENV).is_some() {
18668 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
18669 return;
18670 }
18671 let root = tempfile::tempdir().unwrap();
18672 let first = InferenceEngine::new(test_config(root.path().join("weights")));
18673 let second = InferenceEngine::new(test_config(root.path().join("weights")));
18674
18675 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
18676 assert!(Arc::ptr_eq(&first.vllm_pool, &second.vllm_pool));
18677 assert!(Arc::ptr_eq(
18678 &first.resource_policy_generation,
18679 &second.resource_policy_generation
18680 ));
18681 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
18682 {
18683 assert!(Arc::ptr_eq(&first.text_backends, &second.text_backends));
18684 assert!(Arc::ptr_eq(&first.local_backends, &second.local_backends));
18685 }
18686 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18687 {
18688 assert!(Arc::ptr_eq(&first.backend, &second.backend));
18689 assert!(Arc::ptr_eq(
18690 &first.embedding_backend,
18691 &second.embedding_backend
18692 ));
18693 }
18694
18695 let loads = Arc::new(std::sync::atomic::AtomicU64::new(0));
18696 let first_cache = first._runtime_scope.load_probe.clone();
18697 let second_cache = second._runtime_scope.load_probe.clone();
18698 let mut threads = Vec::new();
18699 for cache in [first_cache, second_cache] {
18700 let loads = loads.clone();
18701 threads.push(std::thread::spawn(move || {
18702 cache
18703 .get_or_load::<()>("same/model", 1, || {
18704 loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
18705 std::thread::sleep(std::time::Duration::from_millis(20));
18706 Ok(())
18707 })
18708 .unwrap()
18709 }));
18710 }
18711 let handles = threads
18712 .into_iter()
18713 .map(|thread| thread.join().unwrap())
18714 .collect::<Vec<_>>();
18715 assert_eq!(loads.load(std::sync::atomic::Ordering::SeqCst), 1);
18716 assert!(Arc::ptr_eq(&handles[0], &handles[1]));
18717
18718 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
18719 .arg("--exact")
18720 .arg("tests::same_state_root_engines_share_runtime_components")
18721 .env(CHILD_ENV, "1")
18722 .stdin(std::process::Stdio::null())
18723 .stdout(std::process::Stdio::null())
18724 .stderr(std::process::Stdio::null())
18725 .kill_on_drop(true)
18726 .spawn()
18727 .unwrap();
18728 first
18729 .vllm_pool
18730 .insert_test_process("vllm-mlx/two-engine", child)
18731 .await;
18732 first.local_admission.mark_resident_allocation(
18733 "vllm-mlx/two-engine",
18734 &resource_policy::vllm_process_allocation_id("vllm-mlx/two-engine"),
18735 1,
18736 );
18737 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
18738 drop(first);
18739 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
18740 assert!(second.local_admission.is_resident("vllm-mlx/two-engine"));
18741 assert!(second.vllm_pool.evict_model("vllm-mlx/two-engine").await);
18742 assert!(!second.local_admission.is_resident("vllm-mlx/two-engine"));
18743 }
18744
18745 #[tokio::test]
18746 async fn last_engine_drop_reaps_vllm_before_new_runtime_admission() {
18747 const CHILD_ENV: &str = "CAR_LAST_ENGINE_VLLM_DROP_TEST_CHILD";
18748 if std::env::var_os(CHILD_ENV).is_some() {
18749 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
18750 return;
18751 }
18752 let root = tempfile::tempdir().unwrap();
18753 let config = test_config(root.path().join("weights"));
18754 let engine = InferenceEngine::new(config.clone());
18755 let coordinator = engine.local_admission.clone();
18756 let schema = engine
18757 .unified_registry()
18758 .all()
18759 .find(|schema| schema.is_vllm_mlx())
18760 .cloned()
18761 .expect("supervised vllm schema");
18762 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
18763 .arg("--exact")
18764 .arg("tests::last_engine_drop_reaps_vllm_before_new_runtime_admission")
18765 .env(CHILD_ENV, "1")
18766 .stdin(std::process::Stdio::null())
18767 .stdout(std::process::Stdio::null())
18768 .stderr(std::process::Stdio::null())
18769 .kill_on_drop(true)
18770 .spawn()
18771 .unwrap();
18772 engine
18773 .vllm_pool
18774 .insert_test_process(&schema.id, child)
18775 .await;
18776 coordinator.mark_resident_allocation(
18777 &schema.id,
18778 &resource_policy::vllm_process_allocation_id(&schema.id),
18779 1,
18780 );
18781
18782 drop(engine);
18783
18784 tokio::time::timeout(std::time::Duration::from_secs(2), async {
18785 while coordinator.teardown_pending(&schema.id) {
18786 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
18787 }
18788 })
18789 .await
18790 .expect("last runtime drop must eventually confirm supervised child exit");
18791 assert!(!coordinator.is_resident(&schema.id));
18792 let next = InferenceEngine::new(config);
18793 assert!(Arc::ptr_eq(&coordinator, &next.local_admission));
18794 let preflight = next.local_model_preflight(&schema.id, 0).unwrap();
18795 assert_eq!(preflight.resident_model_mb, 0);
18796 assert!(preflight.estimated_incremental_mb >= preflight.estimate.weights_mb);
18797 }
18798
18799 #[cfg(unix)]
18800 #[test]
18801 fn symlinked_state_roots_share_exact_coordinator_and_runtime() {
18802 use std::os::unix::fs::symlink;
18803
18804 let fixture = tempfile::tempdir().unwrap();
18805 let real = fixture.path().join("real-state");
18806 std::fs::create_dir(&real).unwrap();
18807 let alias = fixture.path().join("state-alias");
18808 symlink(&real, &alias).unwrap();
18809
18810 let first = InferenceEngine::new(test_config(real.join("weights")));
18811 let second = InferenceEngine::new(test_config(alias.join("weights")));
18812 assert!(Arc::ptr_eq(&first.local_admission, &second.local_admission));
18813 assert!(Arc::ptr_eq(&first._runtime_scope, &second._runtime_scope));
18814 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
18815 }
18816
18817 #[tokio::test]
18818 async fn stream_reservation_lives_until_returned_receiver_is_released() {
18819 let root = tempfile::tempdir().unwrap();
18820 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18821 let schema = crate::registry::builtin_catalog()
18822 .into_iter()
18823 .find(|schema| schema.is_local() && !schema.is_vllm_mlx())
18824 .expect("local model schema");
18825 engine.local_admission.mark_resident(&schema.id, 1);
18826 let reservation = engine.reserve_local_request(&schema, 64).unwrap();
18827 assert_eq!(engine.local_admission.active_request_count(&schema.id), 1);
18828 let (source_tx, source_rx) = tokio::sync::mpsc::channel(1);
18829 let returned =
18830 InferenceEngine::hold_optional_reservation_for_stream(source_rx, Some(reservation));
18831
18832 drop(returned);
18833 source_tx
18834 .send(stream::StreamEvent::TextDelta("release".into()))
18835 .await
18836 .unwrap();
18837 for _ in 0..20 {
18838 if engine.local_admission.active_request_count(&schema.id) == 0 {
18839 break;
18840 }
18841 tokio::task::yield_now().await;
18842 }
18843 assert_eq!(engine.local_admission.active_request_count(&schema.id), 0);
18844
18845 let compact_source = include_str!("lib.rs")
18846 .split_whitespace()
18847 .collect::<String>();
18848 let remote_handoff = [
18849 "Self::hold_optional_reservation_for_stream",
18850 "(receiver,",
18851 "candidate_reservation.take(),",
18852 ");",
18853 ]
18854 .concat();
18855 assert!(compact_source.contains(&remote_handoff));
18856 }
18857
18858 #[tokio::test]
18859 async fn residual_voice_allocation_blocks_removal_with_typed_error() {
18860 let root = tempfile::tempdir().unwrap();
18861 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18862 engine
18863 .local_admission
18864 .mark_resident("voice/removal-fixture", 1);
18865
18866 let error = match engine
18867 .prepare_local_model_removal("voice/removal-fixture")
18868 .await
18869 {
18870 Err(error) => error,
18871 Ok(_) => panic!("voice allocation must be released by its owner first"),
18872 };
18873 assert!(matches!(
18874 error,
18875 crate::resource_policy::ModelMaintenanceError::ResidualResidency {
18876 model_id,
18877 allocation_ids,
18878 } if model_id == "voice/removal-fixture"
18879 && allocation_ids == vec!["voice/removal-fixture"]
18880 ));
18881 }
18882
18883 #[tokio::test]
18884 async fn catalog_voice_id_blocks_removal_of_live_provider_alias() {
18885 let root = tempfile::tempdir().unwrap();
18886 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18887 engine.local_admission.register_model_aliases(
18888 "voice/parakeet-tdt-0.6b",
18889 ["mlx/parakeet-tdt-0.6b-v3:default"],
18890 );
18891 let mut reservation = engine
18892 .local_admission
18893 .reserve_measured_host_allocation(
18894 "voice/parakeet-tdt-0.6b",
18895 "voice/parakeet-tdt-0.6b#voice-allocation-0",
18896 1024 * 1024,
18897 0,
18898 )
18899 .unwrap();
18900 reservation.publish_resident_weights(1024 * 1024);
18901 drop(reservation);
18902
18903 let result = engine
18904 .prepare_local_model_removal("mlx/parakeet-tdt-0.6b-v3:default")
18905 .await;
18906 assert!(matches!(
18907 result,
18908 Err(crate::resource_policy::ModelMaintenanceError::ResidualResidency { .. })
18909 ));
18910 }
18911
18912 struct RefusingRemovalOffload;
18913
18914 #[async_trait::async_trait]
18915 impl crate::offload::LocalGenerationOffload for RefusingRemovalOffload {
18916 async fn generate(
18917 &self,
18918 _request: GenerateRequest,
18919 ) -> Result<InferenceResult, InferenceError> {
18920 unreachable!("removal test does not generate")
18921 }
18922
18923 async fn stream(
18924 &self,
18925 _request: GenerateRequest,
18926 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
18927 unreachable!("removal test does not stream")
18928 }
18929
18930 async fn resident_models(&self) -> Vec<String> {
18931 vec!["worker/refuses-release".into()]
18932 }
18933
18934 async fn release_model(&self, _model_id: &str) -> Result<bool, InferenceError> {
18935 Ok(false)
18936 }
18937 }
18938
18939 #[tokio::test]
18940 async fn worker_release_boolean_is_required_for_model_removal() {
18941 let _offload_guard = crate::offload::test_offload_lock().lock().await;
18942 crate::offload::set_local_offload(Some(Arc::new(RefusingRemovalOffload)));
18943 let root = tempfile::tempdir().unwrap();
18944 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18945 let result = engine
18946 .prepare_local_model_removal("worker/refuses-release")
18947 .await;
18948 crate::offload::set_local_offload(None);
18949
18950 assert!(matches!(
18951 result,
18952 Err(crate::resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
18953 model_id
18954 )) if model_id == "worker/refuses-release"
18955 ));
18956 }
18957
18958 #[test]
18959 fn transient_mlx_vlm_allocation_never_becomes_resident() {
18960 struct FixedProbe;
18961 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
18962 fn available_memory_mb(
18963 &self,
18964 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
18965 Ok(Some(24_000))
18966 }
18967 }
18968
18969 let root = tempfile::tempdir().unwrap();
18970 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
18971 engine.local_admission = Arc::new(
18972 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
18973 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
18974 crate::hardware::HardwareInfo {
18975 total_ram_mb: 32 * 1024,
18976 ..crate::hardware::HardwareInfo::detect()
18977 },
18978 Arc::new(FixedProbe),
18979 ),
18980 );
18981 let mut schema = crate::registry::builtin_catalog()
18982 .into_iter()
18983 .find(|schema| schema.tags.iter().any(|tag| tag == "mlx-vlm-cli"))
18984 .expect("one-shot mlx-vlm schema");
18985 schema.param_count = "1M".into();
18986 schema.quantization = Some(Quantization::parse("Q4"));
18987 let mut reservation = engine.reserve_local_request(&schema, 64).unwrap();
18988 InferenceEngine::reconcile_transient_local_allocation(&mut reservation, 1024 * 1024)
18989 .unwrap();
18990 drop(reservation);
18991 assert!(!engine.local_admission.is_resident(&schema.id));
18992 }
18993
18994 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18995 #[test]
18996 fn non_apple_backend_map_keeps_two_local_model_identities_exact() {
18997 let mut backends = std::collections::HashMap::<String, u32>::new();
18998 backends.insert("local/model-a".into(), 1);
18999 backends.insert("local/model-b".into(), 2);
19000 assert_eq!(backends.get("local/model-a"), Some(&1));
19001 assert_eq!(backends.get("local/model-b"), Some(&2));
19002 assert_eq!(backends.remove("local/model-a"), Some(1));
19003 assert!(!backends.contains_key("local/model-a"));
19004 assert_eq!(backends.get("local/model-b"), Some(&2));
19005 }
19006
19007 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
19008 #[tokio::test]
19009 async fn non_apple_remote_classification_at_zero_gb_routes_without_local_allocation() {
19010 use wiremock::matchers::{method, path};
19011 use wiremock::{Mock, MockServer, ResponseTemplate};
19012
19013 let _credential_scope = crate::openrouter::test_credential_scope();
19014 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
19015 let server = MockServer::start().await;
19016 Mock::given(method("POST"))
19017 .and(path("/v1/chat/completions"))
19018 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
19019 "choices": [{
19020 "message": {"role": "assistant", "content": "positive"},
19021 "finish_reason": "stop"
19022 }],
19023 "usage": {"prompt_tokens": 16, "completion_tokens": 1}
19024 })))
19025 .mount(&server)
19026 .await;
19027
19028 let fixture = tempfile::tempdir().unwrap();
19029 let config = test_config(fixture.path().join("weights"));
19030 let repository =
19031 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19032 crate::resource_policy::ResourcePolicyRepository::save(
19033 &repository,
19034 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19035 )
19036 .unwrap();
19037 let mut engine = InferenceEngine::new(config);
19038 let mut remote = crate::openrouter::curated_schemas()
19039 .into_iter()
19040 .next()
19041 .expect("OpenRouter schema");
19042 remote.id = "openrouter/test/classifier".into();
19043 remote.name = remote.id.clone();
19044 if let ModelSource::RemoteApi { endpoint, .. } = &mut remote.source {
19045 *endpoint = server.uri();
19046 }
19047 let remote_id = remote.id.clone();
19048 engine.unified_registry_mut().register_project_model(remote);
19049
19050 let result = engine
19051 .classify(ClassifyRequest {
19052 work_context: None,
19053 text: "a good outcome".into(),
19054 labels: vec!["positive".into(), "negative".into()],
19055 model: Some(remote_id),
19056 })
19057 .await
19058 .unwrap();
19059 assert_eq!(
19060 result.first().map(|item| item.label.as_str()),
19061 Some("positive")
19062 );
19063 assert_eq!(engine.local_admission.resident_model_mb(), 0);
19064 assert!(
19065 !engine.config.models_dir.exists()
19066 || std::fs::read_dir(&engine.config.models_dir)
19067 .unwrap()
19068 .next()
19069 .is_none(),
19070 "remote classification must not download local weights"
19071 );
19072 }
19073
19074 #[test]
19075 fn adaptive_speech_skips_blocked_local_but_explicit_speech_fails() {
19076 let fixture = TempDir::new().unwrap();
19077 let config = InferenceConfig {
19078 models_dir: fixture.path().join("weights"),
19079 state_root: fixture.path().join("state"),
19080 ..Default::default()
19081 };
19082 let repository =
19083 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19084 crate::resource_policy::ResourcePolicyRepository::save(
19085 &repository,
19086 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19087 )
19088 .unwrap();
19089 let engine = InferenceEngine::new(config);
19090 let local = engine
19091 .unified_registry()
19092 .all()
19093 .find(|schema| {
19094 schema.is_local() && schema.has_capability(ModelCapability::SpeechToText)
19095 })
19096 .cloned()
19097 .expect("built-in local STT model");
19098
19099 assert!(matches!(
19100 engine.admit_speech_candidate(&local, false),
19101 SpeechCandidateAdmission::SkipBlocked(_)
19102 ));
19103 assert!(matches!(
19104 engine.admit_speech_candidate(&local, true),
19105 SpeechCandidateAdmission::FailBlocked(InferenceError::LocalResourceBlocked { .. })
19106 ));
19107
19108 let mut os_owned = local;
19109 os_owned.id = "windows/speech-synthesis:test".into();
19110 os_owned.source = ModelSource::WindowsSpeech {};
19111 assert!(matches!(
19112 engine.admit_speech_candidate(&os_owned, true),
19113 SpeechCandidateAdmission::Proceed(None)
19114 ));
19115 }
19116
19117 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19118 async fn cancelled_native_blocking_work_keeps_its_exact_charge_until_completion() {
19119 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
19120 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
19121 crate::hardware::HardwareInfo {
19122 os: "test".into(),
19123 arch: "test".into(),
19124 cpu_cores: 8,
19125 total_ram_mb: 32 * 1024,
19126 gpu_backend: crate::hardware::GpuBackend::Cpu,
19127 gpu_memory_mb: None,
19128 gpu_devices: Vec::new(),
19129 recommended_model: "fixture".into(),
19130 recommended_context: 4096,
19131 max_model_mb: 32 * 1024,
19132 },
19133 ));
19134 let reservation = coordinator
19135 .reserve_measured_host("detached-native-a", 512 * 1024 * 1024, 0)
19136 .unwrap();
19137 let lease = reservation.detached_lease();
19138 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
19139 let release =
19140 std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
19141 let release_worker = release.clone();
19142 let operation = tokio::spawn(async move {
19143 let _request_reservation = reservation;
19144 run_admitted_blocking(lease, move || {
19145 let _ = started_tx.send(());
19146 let (lock, ready) = &*release_worker;
19147 let mut released = lock
19148 .lock()
19149 .unwrap_or_else(std::sync::PoisonError::into_inner);
19150 while !*released {
19151 released = ready
19152 .wait(released)
19153 .unwrap_or_else(std::sync::PoisonError::into_inner);
19154 }
19155 })
19156 .await
19157 });
19158 started_rx.await.unwrap();
19159 operation.abort();
19160 let _ = operation.await;
19161
19162 let blocked = coordinator.reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0);
19163 assert!(
19164 blocked.is_err(),
19165 "cancelling the await must not advertise memory still owned by spawn_blocking"
19166 );
19167
19168 let (lock, ready) = &*release;
19169 *lock
19170 .lock()
19171 .unwrap_or_else(std::sync::PoisonError::into_inner) = true;
19172 ready.notify_one();
19173 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19174 loop {
19175 if coordinator
19176 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
19177 .is_ok()
19178 {
19179 break;
19180 }
19181 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19182 }
19183 })
19184 .await
19185 .expect("the detached charge must clear after native work really exits");
19186 }
19187
19188 #[test]
19189 fn nonstream_native_text_generation_uses_cancellable_blocking_boundary() {
19190 let source = include_str!("lib.rs");
19191 let mlx = source
19192 .split("async fn generate_mlx(")
19193 .nth(1)
19194 .and_then(|tail| tail.split("async fn generate_local(").next())
19195 .expect("generate_mlx source");
19196 let local = source
19197 .split("async fn generate_local(")
19198 .nth(1)
19199 .and_then(|tail| tail.split("fn apply_top_k_top_p").next())
19200 .expect("generate_local source");
19201
19202 assert!(
19203 mlx.contains("run_admitted_blocking"),
19204 "native MLX decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
19205 );
19206 assert!(
19207 local.contains("run_admitted_blocking"),
19208 "polymorphic local decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
19209 );
19210 }
19211
19212 #[cfg(unix)]
19213 #[tokio::test]
19214 async fn cancelled_speech_subprocess_is_killed_and_reaped_before_charge_clears() {
19215 use std::os::unix::fs::PermissionsExt;
19216
19217 let fixture = TempDir::new().unwrap();
19218 let runtime = SpeechRuntime::new(fixture.path().join("speech-runtime"));
19219 std::fs::create_dir_all(runtime.stt_program.parent().unwrap()).unwrap();
19220 std::fs::write(
19221 &runtime.stt_program,
19222 b"#!/bin/sh\necho started > \"$1\"\nsleep 1\necho continued > \"$2\"\n",
19223 )
19224 .unwrap();
19225 let mut permissions = std::fs::metadata(&runtime.stt_program)
19226 .unwrap()
19227 .permissions();
19228 permissions.set_mode(0o755);
19229 std::fs::set_permissions(&runtime.stt_program, permissions).unwrap();
19230
19231 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
19232 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
19233 crate::hardware::HardwareInfo {
19234 total_ram_mb: 32 * 1024,
19235 ..crate::hardware::HardwareInfo::detect()
19236 },
19237 ));
19238 let reservation = coordinator
19239 .reserve_measured_host("mlx-audio-a", 512 * 1024 * 1024, 0)
19240 .unwrap();
19241 let lease = reservation.detached_lease();
19242 let started = fixture.path().join("started");
19243 let continued = fixture.path().join("continued");
19244 let args = vec![
19245 started.display().to_string(),
19246 continued.display().to_string(),
19247 ];
19248 let command = tokio::spawn(async move {
19249 let _request_reservation = reservation;
19250 run_mlx_audio_command(&runtime, "stt.generate", &args, lease).await
19251 });
19252 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19253 while !started.exists() {
19254 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19255 }
19256 })
19257 .await
19258 .expect("speech fixture child must start");
19259 command.abort();
19260 let _ = command.await;
19261
19262 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19263 loop {
19264 if coordinator
19265 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
19266 .is_ok()
19267 {
19268 break;
19269 }
19270 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19271 }
19272 })
19273 .await
19274 .expect("charge clears only after the cancelled child is reaped");
19275 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
19276 assert!(
19277 !continued.exists(),
19278 "the cancelled local speech process must not keep allocating in the background"
19279 );
19280 }
19281
19282 #[test]
19283 fn local_model_eviction_surface_includes_every_in_process_cache() {
19284 let source = include_str!("lib.rs");
19285 for cache in [
19286 "self.text_backends.evict_if_idle(model_id)",
19287 "self.local_backends.evict_if_idle(model_id)",
19288 "self.flux_cache.evict_if_idle(model_id)",
19289 "self.ltx_cache.evict_if_idle(model_id)",
19290 "self.kokoro_cache.evict_if_idle(model_id)",
19291 "self.local_backends.evict_idle()",
19292 ] {
19293 assert!(source.contains(cache), "missing eviction seam: {cache}");
19294 }
19295 }
19296
19297 #[tokio::test]
19303 async fn catalog_refresh_uses_authority_hints_without_secret_reads() {
19304 if !crate::run_in_isolated_test_process(
19305 "tests::catalog_refresh_uses_authority_hints_without_secret_reads",
19306 "CAR_CATALOG_ZERO_SECRET_READ_CHILD",
19307 ) {
19308 return;
19309 }
19310 let _environment = ENV_MUTEX.lock().await;
19311 let fixture = TempDir::new().unwrap();
19312 let credential_envs = [
19313 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19314 car_auth::PARSLEE_API_BASE_KEY,
19315 crate::openrouter::API_KEY_ENV,
19316 "OPENAI_API_KEY",
19317 "ANTHROPIC_API_KEY",
19318 "GOOGLE_API_KEY",
19319 "ELEVENLABS_API_KEY",
19320 ];
19321 let mut restored_names = vec![car_home::ENV_VAR, "CAR_SECRETS_FILE_DIR"];
19322 restored_names.extend(credential_envs);
19323 let _restore = RestoredEnvironment::capture(&restored_names);
19324 unsafe {
19325 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
19326 std::env::set_var(
19327 "CAR_SECRETS_FILE_DIR",
19328 fixture.path().join("isolated-secrets"),
19329 );
19330 for name in credential_envs {
19331 std::env::remove_var(name);
19332 }
19333 }
19334
19335 let before = car_secrets::secret_store_activity();
19336 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
19337
19338 let listed = engine.list_models_unified();
19340 let schemas = engine.list_schemas();
19341 for provider in ["parslee", "openai", "anthropic", "google", "elevenlabs"] {
19342 assert!(
19343 schemas.iter().any(|schema| schema.provider == provider),
19344 "real builtin catalog lost the credential-bearing {provider} rows"
19345 );
19346 }
19347 assert!(
19348 schemas.iter().any(|schema| matches!(
19349 schema.source,
19350 ModelSource::RemoteApi {
19351 protocol: ApiProtocol::OpenRouter,
19352 ..
19353 }
19354 )),
19355 "real builtin catalog lost the reviewed OpenRouter rows"
19356 );
19357 assert_eq!(listed.len(), schemas.len());
19358
19359 let schema_refs: Vec<&ModelSchema> = schemas.iter().collect();
19361 let _setup_plan = crate::recommend(
19362 &schema_refs,
19363 &HardwareInfo::detect(),
19364 UseCase::default(),
19365 QualityTier::default(),
19366 Privacy::OnDevice,
19367 );
19368
19369 let _speech = engine.speech_health();
19371 let _concierge = engine.concierge_status(false).await;
19372 let _health = engine.model_health().await;
19373
19374 let after = car_secrets::secret_store_activity();
19375 assert_eq!(
19376 after, before,
19377 "passive builtin catalog surfaces performed secret-store operations"
19378 );
19379 }
19380
19381 #[tokio::test]
19382 async fn list_models_unified_and_model_health_are_zero_secret_store_probes() {
19383 if !crate::run_in_isolated_test_process(
19384 "tests::list_models_unified_and_model_health_are_zero_secret_store_probes",
19385 "CAR_DIRECT_MODEL_SURFACES_ZERO_SECRET_CHILD",
19386 ) {
19387 return;
19388 }
19389 let _environment = ENV_MUTEX.lock().await;
19390 let fixture = TempDir::new().unwrap();
19391 let _restore = RestoredEnvironment::capture(&[
19392 car_home::ENV_VAR,
19393 "CAR_SECRETS_FILE_DIR",
19394 crate::openrouter::API_KEY_ENV,
19395 ]);
19396 unsafe {
19397 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
19398 std::env::set_var(
19399 "CAR_SECRETS_FILE_DIR",
19400 fixture.path().join("isolated-secrets"),
19401 );
19402 std::env::remove_var(crate::openrouter::API_KEY_ENV);
19403 }
19404 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
19405 let before = car_secrets::secret_store_activity();
19406
19407 let rows = engine.list_models_unified();
19408 let health = engine.model_health().await;
19409
19410 assert!(!rows.is_empty(), "the unified catalog fixture must exist");
19411 assert!(health.total_models > 0, "the health fixture must exist");
19412 let after = car_secrets::secret_store_activity();
19413 assert_eq!(after.status_attempts, before.status_attempts);
19414 assert_eq!(after.get_attempts, before.get_attempts);
19415
19416 assert!(
19426 rows.iter().all(|row| row.credential_required.is_none()),
19427 "a passive catalog read must not populate `credential_required` — its \
19428 availability verdict cannot see the secret store, so any credential it \
19429 named might already be present: {:?}",
19430 rows.iter()
19431 .filter(|row| row.credential_required.is_some())
19432 .map(|row| (&row.id, &row.credential_required))
19433 .collect::<Vec<_>>()
19434 );
19435 }
19436
19437 #[test]
19447 fn car_home_moves_the_catalog_and_discovery_caches_but_never_the_weights() {
19448 let _environment = crate::openrouter::test_environment_scope();
19449 let prior = std::env::var_os(car_home::ENV_VAR);
19450
19451 unsafe { std::env::remove_var(car_home::ENV_VAR) };
19452 let shared = InferenceConfig::default();
19453 let default_catalog = crate::catalog::cache_path(&shared.state_root);
19454 let default_discovery = crate::discovery::cache_path(&shared.state_models_dir());
19455
19456 let alt = Path::new("/tmp/car-home-inference-cache-test");
19457 unsafe { std::env::set_var(car_home::ENV_VAR, alt) };
19458 let relocated = InferenceConfig::default();
19459 let catalog = crate::catalog::cache_path(&relocated.state_root);
19460 let discovery = crate::discovery::cache_path(&relocated.state_models_dir());
19461
19462 match prior {
19463 Some(value) => unsafe { std::env::set_var(car_home::ENV_VAR, value) },
19464 None => unsafe { std::env::remove_var(car_home::ENV_VAR) },
19465 }
19466
19467 assert_eq!(catalog, alt.join(crate::catalog::CATALOG_CACHE_FILE));
19468 assert_eq!(
19469 discovery,
19470 alt.join("models")
19471 .join(crate::discovery::DISCOVERED_MODELS_FILE)
19472 );
19473 assert_ne!(
19474 catalog, default_catalog,
19475 "the catalog cache must not resolve back into the shared root",
19476 );
19477 assert_ne!(
19478 discovery, default_discovery,
19479 "the discovery cache must not resolve back into the shared root",
19480 );
19481
19482 assert_eq!(
19483 relocated.models_dir, shared.models_dir,
19484 "the weights cache is machine-global and must not follow CAR_HOME",
19485 );
19486 assert!(
19487 !relocated.models_dir.starts_with(alt),
19488 "the weights cache must not be dragged under the override",
19489 );
19490 }
19491
19492 #[derive(Clone, Copy)]
19493 enum CacheRoutingSurface {
19494 Generate,
19495 Stream,
19496 }
19497
19498 async fn invoke_cache_routed_openrouter(
19504 surface: CacheRoutingSurface,
19505 cache_read_estimate: usize,
19506 cache_write_estimate: usize,
19507 ) -> String {
19508 use wiremock::matchers::{method, path};
19509 use wiremock::{Mock, MockServer, ResponseTemplate};
19510
19511 let _credential_scope = crate::openrouter::test_credential_scope();
19512 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
19513 let server = MockServer::start().await;
19514 let response = match surface {
19515 CacheRoutingSurface::Generate => ResponseTemplate::new(200).set_body_json(
19516 serde_json::json!({
19517 "choices": [{
19518 "message": {"role": "assistant", "content": "cache-route-ok"},
19519 "finish_reason": "stop"
19520 }],
19521 "usage": {"prompt_tokens": 40_000, "completion_tokens": 8}
19522 }),
19523 ),
19524 CacheRoutingSurface::Stream => ResponseTemplate::new(200).set_body_raw(
19525 concat!(
19526 "data: {\"choices\":[{\"delta\":{\"content\":\"cache-route-ok\"},\"finish_reason\":\"stop\"}]}\n\n",
19527 "data: [DONE]\n\n"
19528 ),
19529 "text/event-stream",
19530 ),
19531 };
19532 Mock::given(method("POST"))
19533 .and(path("/v1/chat/completions"))
19534 .respond_with(response)
19535 .mount(&server)
19536 .await;
19537
19538 let tmp = TempDir::new().unwrap();
19539 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19540 let mut uncached_cheap = crate::openrouter::curated_schemas()
19541 .into_iter()
19542 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
19543 .unwrap();
19544 uncached_cheap.id = "openrouter/test/uncached-cheap".into();
19545 uncached_cheap.name = uncached_cheap.id.clone();
19546 uncached_cheap.cost = CostModel {
19547 input_per_mtok: Some(1.0),
19548 output_per_mtok: Some(1.0),
19549 cache_read_input_per_mtok: Some(100.0),
19550 cache_write_input_per_mtok: Some(100.0),
19551 ..Default::default()
19552 };
19553 if let ModelSource::RemoteApi { endpoint, .. } = &mut uncached_cheap.source {
19554 *endpoint = server.uri();
19555 }
19556
19557 let mut cached_cheap = uncached_cheap.clone();
19558 cached_cheap.id = "openrouter/test/cached-cheap".into();
19559 cached_cheap.name = cached_cheap.id.clone();
19560 cached_cheap.cost = CostModel {
19561 input_per_mtok: Some(80.0),
19562 output_per_mtok: Some(1.0),
19563 cache_read_input_per_mtok: Some(0.001),
19564 cache_write_input_per_mtok: Some(0.001),
19565 ..Default::default()
19566 };
19567
19568 let uncached_id = uncached_cheap.id.clone();
19569 let cached_id = cached_cheap.id.clone();
19570 engine
19571 .unified_registry_mut()
19572 .register_project_model(uncached_cheap);
19573 engine
19574 .unified_registry_mut()
19575 .register_project_model(cached_cheap);
19576 let exclude_models = engine
19577 .list_schemas()
19578 .into_iter()
19579 .map(|schema| schema.id)
19580 .filter(|id| id != &uncached_id && id != &cached_id)
19581 .collect();
19582
19583 let mut params = GenerateParams {
19587 max_tokens: 8,
19588 ..Default::default()
19589 };
19590 assert_eq!(params.estimated_cache_read_input_tokens, 0);
19591 assert_eq!(params.estimated_cache_write_input_tokens, 0);
19592 params.estimated_cache_read_input_tokens = cache_read_estimate;
19593 params.estimated_cache_write_input_tokens = cache_write_estimate;
19594 let req = GenerateRequest {
19595 prompt: "x".repeat(160_000),
19596 params,
19597 cache_control: true,
19598 intent: Some(IntentHint {
19599 prefer_quality: true,
19600 exclude_models,
19601 ..Default::default()
19602 }),
19603 ..Default::default()
19604 };
19605
19606 match surface {
19607 CacheRoutingSurface::Generate => {
19608 engine
19609 .generate_tracked(req)
19610 .await
19611 .expect("mocked OpenRouter generation should succeed")
19612 .model_used
19613 }
19614 CacheRoutingSurface::Stream => {
19615 let mut handle = engine
19616 .generate_tracked_stream(req)
19617 .await
19618 .expect("mocked OpenRouter stream should start");
19619 let selected = handle.model_used.clone();
19620 while handle.events.recv().await.is_some() {}
19621 selected
19622 }
19623 }
19624 }
19625
19626 #[tokio::test(flavor = "current_thread")]
19627 async fn tracked_generate_uses_explicit_cache_estimate_and_defaults_to_zero() {
19628 let without_estimate =
19629 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 0, 0).await;
19630 let with_estimate =
19631 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 40_000, 0).await;
19632 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
19633 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
19634 }
19635
19636 #[tokio::test(flavor = "current_thread")]
19637 async fn tracked_stream_uses_explicit_cache_estimate_and_defaults_to_zero() {
19638 let without_estimate =
19639 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 0).await;
19640 let with_estimate =
19641 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 40_000).await;
19642 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
19643 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
19644 }
19645
19646 #[tokio::test(flavor = "current_thread")]
19647 async fn authenticated_openrouter_registry_stays_static_and_rejects_unknown_ids() {
19648 let _credential_scope = crate::openrouter::test_credential_scope();
19649 crate::openrouter::set_test_credential(Some("static-key"));
19650 let tmp = TempDir::new().unwrap();
19651 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19652 let personal: Vec<_> = engine
19653 .list_schemas()
19654 .into_iter()
19655 .filter(|schema| schema.id.starts_with("openrouter/"))
19656 .collect();
19657 assert_eq!(personal.len(), crate::openrouter::curated_model_count());
19658 assert!(personal.iter().all(ModelSchema::available_now));
19659 assert!(personal
19660 .iter()
19661 .all(|schema| schema.trust_tier == TrustTier::Curated));
19662 assert!(personal
19663 .iter()
19664 .all(|schema| !schema.tags.iter().any(|tag| tag == "dynamic")));
19665
19666 for unknown in [
19667 "openrouter/vendor/brand-new-model",
19668 "openrouter/openai/gpt-5.4-typo",
19669 ] {
19670 assert!(engine
19671 .list_schemas()
19672 .iter()
19673 .all(|schema| schema.id != unknown));
19674 assert_eq!(engine.model_context_window(unknown), 0);
19675 let error = engine
19676 .generate_tracked(GenerateRequest {
19677 prompt: "must fail before transport".into(),
19678 model: Some(unknown.into()),
19679 params: GenerateParams {
19680 strict_model: true,
19681 ..Default::default()
19682 },
19683 ..Default::default()
19684 })
19685 .await
19686 .expect_err("unregistered personal OpenRouter ids must not reach inference");
19687 assert!(
19688 matches!(&error, InferenceError::ModelNotFound(id) if id == unknown),
19689 "{unknown}: {error}"
19690 );
19691 let stream_error = match engine
19692 .generate_tracked_stream(GenerateRequest {
19693 prompt: "must fail before stream transport".into(),
19694 model: Some(unknown.into()),
19695 ..Default::default()
19696 })
19697 .await
19698 {
19699 Ok(_) => panic!("unregistered ids must also fail before streaming"),
19700 Err(error) => error,
19701 };
19702 assert!(
19703 matches!(&stream_error, InferenceError::ModelNotFound(id) if id == unknown),
19704 "{unknown}: {stream_error}"
19705 );
19706 }
19707 }
19708
19709 #[tokio::test(flavor = "current_thread")]
19710 async fn static_openrouter_rows_participate_in_adaptive_routing_only_with_a_key() {
19711 let (_credential_scope, _provider_env) =
19712 crate::openrouter::test_credential_and_environment_scope_async().await;
19713 let tmp = TempDir::new().unwrap();
19714 crate::openrouter::set_test_credential(Some("static-key"));
19715 unsafe {
19716 std::env::set_var("CAR_STATIC_ROUTING_PEER_KEY", "peer-key");
19717 }
19718 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19719 let mut peer = remote_stream_fixture_schema(
19720 "test/openai-routing-peer",
19721 "http://127.0.0.1:9".into(),
19722 schema::ApiProtocol::OpenAiCompat,
19723 "CAR_STATIC_ROUTING_PEER_KEY",
19724 );
19725 peer.provider = "openai".into();
19726 peer.trust_tier = TrustTier::Curated;
19727 engine.unified_registry_mut().register_project_model(peer);
19728 let reviewed: std::collections::HashSet<_> = engine
19729 .list_schemas()
19730 .into_iter()
19731 .filter(|schema| schema.id.starts_with("openrouter/"))
19732 .map(|schema| schema.id)
19733 .collect();
19734 assert_eq!(reviewed.len(), crate::openrouter::curated_model_count());
19735
19736 let with_key = engine
19737 .route_adaptive_with_intent(
19738 "Answer this simple question cheaply.",
19739 Some(IntentHint::default()),
19740 )
19741 .await;
19742 let openrouter_candidates: Vec<_> = std::iter::once(with_key.model_id.as_str())
19743 .chain(
19744 with_key
19745 .candidates
19746 .iter()
19747 .map(|candidate| candidate.model_id.as_str()),
19748 )
19749 .chain(with_key.fallbacks.iter().map(String::as_str))
19750 .filter(|id| id.starts_with("openrouter/"))
19751 .collect();
19752 assert!(
19753 !openrouter_candidates.is_empty(),
19754 "keyed adaptive decision must include a reviewed OpenRouter row: {with_key:?}"
19755 );
19756 assert!(openrouter_candidates
19757 .iter()
19758 .all(|id| reviewed.contains(*id)));
19759 assert!(
19760 std::iter::once(with_key.model_id.as_str())
19761 .chain(with_key.fallbacks.iter().map(String::as_str),)
19762 .any(|id| !id.starts_with("openrouter/")),
19763 "fallback chain must retain cross-provider alternatives: {with_key:?}"
19764 );
19765
19766 crate::openrouter::set_test_credential(None);
19767 let without_key = engine
19768 .route_adaptive_with_intent(
19769 "Answer this simple question cheaply.",
19770 Some(IntentHint::default()),
19771 )
19772 .await;
19773 assert!(!std::iter::once(without_key.model_id.as_str())
19774 .chain(
19775 without_key
19776 .candidates
19777 .iter()
19778 .map(|candidate| candidate.model_id.as_str()),
19779 )
19780 .chain(without_key.fallbacks.iter().map(String::as_str))
19781 .any(|id| id.starts_with("openrouter/")));
19782 unsafe {
19783 std::env::remove_var("CAR_STATIC_ROUTING_PEER_KEY");
19784 }
19785 }
19786
19787 #[tokio::test(flavor = "current_thread")]
19788 async fn v2_parslee_auth_drives_managed_registration_routing_lane_and_logout() {
19789 if !crate::run_in_isolated_test_process(
19790 "tests::v2_parslee_auth_drives_managed_registration_routing_lane_and_logout",
19791 "CAR_V2_ROUTING_AUTH_CHILD",
19792 ) {
19793 return;
19794 }
19795 let tmp = TempDir::new().unwrap();
19796 let (_credential_scope, _provider_env) =
19797 crate::openrouter::test_credential_and_environment_scope_async().await;
19798 let _restore = RestoredEnvironment::capture(&[
19799 "CAR_SECRETS_FILE_DIR",
19800 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19801 ]);
19802 let _home = crate::openrouter::StateRootScope::new();
19809 crate::openrouter::clear_gateway_unconfigured();
19810 crate::parslee_credential::clear_credential_rejected();
19811 let secrets_dir = tmp.path().join("secrets");
19812 unsafe {
19813 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
19814 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
19815 }
19816 crate::openrouter::set_test_credential(Some("personal-openrouter-key"));
19817
19818 let store = car_secrets::SecretStore::new();
19819 let state_ref =
19820 car_secrets::SecretRef::with_default_service(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
19821 store
19822 .publish(
19823 &state_ref,
19824 &serde_json::json!({
19825 "schema": 2,
19826 "revision": 7,
19827 "generation": 3,
19828 "active": {
19829 "account_id": "account-v2",
19830 "access_token": "v2-access",
19831 "refresh_token": "v2-refresh",
19832 "expires_at": 9_999_999_999_u64,
19833 "api_base": "https://api.parslee.ai"
19834 },
19835 "accounts": [{
19836 "account_id": "account-v2",
19837 "access_token": "v2-access",
19838 "refresh_token": "v2-refresh",
19839 "expires_at": 9_999_999_999_u64,
19840 "api_base": "https://api.parslee.ai"
19841 }],
19842 "tombstone": false
19843 })
19844 .to_string(),
19845 )
19846 .unwrap();
19847 assert_eq!(car_auth::access_token().as_deref(), Some("v2-access"));
19848 assert!(
19849 !store
19850 .status(&car_secrets::SecretRef::with_default_service(
19851 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19852 ))
19853 .unwrap()
19854 .exists,
19855 "the regression must exercise V2 state without the import-only legacy slot"
19856 );
19857
19858 let managed_ids = [
19859 "parslee/openrouter/frontier-general",
19860 "parslee/openrouter/balanced-general",
19861 ];
19862 let personal_id = "openrouter/deepseek/deepseek-v3.2";
19863 let rogue_id = "community/custom-oauth";
19864 let schemas = crate::openrouter::curated_schemas();
19865 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("registry-models"));
19866 for id in managed_ids.into_iter().chain(std::iter::once(personal_id)) {
19867 registry.register_project_model(
19868 schemas
19869 .iter()
19870 .find(|schema| schema.id == id)
19871 .unwrap_or_else(|| panic!("missing curated schema {id}"))
19872 .clone(),
19873 );
19874 }
19875 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
19876 for id in managed_ids {
19877 assert!(
19878 registry.get(id).unwrap().available_now(),
19879 "{id} must be available immediately when registered from V2 auth"
19880 );
19881 }
19882 assert!(registry.get(personal_id).unwrap().available_now());
19883 let mut rogue = schemas
19884 .iter()
19885 .find(|schema| schema.id == managed_ids[0])
19886 .unwrap()
19887 .clone();
19888 rogue.id = rogue_id.into();
19889 rogue.provider = "community".into();
19890 if let ModelSource::Proprietary {
19891 provider, endpoint, ..
19892 } = &mut rogue.source
19893 {
19894 *provider = "community".into();
19895 *endpoint = "https://untrusted.example".into();
19896 } else {
19897 panic!("managed fixture must remain proprietary");
19898 }
19899 registry.register(rogue);
19900 assert!(
19901 !registry.get(rogue_id).unwrap().available_now(),
19902 "a non-Parslee OAuth schema must not inherit Parslee V2 availability"
19903 );
19904
19905 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
19906 for id in managed_ids {
19907 assert!(
19908 registry.get(id).unwrap().available_now(),
19909 "{id} must stay available in a refreshed V2-auth snapshot"
19910 );
19911 }
19912 assert!(
19913 !registry.get(rogue_id).unwrap().available_now(),
19914 "refresh must keep non-Parslee OAuth schemas unavailable"
19915 );
19916
19917 let router = AdaptiveRouter::new(
19918 crate::hardware::HardwareInfo::detect(),
19919 RoutingConfig {
19920 prefer_local: false,
19921 prior_strength: 1_000_000.0,
19922 quality_first_cold_start: false,
19923 ..RoutingConfig::default()
19924 },
19925 );
19926 let tracker = OutcomeTracker::new();
19927 let intent = IntentHint {
19928 task: Some(crate::intent::TaskHint::Chat),
19929 exclude_models: vec![personal_id.into()],
19930 ..Default::default()
19931 };
19932 let decision = router.route_with(crate::adaptive_router::RouteRequest {
19933 intent: Some(&intent),
19934 ..crate::adaptive_router::RouteRequest::new(
19935 "Explain this architecture.",
19936 ®istry,
19937 &tracker,
19938 )
19939 });
19940 assert!(
19941 managed_ids.contains(&decision.model_id.as_str()),
19942 "managed V2-auth alias must be selectable: {decision:?}"
19943 );
19944 assert!(
19945 decision
19946 .candidates
19947 .iter()
19948 .any(|candidate| managed_ids.contains(&candidate.model_id.as_str())),
19949 "managed V2-auth alias must appear in adaptive candidates: {decision:?}"
19950 );
19951 assert!(
19952 decision
19953 .fallbacks
19954 .iter()
19955 .any(|id| managed_ids.contains(&id.as_str())),
19956 "managed V2-auth alias must appear in adaptive fallbacks: {decision:?}"
19957 );
19958 assert!(
19959 !std::iter::once(decision.model_id.as_str())
19960 .chain(
19961 decision
19962 .candidates
19963 .iter()
19964 .map(|candidate| candidate.model_id.as_str())
19965 )
19966 .chain(decision.fallbacks.iter().map(String::as_str))
19967 .any(|id| id == rogue_id),
19968 "a non-Parslee OAuth schema must never enter adaptive selection, candidates, or fallbacks: {decision:?}"
19969 );
19970
19971 let engine = InferenceEngine::new(test_config(tmp.path().join("engine-models")));
19972 let managed_lane_id = managed_ids[0];
19973 engine.lane_defaults_cache.write().unwrap().set(
19974 None,
19975 crate::intent::UseCase::Assistant,
19976 managed_lane_id.into(),
19977 1,
19978 );
19979 let request = GenerateRequest {
19980 prompt: "lane default".into(),
19981 intent: Some(IntentHint {
19982 task: Some(crate::intent::TaskHint::Chat),
19983 ..Default::default()
19984 }),
19985 ..Default::default()
19986 };
19987 assert_eq!(
19988 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
19989 Some(managed_lane_id.to_string()),
19990 "a managed lane default must resolve from authoritative V2 auth"
19991 );
19992
19993 car_auth::logout()
19994 .await
19995 .expect("production logout must publish a signed-out tombstone");
19996 let persisted_logout: serde_json::Value =
19997 serde_json::from_str(&store.get(&state_ref).unwrap()).unwrap();
19998 assert_eq!(persisted_logout["tombstone"], true);
19999 assert_eq!(persisted_logout["accounts"], serde_json::json!([]));
20000 assert!(persisted_logout.get("active").is_none());
20001 assert_eq!(car_auth::access_token(), None);
20002
20003 registry.refresh_routing_availability(None, true);
20004 for id in managed_ids {
20005 assert!(
20006 !registry.get(id).unwrap().available_now(),
20007 "{id} must disappear from routing after the signed-out tombstone"
20008 );
20009 }
20010 assert!(
20011 registry.get(personal_id).unwrap().available_now(),
20012 "personal rows must continue to follow their independent personal-key seam"
20013 );
20014
20015 let signed_out = engine.routing_registry_snapshot().await;
20016 assert!(
20017 signed_out
20018 .list()
20019 .into_iter()
20020 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
20021 .all(|schema| !schema.available_now()),
20022 "the next engine snapshot must exclude every managed alias after logout"
20023 );
20024 assert!(
20025 signed_out.get(personal_id).unwrap().available_now(),
20026 "logout must not disable a still-keyed personal OpenRouter row"
20027 );
20028 assert_eq!(
20029 engine.lane_pin_for(&request, &signed_out),
20030 None,
20031 "a signed-out managed lane default must stop resolving"
20032 );
20033
20034 crate::openrouter::set_test_credential(None);
20035 registry.refresh_availability();
20036 assert!(
20037 !registry.get(personal_id).unwrap().available_now(),
20038 "personal row availability must still turn off with the personal-key seam"
20039 );
20040 }
20041
20042 #[tokio::test(flavor = "current_thread")]
20043 async fn legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates() {
20044 if !crate::run_in_isolated_test_process(
20045 "tests::legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates",
20046 "CAR_LEGACY_V2_MIGRATION_CHILD",
20047 ) {
20048 return;
20049 }
20050 let tmp = TempDir::new().unwrap();
20051 let _provider_env = crate::openrouter::test_environment_scope_async().await;
20052 let _restore = RestoredEnvironment::capture(&[
20053 "CAR_SECRETS_FILE_DIR",
20054 car_home::ENV_VAR,
20055 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20056 ]);
20057 let _home = crate::openrouter::StateRootScope::new();
20072 crate::openrouter::clear_gateway_unconfigured();
20073 crate::parslee_credential::clear_credential_rejected();
20074 unsafe {
20075 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
20076 std::env::set_var(car_home::ENV_VAR, tmp.path().join("car-home"));
20077 }
20078
20079 let secret = |key| car_secrets::SecretRef::with_default_service(key);
20080 let seed_legacy = |store: &car_secrets::SecretStore| {
20081 store
20082 .put(
20083 &secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY),
20084 "legacy-access",
20085 )
20086 .unwrap();
20087 store
20088 .put(
20089 &secret(car_secrets::PARSLEE_ACTIVE_ACCOUNT_ID_KEY),
20090 "legacy-account",
20091 )
20092 .unwrap();
20093 store.put(
20094 &secret(car_secrets::PARSLEE_ACCOUNTS_KEY),
20095 r#"{"active":"legacy-account","accounts":[{"id":"legacy-account","email":"legacy@example.test"}]}"#,
20096 )
20097 .unwrap();
20098 };
20099 let state_ref = secret(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
20100 let managed = crate::openrouter::curated_schemas()
20101 .into_iter()
20102 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
20103 .unwrap();
20104
20105 unsafe {
20106 std::env::set_var("CAR_SECRETS_FILE_DIR", tmp.path().join("tombstone-secrets"));
20107 }
20108 let tombstoned_store = car_secrets::SecretStore::new();
20109 seed_legacy(&tombstoned_store);
20110 tombstoned_store
20111 .publish(
20112 &state_ref,
20113 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
20114 )
20115 .unwrap();
20116 let mut tombstoned_registry =
20117 UnifiedRegistry::new_empty(tmp.path().join("tombstoned-models"));
20118 tombstoned_registry.register_project_model(managed.clone());
20119 assert!(
20120 !tombstoned_registry
20121 .get(&managed.id)
20122 .unwrap()
20123 .available_now(),
20124 "passive catalog registration must remain disabled without a configured hint"
20125 );
20126 let tombstoned = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
20127 .await
20128 .unwrap();
20129 assert!(tombstoned.is_none(), "the V2 tombstone is authoritative");
20130 tombstoned_registry.refresh_routing_availability(None, true);
20131 assert!(!tombstoned_registry
20132 .get(&managed.id)
20133 .unwrap()
20134 .available_now());
20135
20136 unsafe {
20137 std::env::set_var(
20138 "CAR_SECRETS_FILE_DIR",
20139 tmp.path().join("legacy-only-secrets"),
20140 );
20141 }
20142 let legacy_store = car_secrets::SecretStore::new();
20143 seed_legacy(&legacy_store);
20144 assert!(!legacy_store.status(&state_ref).unwrap().exists);
20145 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("legacy-models"));
20146 registry.register_project_model(managed.clone());
20147 assert!(
20148 !registry.get(&managed.id).unwrap().available_now(),
20149 "passive registration must not inspect attributable legacy slots"
20150 );
20151 registry.refresh_availability();
20152 assert!(
20153 !registry.get(&managed.id).unwrap().available_now(),
20154 "passive refresh must remain secret-store free until request-time migration"
20155 );
20156 let resolved = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
20157 .await
20158 .unwrap()
20159 .expect("request-time auth reconciliation must migrate attributable legacy state");
20160 registry.refresh_routing_availability(Some(&resolved.api_base), false);
20161 assert!(registry.get(&managed.id).unwrap().available_now());
20162 assert!(
20163 legacy_store.status(&state_ref).unwrap().exists,
20164 "request-time auth reconciliation must publish the migrated V2 record"
20165 );
20166 assert!(
20167 !legacy_store
20168 .status(&secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY))
20169 .unwrap()
20170 .exists,
20171 "successful V2 migration must clean the legacy access slot"
20172 );
20173 }
20174
20175 #[tokio::test]
20176 async fn reviewed_openrouter_lane_default_tracks_live_credential_availability() {
20177 let _credential_scope = crate::openrouter::test_credential_scope();
20178 crate::openrouter::set_test_credential(Some("static-key"));
20179 let tmp = TempDir::new().unwrap();
20180 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20181 let reviewed_id = "openrouter/deepseek/deepseek-v3.2";
20182 engine.lane_defaults_cache.write().unwrap().set(
20183 None,
20184 crate::intent::UseCase::Assistant,
20185 reviewed_id.into(),
20186 1,
20187 );
20188 let request = GenerateRequest {
20189 prompt: "lane default".into(),
20190 intent: Some(IntentHint {
20191 task: Some(crate::intent::TaskHint::Chat),
20192 ..Default::default()
20193 }),
20194 ..Default::default()
20195 };
20196 assert_eq!(
20197 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
20198 Some(reviewed_id.to_string()),
20199 "a reviewed keyed row must be eligible as a lane default"
20200 );
20201 crate::openrouter::set_test_credential(None);
20202 assert_eq!(
20203 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
20204 None,
20205 "the same static lane default must stop being eligible immediately after key removal"
20206 );
20207 }
20208
20209 #[tokio::test(flavor = "current_thread")]
20210 async fn personal_openrouter_baseline_remains_visible_but_disabled_without_a_credential() {
20211 let _credential_scope = crate::openrouter::test_credential_scope();
20212 crate::openrouter::set_test_credential(None);
20213 let tmp = TempDir::new().unwrap();
20214 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20215
20216 let personal: Vec<_> = engine
20217 .list_schemas()
20218 .into_iter()
20219 .filter(|schema| schema.id.starts_with("openrouter/"))
20220 .collect();
20221 assert_eq!(
20222 personal.len(),
20223 crate::openrouter::curated_model_count(),
20224 "the vetted personal rows stay discoverable as a disabled baseline"
20225 );
20226 assert!(
20227 personal.iter().all(|schema| !schema.available_now()),
20228 "no-key baseline rows must never become routing candidates"
20229 );
20230
20231 let error = engine
20232 .generate_tracked(GenerateRequest {
20233 prompt: "hello".into(),
20234 model: Some("openrouter/openai/gpt-5.4".into()),
20235 params: GenerateParams {
20236 strict_model: true,
20237 ..Default::default()
20238 },
20239 ..Default::default()
20240 })
20241 .await
20242 .expect_err("explicitly selecting a disabled baseline row must be actionable");
20243 let message = error.to_string();
20244 assert!(
20245 message.contains("car keys set openrouter") && message.contains("CarHost"),
20246 "disabled personal row must explain how to connect OpenRouter: {message}"
20247 );
20248 }
20249
20250 #[tokio::test]
20251 async fn reviewed_openrouter_metadata_stays_static_across_credential_changes() {
20252 let _credential_scope = crate::openrouter::test_credential_scope();
20253 crate::openrouter::set_test_credential(Some("static-key"));
20254
20255 let tmp = TempDir::new().unwrap();
20256 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20257 let live = engine.routing_registry_snapshot().await;
20258 let vetted = live.get("openrouter/openai/gpt-5.4").unwrap();
20259 assert!(vetted.available_now());
20260 assert_eq!(vetted.context_length, 1_050_000);
20261 assert_eq!(vetted.max_output_tokens, Some(128_000));
20262 assert_eq!(vetted.cost.input_per_mtok, Some(2.5));
20263 assert_eq!(vetted.cost.output_per_mtok, Some(15.0));
20264 assert!(
20265 !vetted.cost.pricing_tiers.is_empty(),
20266 "reviewed high-context pricing tiers must remain in the static row"
20267 );
20268 assert_eq!(vetted.trust_tier, TrustTier::Curated);
20269 assert!(vetted.tags.iter().any(|tag| tag == "frontier"));
20270 assert!(vetted.has_capability(ModelCapability::Code));
20271 assert!(vetted
20272 .supported_params
20273 .contains(&schema::GenerateParam::ExtendedThinking));
20274
20275 assert!(live.get("openrouter/vendor/unreviewed-model").is_none());
20276
20277 crate::openrouter::set_test_credential(None);
20278 let reverted = engine.routing_registry_snapshot().await;
20279 assert!(reverted.get("openrouter/vendor/unreviewed-model").is_none());
20280 let baseline = reverted.get("openrouter/openai/gpt-5.4").unwrap();
20281 assert!(
20282 !baseline.available_now(),
20283 "credential removal must disable the reviewed row without removing it"
20284 );
20285 }
20286
20287 #[test]
20306 fn model_pulled_at_runtime_is_ready_without_a_daemon_restart() {
20307 let tmp = TempDir::new().unwrap();
20308 let models_dir = tmp.path().join("models");
20309 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20310 engine.register_model(ModelSchema {
20311 id: "mlx/pulled-later".into(),
20312 name: "PulledLater".into(),
20313 provider: "local".into(),
20314 family: "qwen3".into(),
20315 version: "test".into(),
20316 capabilities: vec![ModelCapability::Generate, ModelCapability::Code],
20317 context_length: 4096,
20318 max_output_tokens: None,
20319 param_count: String::new(),
20320 quantization: None,
20321 performance: schema::PerformanceEnvelope::default(),
20322 cost: schema::CostModel::default(),
20323 source: ModelSource::Mlx {
20324 hf_repo: "example/pulled-later".into(),
20325 hf_weight_file: None,
20326 },
20327 tags: vec![],
20328 supported_params: vec![],
20329 public_benchmarks: vec![],
20330 trust_tier: TrustTier::Curated,
20331 deprecated: false,
20332 available: true,
20333 weights_ready: false,
20334 });
20335
20336 let engine = &engine;
20338 let is_ready = || {
20339 engine
20340 .list_schemas()
20341 .into_iter()
20342 .find(|s| s.id == "mlx/pulled-later")
20343 .expect("registered model must be listed")
20344 .weights_ready
20345 };
20346
20347 assert!(!is_ready(), "precondition: no weights on disk yet");
20348
20349 let dir = models_dir.join("PulledLater");
20352 std::fs::create_dir_all(&dir).unwrap();
20353 std::fs::write(dir.join("config.json"), "{}").unwrap();
20354 std::fs::write(dir.join("model.safetensors"), b"weights").unwrap();
20355
20356 assert!(
20357 is_ready(),
20358 "a model pulled at runtime must be ready without restarting the daemon"
20359 );
20360
20361 std::fs::remove_file(dir.join("model.safetensors")).unwrap();
20364 assert!(
20365 !is_ready(),
20366 "readiness must track the disk in both directions, not latch"
20367 );
20368 }
20369
20370 #[tokio::test(flavor = "current_thread")]
20371 async fn missing_openrouter_key_is_typed_without_changing_its_string_contract() {
20372 let (_credential_scope, _provider_environment) =
20376 crate::openrouter::test_credential_and_environment_scope_async().await;
20377 let home = crate::openrouter::StateRootScope::new();
20378 let _restore = RestoredEnvironment::capture(&["CAR_SECRETS_FILE_DIR"]);
20379 unsafe {
20380 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20381 }
20382
20383 crate::openrouter::set_test_credential(None);
20387
20388 let models_dir = home.path().join("models");
20389 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20390 let schema = crate::openrouter::curated_schemas()
20391 .into_iter()
20392 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
20393 .expect("personal OpenRouter fixture");
20394 let model_id = schema.id.clone();
20395 engine.unified_registry_mut().register_project_model(schema);
20396
20397 let error = engine
20398 .generate_tracked(GenerateRequest {
20399 prompt: "must stop before provider dispatch".into(),
20400 model: Some(model_id.clone()),
20401 params: GenerateParams {
20402 strict_model: true,
20403 ..Default::default()
20404 },
20405 ..Default::default()
20406 })
20407 .await
20408 .expect_err("an OpenRouter route without a key must fail before dispatch");
20409
20410 let tracker = engine.outcome_tracker();
20411 assert!(
20412 tracker.read().await.profile(&model_id).is_none(),
20413 "a missing OpenRouter key must not create model health"
20414 );
20415 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20416 receipts.extend(tracker.write().await.drain_ledger());
20417 let receipts = receipts
20418 .iter()
20419 .filter(|receipt| receipt.model_id == model_id)
20420 .collect::<Vec<_>>();
20421 assert_eq!(receipts.len(), 1);
20422 assert_eq!(receipts[0].success, None);
20423 assert_eq!(receipts[0].quality, None);
20424 assert!(
20425 engine
20426 .adaptive_router
20427 .circuit_breakers
20428 .lock()
20429 .unwrap()
20430 .state(&model_id)
20431 .is_none(),
20432 "a missing OpenRouter key must not create breaker state"
20433 );
20434
20435 let message = "OpenRouter requires a key — run `car keys set openrouter` or connect your OpenRouter account in CarHost";
20436 assert_eq!(
20437 error.to_string(),
20438 format!("inference failed: {message}"),
20439 "guard: the pre-existing Display contract must stay byte-identical"
20440 );
20441 assert!(
20442 !is_auth_failure_message(&error.to_string()),
20443 "guard: an OpenRouter key is not repaired by Parslee sign-in"
20444 );
20445 assert_eq!(
20446 classify_fallback_reason(&error),
20447 classify_fallback_reason(&InferenceError::InferenceFailed(message.into())),
20448 "retyping must preserve prior fallback classification"
20449 );
20450 match error {
20451 InferenceError::ProviderKeyMissing {
20452 provider,
20453 model,
20454 env_vars,
20455 message: actual_message,
20456 } => {
20457 assert_eq!(provider, "openrouter");
20458 assert_eq!(model, model_id);
20459 assert_eq!(env_vars, vec![crate::openrouter::API_KEY_ENV]);
20460 assert_eq!(actual_message, message);
20461 }
20462 other => panic!("expected ProviderKeyMissing, got {other:?}"),
20463 }
20464 }
20465
20466 #[tokio::test(flavor = "current_thread")]
20467 async fn missing_generic_remote_key_is_typed_without_changing_its_string_contract() {
20468 const KEY_ENV: &str = "CAR_TEST_A47_GENERIC_PROVIDER_KEY_DO_NOT_SET";
20469 const EXTRA_KEY_ENV_1: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_1_DO_NOT_SET";
20470 const EXTRA_KEY_ENV_2: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_2_DO_NOT_SET";
20471 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
20472 let home = crate::openrouter::StateRootScope::new();
20473 let _restore = RestoredEnvironment::capture(&[
20474 "CAR_SECRETS_FILE_DIR",
20475 KEY_ENV,
20476 EXTRA_KEY_ENV_1,
20477 EXTRA_KEY_ENV_2,
20478 ]);
20479 unsafe {
20480 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20481 std::env::remove_var(KEY_ENV);
20482 std::env::remove_var(EXTRA_KEY_ENV_1);
20483 std::env::remove_var(EXTRA_KEY_ENV_2);
20484 }
20485
20486 let endpoint = "http://127.0.0.1:9/a47-missing-key";
20487 let model_id = "test/a47-generic-missing-key";
20488 let models_dir = home.path().join("models");
20489 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20490 let mut schema = remote_stream_fixture_schema(
20491 model_id,
20492 endpoint.into(),
20493 schema::ApiProtocol::OpenAiCompat,
20494 KEY_ENV,
20495 );
20496 let ModelSource::RemoteApi { api_key_envs, .. } = &mut schema.source else {
20497 panic!("generic remote fixture must use RemoteApi");
20498 };
20499 *api_key_envs = vec![EXTRA_KEY_ENV_1.into(), EXTRA_KEY_ENV_2.into()];
20500 engine.unified_registry_mut().register_project_model(schema);
20501
20502 let error = engine
20503 .generate_tracked(GenerateRequest {
20504 prompt: "must stop before provider dispatch".into(),
20505 model: Some(model_id.into()),
20506 params: GenerateParams {
20507 strict_model: true,
20508 ..Default::default()
20509 },
20510 ..Default::default()
20511 })
20512 .await
20513 .expect_err("a generic remote route without a key must fail before dispatch");
20514
20515 let tracker = engine.outcome_tracker();
20516 assert!(
20517 tracker.read().await.profile(model_id).is_none(),
20518 "a missing generic provider key must not create model health"
20519 );
20520 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20521 receipts.extend(tracker.write().await.drain_ledger());
20522 let receipts = receipts
20523 .iter()
20524 .filter(|receipt| receipt.model_id == model_id)
20525 .collect::<Vec<_>>();
20526 assert_eq!(receipts.len(), 1);
20527 assert_eq!(receipts[0].success, None);
20528 assert_eq!(receipts[0].quality, None);
20529 assert!(
20530 engine
20531 .adaptive_router
20532 .circuit_breakers
20533 .lock()
20534 .unwrap()
20535 .state(model_id)
20536 .is_none(),
20537 "a missing generic provider key must not create breaker state"
20538 );
20539
20540 let message = format!(
20541 "no API keys available for endpoint {endpoint} (checked env vars: [\"{KEY_ENV}\", \"{EXTRA_KEY_ENV_1}\", \"{EXTRA_KEY_ENV_2}\"])"
20542 );
20543 assert_eq!(
20544 error.to_string(),
20545 format!("inference failed: {message}"),
20546 "guard: the pre-existing Display contract must stay byte-identical"
20547 );
20548 assert!(
20549 !is_auth_failure_message(&error.to_string()),
20550 "guard: a generic provider key is not repaired by Parslee sign-in"
20551 );
20552 assert_eq!(
20553 classify_fallback_reason(&error),
20554 classify_fallback_reason(&InferenceError::InferenceFailed(message.clone())),
20555 "retyping must preserve prior fallback classification"
20556 );
20557 match error {
20558 InferenceError::ProviderKeyMissing {
20559 provider,
20560 model,
20561 env_vars,
20562 message: actual_message,
20563 } => {
20564 assert_eq!(provider, "test");
20565 assert_eq!(model, model_id);
20566 assert_eq!(env_vars, vec![KEY_ENV, EXTRA_KEY_ENV_1, EXTRA_KEY_ENV_2]);
20567 assert_eq!(actual_message, message);
20568 }
20569 other => panic!("expected ProviderKeyMissing, got {other:?}"),
20570 }
20571 }
20572
20573 #[test]
20577 fn credential_unavailable_outcomes_do_not_degrade_models_or_open_breakers() {
20578 for reason in [
20579 CredentialFailure::Expired { expires_at: 42 },
20580 CredentialFailure::SignedOut,
20581 CredentialFailure::StoreUnreadable,
20582 CredentialFailure::EnvVarMissing {
20583 env_var: "TEST_PROVIDER_KEY".into(),
20584 },
20585 CredentialFailure::RaceRetryable,
20586 ] {
20587 let model_id = "parslee/test-credential-outcome";
20588 let error = InferenceError::CredentialUnavailable {
20589 provider: "parslee".into(),
20590 model: model_id.into(),
20591 reason: reason.clone(),
20592 detail: "credential resolution stopped before dispatch".into(),
20593 };
20594 let mut tracker = OutcomeTracker::new();
20595 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
20596 record_dispatch_failure(&mut tracker, &trace, &error);
20597
20598 assert!(
20599 tracker.profile(model_id).is_none(),
20600 "{reason:?} changed the model profile: {:?}",
20601 tracker.profile(model_id)
20602 );
20603 let receipts = tracker.drain_ledger();
20604 assert_eq!(receipts.len(), 1, "{reason:?}");
20605 assert_eq!(receipts[0].success, None, "{reason:?}");
20606 assert_eq!(receipts[0].quality, None, "{reason:?}");
20607
20608 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
20609 if error_counts_against_circuit_breaker(&error) {
20610 breakers.record_failure(model_id);
20611 }
20612 assert!(
20613 breakers.state(model_id).is_none(),
20614 "{reason:?} created per-model breaker state"
20615 );
20616 }
20617 }
20618
20619 #[tokio::test(flavor = "current_thread")]
20620 async fn signed_out_managed_attempts_leave_health_and_breaker_untouched() {
20621 if !crate::run_in_isolated_test_process(
20622 "tests::signed_out_managed_attempts_leave_health_and_breaker_untouched",
20623 "CAR_SIGNED_OUT_MODEL_HEALTH_CHILD",
20624 ) {
20625 return;
20626 }
20627
20628 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
20629 let home = crate::openrouter::StateRootScope::new();
20630 let _restore = RestoredEnvironment::capture(&[
20631 "CAR_SECRETS_FILE_DIR",
20632 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20633 car_auth::PARSLEE_API_BASE_KEY,
20634 ]);
20635 unsafe {
20636 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20637 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
20638 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
20639 }
20640 crate::openrouter::clear_gateway_unconfigured();
20641 crate::parslee_credential::clear_credential_rejected();
20642
20643 let store = car_secrets::SecretStore::new();
20644 store
20645 .publish(
20646 &car_secrets::SecretRef::with_default_service(
20647 car_secrets::PARSLEE_AUTH_STATE_V2_KEY,
20648 ),
20649 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
20650 )
20651 .unwrap();
20652
20653 let models_dir = home.path().join("models");
20654 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20655 let schema = crate::openrouter::curated_schemas()
20656 .into_iter()
20657 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
20658 .expect("managed alias fixture");
20659 let model_id = schema.id.clone();
20660 engine.unified_registry_mut().register_project_model(schema);
20661
20662 for _ in 0..3 {
20666 let error = engine
20667 .generate_tracked(GenerateRequest {
20668 prompt: "must stop before provider dispatch".into(),
20669 model: Some(model_id.clone()),
20670 params: GenerateParams {
20671 strict_model: true,
20672 ..Default::default()
20673 },
20674 ..Default::default()
20675 })
20676 .await
20677 .expect_err("signed-out managed inference must be rejected");
20678 assert!(
20679 matches!(
20680 &error,
20681 InferenceError::CredentialUnavailable {
20682 reason: CredentialFailure::SignedOut,
20683 ..
20684 }
20685 ),
20686 "unexpected signed-out error: {error:?}"
20687 );
20688 }
20689
20690 let tracker = engine.outcome_tracker();
20691 assert!(
20692 tracker.read().await.profile(&model_id).is_none(),
20693 "signed-out attempts must not create a model-health profile"
20694 );
20695 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20699 receipts.extend(tracker.write().await.drain_ledger());
20700 let signed_out_receipts = receipts
20701 .iter()
20702 .filter(|entry| entry.model_id == model_id)
20703 .collect::<Vec<_>>();
20704 assert_eq!(signed_out_receipts.len(), 3);
20705 assert!(signed_out_receipts
20706 .iter()
20707 .all(|entry| entry.success.is_none() && entry.quality.is_none()));
20708 assert!(
20709 engine
20710 .adaptive_router
20711 .circuit_breakers
20712 .lock()
20713 .unwrap()
20714 .state(&model_id)
20715 .is_none(),
20716 "signed-out attempts must not create per-model breaker state"
20717 );
20718 }
20719
20720 fn admission_refusals() -> Vec<InferenceError> {
20724 use resource_policy::{LocalLoadVerdict, ModelMemoryEstimate, ModelResourceEvidence};
20725 [
20726 LocalLoadVerdict::LiveMemoryUnknown,
20727 LocalLoadVerdict::DisabledByPolicy,
20728 LocalLoadVerdict::ExceedsConfiguredCeiling,
20729 LocalLoadVerdict::InsufficientLiveMemory,
20730 LocalLoadVerdict::ModelMaintenance,
20731 LocalLoadVerdict::PendingTeardown,
20732 ]
20733 .into_iter()
20734 .map(|verdict| {
20735 InferenceError::from(resource_policy::LocalAdmissionError {
20736 preflight: resource_policy::LocalLoadPreflight {
20737 model_id: "vllm-mlx/big:4bit".into(),
20738 estimate: ModelMemoryEstimate {
20739 weights_mb: 16_000,
20740 runtime_overhead_mb: 1_000,
20741 context_overhead_mb: 2_000,
20742 transient_margin_mb: 1_000,
20743 estimated_peak_mb: 20_000,
20744 evidence: ModelResourceEvidence::CatalogExact,
20745 },
20746 configured_ceiling_mb: 26_214,
20747 resident_model_mb: 0,
20748 active_reservations_mb: 0,
20749 estimated_incremental_mb: 20_000,
20750 accelerator_total_mb: None,
20751 accelerator_resident_mb: None,
20752 accelerator_incremental_mb: None,
20753 live_available_mb: Some(4_000),
20754 emergency_reserve_mb: 6_553,
20755 verdict,
20756 },
20757 })
20758 })
20759 .collect()
20760 }
20761
20762 #[test]
20766 fn local_admission_refusal_is_not_a_model_failure() {
20767 let model_id = "vllm-mlx/big:4bit";
20768 for refusal in admission_refusals() {
20769 assert!(matches!(
20770 refusal,
20771 InferenceError::LocalResourceBlocked { .. }
20772 ));
20773 let mut tracker = OutcomeTracker::new();
20774 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
20775 record_dispatch_failure(&mut tracker, &trace, &refusal);
20776 assert!(
20777 tracker.profile(model_id).is_none(),
20778 "admission refusal changed the model profile: {refusal}"
20779 );
20780 let receipt = tracker.drain_ledger().pop().expect("receipt kept");
20781 assert_eq!(receipt.success, None, "{refusal}");
20782 assert_eq!(receipt.quality, None, "{refusal}");
20783 assert!(!error_counts_against_circuit_breaker(&refusal), "{refusal}");
20784 }
20785 }
20786
20787 #[test]
20788 fn content_refusal_closes_its_parenthetical_in_every_shape() {
20789 let render = |kind: Option<&str>, code: Option<&str>| {
20790 InferenceError::ContentRefused {
20791 provider: "p".into(),
20792 kind: kind.map(Into::into),
20793 code: code.map(Into::into),
20794 message: "m".into(),
20795 }
20796 .to_string()
20797 };
20798 assert_eq!(
20799 render(Some("k"), Some("c")),
20800 "p refused this request on content grounds (type=k, code=c): m"
20801 );
20802 assert_eq!(
20803 render(Some("k"), None),
20804 "p refused this request on content grounds (type=k): m"
20805 );
20806 assert_eq!(
20807 render(None, Some("c")),
20808 "p refused this request on content grounds (code=c): m"
20809 );
20810 assert_eq!(
20811 render(None, None),
20812 "p refused this request on content grounds: m"
20813 );
20814 }
20815
20816 fn legacy_unattributed_errors() -> Vec<String> {
20821 let mut legacy: Vec<String> = admission_refusals()
20822 .iter()
20823 .map(ToString::to_string)
20824 .collect();
20825 legacy.push(
20826 InferenceError::ProviderKeyMissing {
20827 provider: "anthropic".into(),
20828 model: "m".into(),
20829 env_vars: vec!["ANTHROPIC_API_KEY".into()],
20830 message: "no API keys available for endpoint https://api.anthropic.com (checked env vars: [\"ANTHROPIC_API_KEY\"])".into(),
20831 }
20832 .to_string(),
20833 );
20834 legacy.push(
20835 InferenceError::InferenceFailed(
20836 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required".into(),
20837 )
20838 .to_string(),
20839 );
20840 legacy.push(
20841 InferenceError::InferenceFailed(
20842 "FoundationModels declined the request (guardrailViolation): unsafe".into(),
20843 )
20844 .to_string(),
20845 );
20846 legacy.push(
20847 InferenceError::InferenceFailed(
20848 InferenceError::UnsupportedMode {
20849 mode: "apple-foundation-models",
20850 backend: "remote",
20851 reason: "no",
20852 }
20853 .to_string(),
20854 )
20855 .to_string(),
20856 );
20857 legacy
20858 }
20859
20860 fn write_legacy_ledger(
20864 ledger: &std::path::Path,
20865 real: &str,
20866 ghost: &str,
20867 ) -> Vec<outcome::ModelProfile> {
20868 let mut writer = OutcomeTracker::new();
20869 for error in legacy_unattributed_errors() {
20870 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
20871 writer.record_failure(&t, &error);
20872 }
20873 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
20874 writer.record_failure(&t, GENUINE_FAILURE);
20875 let t = writer.record_start(ghost, InferenceTask::Generate, "explicit model");
20876 writer.record_failure(
20877 &t,
20878 &InferenceError::InferenceFailed(
20879 InferenceError::ModelNotFound(ghost.into()).to_string(),
20880 )
20881 .to_string(),
20882 );
20883 outcome::append_ledger_entries(ledger, &writer.drain_ledger()).unwrap();
20884 writer.export_profiles()
20885 }
20886
20887 fn reconcile_for_test(
20890 ledger: &std::path::Path,
20891 tracker: &mut OutcomeTracker,
20892 is_registered: &dyn Fn(&str) -> bool,
20893 ) -> std::io::Result<outcome::LedgerReconciliation> {
20894 let plan = outcome::plan_reconcile(
20895 ledger,
20896 tracker,
20897 is_registered,
20898 &std::collections::HashMap::new(),
20899 );
20900 if let Some(entries) = &plan.ledger {
20901 outcome::write_reconciled_ledger(ledger, entries)?;
20902 }
20903 if let Some(profiles) = plan.profiles {
20904 tracker.commit_reconciled(profiles);
20905 }
20906 Ok(plan.report)
20907 }
20908
20909 const GENUINE_FAILURE: &str =
20910 "inference failed: forward: \"[metal::malloc] Attempting to allocate 41846087808 bytes\"";
20911
20912 fn profile_json(tracker: &OutcomeTracker, id: &str) -> serde_json::Value {
20913 let mut v = serde_json::to_value(tracker.profile(id).unwrap()).unwrap();
20914 v.as_object_mut().unwrap().remove("updated_at");
20915 v
20916 }
20917
20918 #[test]
20923 fn reconcile_takes_back_misattributed_failures_and_their_quality_evidence() {
20924 let dir = tempfile::tempdir().unwrap();
20925 let ledger = dir.path().join("outcome_ledger.jsonl");
20926 let (real, ghost) = ("mlx/real:4bit", "nonexistent-model-id-xyz");
20927 let profiles = write_legacy_ledger(&ledger, real, ghost);
20928 let legacy = legacy_unattributed_errors().len();
20929
20930 let mut tracker = OutcomeTracker::new();
20931 tracker.import_profiles(profiles);
20932 let before = tracker.profile(real).unwrap().clone();
20933 assert_eq!(before.fail_count, legacy as u64 + 1);
20934 assert!(
20935 before.quality_observations > 1,
20936 "the legacy rows moved the EMA"
20937 );
20938 assert!(
20939 before.ema_quality < 0.4 - 1e-9,
20940 "buried below one failure's worth"
20941 );
20942
20943 let is_registered = |id: &str| id == real;
20944 let report = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
20945 assert_eq!(report.reclassified, legacy);
20946 assert_eq!(report.unregistered, 1);
20947 assert_eq!(report.profiles_rebuilt, 1);
20948 assert_eq!(report.profiles_removed, 1);
20949
20950 let p = tracker.profile(real).unwrap();
20951 assert_eq!((p.fail_count, p.total_calls, p.success_count), (1, 1, 0));
20952 assert_eq!(p.quality_observations, 1);
20953 assert!(
20954 (p.ema_quality - 0.4).abs() < 1e-12,
20955 "0.5 stepped once toward 0: {}",
20956 p.ema_quality
20957 );
20958 let ts = &p.task_stats["generate"];
20959 assert_eq!((ts.failures, ts.calls, ts.quality_observations), (1, 0, 1));
20961 assert!(tracker.profile(ghost).is_none());
20962
20963 let rows = outcome::read_ledger(&ledger, 0);
20964 assert_eq!(rows.len(), legacy + 2, "nothing is deleted");
20965 for row in &rows {
20966 match row.error.as_deref() {
20967 Some(GENUINE_FAILURE) => {
20968 assert_eq!(row.success, Some(false));
20969 assert_eq!(row.attribution, None);
20970 }
20971 _ if row.model_id == ghost => {
20972 assert_eq!(row.success, None);
20973 assert_eq!(
20974 row.attribution,
20975 Some(outcome::UnattributedReason::Unregistered)
20976 );
20977 }
20978 Some(error) => {
20979 use outcome::UnattributedReason as R;
20980 assert_eq!(row.success, None);
20981 assert_eq!(row.quality, None);
20982 let expected = if error.contains("memory")
20984 || error.contains("allocation")
20985 || error.contains("maintained")
20986 || error.contains("previous local model")
20987 {
20988 R::ResourceBlocked
20989 } else if error.contains("API keys") || error.contains("org lookup") {
20990 R::Account
20991 } else if error.contains("declined") {
20992 R::ContentRefused
20993 } else {
20994 R::Capability
20995 };
20996 assert_eq!(row.attribution, Some(expected), "{error}");
20997 }
20998 None => panic!("every legacy row carries its error"),
20999 }
21000 }
21001 #[cfg(unix)]
21002 {
21003 use std::os::unix::fs::PermissionsExt;
21004 let mode = std::fs::metadata(&ledger).unwrap().permissions().mode() & 0o777;
21005 assert_eq!(mode, 0o600, "a rewritten ledger stays owner-only");
21006 }
21007
21008 let rebuilt = profile_json(&tracker, real);
21010 let again = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
21011 assert_eq!(
21012 again.reclassified + again.unregistered + again.profiles_removed,
21013 0
21014 );
21015 assert_eq!(profile_json(&tracker, real), rebuilt);
21016 }
21017
21018 #[test]
21024 fn reconcile_repairs_a_ledger_the_first_migration_already_rewrote() {
21025 let dir = tempfile::tempdir().unwrap();
21026 let ledger = dir.path().join("outcome_ledger.jsonl");
21027 let real = "mlx/real:4bit";
21028 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
21029
21030 let mut clean = OutcomeTracker::new();
21032 clean.import_profiles(profiles.clone());
21033 let clean_ledger = dir.path().join("clean.jsonl");
21034 std::fs::copy(&ledger, &clean_ledger).unwrap();
21035 reconcile_for_test(&clean_ledger, &mut clean, &|id| id == real).unwrap();
21036 let expected = profile_json(&clean, real);
21037
21038 let rows: Vec<_> = outcome::read_ledger(&ledger, 0)
21041 .into_iter()
21042 .map(|mut r| {
21043 if r.model_id == real && r.error.as_deref() != Some(GENUINE_FAILURE) {
21044 r.success = None;
21045 r.quality = None;
21046 }
21047 r
21048 })
21049 .collect();
21050 std::fs::remove_file(&ledger).unwrap();
21051 outcome::append_ledger_entries(&ledger, &rows).unwrap();
21052 let mut v2 = profiles;
21053 for p in &mut v2 {
21054 if p.model_id == real {
21055 p.fail_count = 1;
21056 p.total_calls = 1;
21057 p.task_stats.get_mut("generate").unwrap().failures = 1;
21058 }
21059 }
21060 let mut tracker = OutcomeTracker::new();
21061 tracker.import_profiles(v2);
21062 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21063 assert_eq!(profile_json(&tracker, real), expected);
21064 }
21065
21066 #[test]
21071 fn reconcile_clamps_a_broken_profile_and_keeps_ids_that_ran() {
21072 let dir = tempfile::tempdir().unwrap();
21073 let ledger = dir.path().join("outcome_ledger.jsonl");
21074 let (model, renamed) = ("parslee/reasoning", "openai/gpt-5.4");
21075 let mut writer = OutcomeTracker::new();
21076 for id in [model, renamed] {
21077 for _ in 0..4 {
21078 let t = writer.record_start(id, InferenceTask::Generate, "r");
21079 writer.record_failure(
21080 &t,
21081 "inference failed: Parslee inference returned no content",
21082 );
21083 }
21084 }
21085 let mut rows = writer.drain_ledger();
21086 let mut ok = rows[0].clone();
21087 ok.model_id = renamed.into();
21088 ok.success = Some(true);
21089 ok.error = None;
21090 rows.push(ok);
21091 outcome::append_ledger_entries(&ledger, &rows).unwrap();
21092
21093 let mut profiles = writer.export_profiles();
21094 for p in &mut profiles {
21095 if p.model_id == model {
21096 p.success_count = 1522;
21097 p.total_calls = 1519;
21098 p.fail_count = 0;
21099 p.task_stats.get_mut("generate").unwrap().failures = 0;
21100 }
21101 }
21102 let mut tracker = OutcomeTracker::new();
21103 tracker.import_profiles(profiles);
21104 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == model).unwrap();
21105 assert_eq!(report.unregistered, 0, "an id that ran is not unregistered");
21106 let p = tracker.profile(model).unwrap();
21107 assert_eq!(
21108 (p.success_count, p.fail_count, p.total_calls),
21109 (1522, 0, 1522)
21110 );
21111 assert_eq!(tracker.profile(renamed).unwrap().fail_count, 4);
21112 assert!(outcome::read_ledger(&ledger, 0)
21113 .iter()
21114 .filter(|r| r.model_id == renamed && r.success != Some(true))
21115 .all(|r| r.success == Some(false)));
21116 }
21117
21118 #[test]
21124 fn reconcile_keeps_successes_the_ledger_never_saw() {
21125 let dir = tempfile::tempdir().unwrap();
21126 let ledger = dir.path().join("outcome_ledger.jsonl");
21127 let model = "mlx/qwen3-30b-a3b:4bit";
21128 let mut writer = OutcomeTracker::new();
21129 for _ in 0..20 {
21130 let t = writer.record_start(model, InferenceTask::Generate, "r");
21131 writer.record_complete(&t, 100, 10, 20);
21132 }
21133 for refusal in admission_refusals().iter().take(3) {
21134 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
21135 writer.record_failure(&t, &refusal.to_string());
21136 }
21137 let written = writer.drain_ledger();
21139 assert!(written.iter().all(|r| r.success == Some(false)));
21140 outcome::append_ledger_entries(&ledger, &written).unwrap();
21141 let mut tracker = OutcomeTracker::new();
21142 tracker.import_profiles(writer.export_profiles());
21143 let before = tracker.profile(model).unwrap().clone();
21144 assert_eq!((before.success_count, before.fail_count), (20, 3));
21145
21146 reconcile_for_test(&ledger, &mut tracker, &|_| true).unwrap();
21147 let p = tracker.profile(model).unwrap();
21148 assert_eq!((p.success_count, p.fail_count), (20, 0));
21149 assert_eq!(p.total_calls, before.total_calls - 3);
21150 assert_eq!(p.quality_observations, 0);
21151 assert_eq!(p.ema_quality, 0.5);
21152 }
21153
21154 #[test]
21158 fn reconcile_replays_a_seeded_profile_from_its_benchmark_prior() {
21159 let dir = tempfile::tempdir().unwrap();
21160 let ledger = dir.path().join("outcome_ledger.jsonl");
21161 let model = "mlx/qwen3-4b:4bit";
21162 let mut seeded = outcome::ModelProfile::new(model.into());
21163 seeded.ema_quality = 0.7;
21164 seeded.prior_sample_size = 3;
21165 let mut writer = OutcomeTracker::new();
21166 writer.import_profiles(vec![seeded]);
21167 for refusal in admission_refusals().iter().take(2) {
21168 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
21169 writer.record_failure(&t, &refusal.to_string());
21170 }
21171 outcome::append_ledger_entries(&ledger, &writer.drain_ledger()).unwrap();
21172 let mut tracker = OutcomeTracker::new();
21173 tracker.import_profiles(writer.export_profiles());
21174 let before = tracker.profile(model).unwrap().clone();
21175 assert_eq!(before.quality_observations, 2);
21176 assert!(before.ema_quality < 0.7 * 0.8 * 0.8 + 1e-9);
21177
21178 let seeds = std::collections::HashMap::from([(
21179 model.to_string(),
21180 outcome::QualitySeed {
21181 overall: Some(0.7),
21182 tasks: std::collections::HashMap::new(),
21183 },
21184 )]);
21185 let plan = outcome::plan_reconcile(&ledger, &tracker, &|_| true, &seeds);
21186 assert_eq!(plan.report.profiles_rebuilt, 1);
21187 tracker.commit_reconciled(plan.profiles.expect("a corrected profile set"));
21188 let p = tracker.profile(model).unwrap();
21189 assert_eq!(p.quality_observations, 0);
21190 assert!(
21191 (p.ema_quality - 0.7).abs() < 1e-12,
21192 "back to the seed: {}",
21193 p.ema_quality
21194 );
21195 assert_eq!(p.prior_sample_size, 3, "the prior's weight is the prior's");
21196 }
21197
21198 #[test]
21202 fn reconcile_is_quiet_in_steady_state() {
21203 let dir = tempfile::tempdir().unwrap();
21204 let ledger = dir.path().join("outcome_ledger.jsonl");
21205 let real = "mlx/real:4bit";
21206 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
21207 let mut tracker = OutcomeTracker::new();
21208 tracker.import_profiles(profiles);
21209 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21210
21211 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21213 tracker.record_complete(&t, 120, 10, 20);
21214 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21215 tracker.record_complete(&t, 80, 10, 0);
21216 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21217 tracker.record_failure(&t, GENUINE_FAILURE);
21218 outcome::append_ledger_entries(&ledger, &tracker.drain_ledger()).unwrap();
21219 let settled = profile_json(&tracker, real);
21220
21221 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21222 assert_eq!(report, outcome::LedgerReconciliation::default());
21223 assert_eq!(profile_json(&tracker, real), settled);
21224
21225 outcome::prune_ledger(&ledger, 2).unwrap();
21228 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21229 assert_eq!(report, outcome::LedgerReconciliation::default());
21230 assert_eq!(profile_json(&tracker, real), settled);
21231 }
21232
21233 #[test]
21237 fn neutral_profile_is_not_backfilled_on_load() {
21238 let dir = tempfile::tempdir().unwrap();
21239 let path = dir.path().join("profiles.json");
21240 let mut writer = OutcomeTracker::new();
21241 let t = writer.record_start("m", InferenceTask::Generate, "r");
21242 writer.record_failure(&t, "daemon read timeout on infer after 30s");
21243 let p = writer.profile("m").unwrap();
21244 assert_eq!((p.fail_count, p.quality_observations), (1, 0));
21245 assert_eq!(p.ema_quality, 0.5);
21246 writer.save_to_file(&path).unwrap();
21247 let mut reader = OutcomeTracker::new();
21248 reader.load_from_file(&path).unwrap();
21249 assert_eq!(reader.profile("m").unwrap().quality_observations, 0);
21250 }
21251
21252 #[tokio::test]
21256 async fn reconcile_refuses_when_profiles_could_not_be_read() {
21257 let root = tempfile::tempdir().unwrap();
21258 let state = root.path().join("state");
21259 let models = state.join("models");
21260 std::fs::create_dir_all(&models).unwrap();
21261 let ledger = models.join("outcome_ledger.jsonl");
21262 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
21263 std::fs::write(models.join("outcome_profiles.json"), b"{ not json").unwrap();
21264 let before = std::fs::read(&ledger).unwrap();
21265 let engine = InferenceEngine::new(InferenceConfig {
21266 state_root: state.clone(),
21267 models_dir: root.path().join("weights"),
21268 ..InferenceConfig::default()
21269 });
21270 let report = engine.reconcile_outcome_ledger().await.unwrap();
21271 assert_eq!(report, outcome::LedgerReconciliation::default());
21272 assert_eq!(std::fs::read(&ledger).unwrap(), before, "ledger untouched");
21273 assert_eq!(
21274 std::fs::read(models.join("outcome_profiles.json")).unwrap(),
21275 b"{ not json",
21276 "profiles untouched"
21277 );
21278 }
21279
21280 #[cfg(unix)]
21283 #[tokio::test]
21284 async fn retire_plan_includes_the_projection_and_engine_refusals() {
21285 let _serial = RETIRE_LOCKS.lock().await;
21286 let root = tempfile::tempdir().unwrap();
21287 let root_path = root.path().canonicalize().unwrap();
21288 let hub = root_path.join("hub");
21289 let models = root_path.join("weights");
21290 let engine = InferenceEngine::new(InferenceConfig {
21291 state_root: root_path.join("state"),
21292 models_dir: models.clone(),
21293 ..InferenceConfig::default()
21294 });
21295 let schema = engine
21296 .unified_registry()
21297 .all()
21298 .find(|m| m.id == plain_mlx_id(&engine))
21299 .unwrap()
21300 .clone();
21301 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
21302 unreachable!()
21303 };
21304 let repo = hf_cache::repo_dir_in(&hub, hf_repo);
21305 std::fs::create_dir_all(repo.join("snapshots/s")).unwrap();
21306 std::fs::create_dir_all(repo.join("blobs")).unwrap();
21307 std::fs::write(repo.join("blobs/w"), vec![0u8; 64]).unwrap();
21308 std::os::unix::fs::symlink(repo.join("blobs/w"), repo.join("snapshots/s/w")).unwrap();
21309 std::fs::create_dir_all(&models).unwrap();
21310 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
21311
21312 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21313 assert!(!plan.is_refused(), "{plan:?}");
21314 assert_eq!(
21315 plan.freed_bytes, 64,
21316 "the link frees nothing; the repo frees its blob"
21317 );
21318 assert_eq!(
21319 plan.projections,
21320 vec![retire::Projection {
21321 path: models.join(&schema.name),
21322 receipt_for: None,
21323 }]
21324 );
21325 assert!(matches!(
21326 &plan.deletions[..],
21327 [retire::HubDeletion::Repo { .. }]
21328 ));
21329
21330 let guard = crate::download::acquire_model_lock(&schema.id).await;
21331 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21332 assert!(plan.refusals.contains(&retire::RetireRefusal::Downloading));
21333 drop(guard);
21334
21335 engine.lane_defaults_cache.write().unwrap().set(
21336 None,
21337 crate::intent::UseCase::Coding,
21338 schema.id.clone(),
21339 1,
21340 );
21341 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21342 assert!(plan
21343 .refusals
21344 .iter()
21345 .any(|r| matches!(r, retire::RetireRefusal::LaneDefault { .. })));
21346 }
21347
21348 #[cfg(unix)]
21351 #[tokio::test]
21352 async fn retire_plan_contains_projections_and_sees_sibling_downloads() {
21353 let _serial = RETIRE_LOCKS.lock().await;
21354 let root = tempfile::tempdir().unwrap();
21355 let root_path = root.path().canonicalize().unwrap();
21356 let hub = root_path.join("hub");
21357 std::fs::create_dir_all(&hub).unwrap();
21358 let mut engine = InferenceEngine::new(InferenceConfig {
21359 state_root: root_path.join("state"),
21360 models_dir: root_path.join("weights"),
21361 ..InferenceConfig::default()
21362 });
21363 let mut escaping = engine
21364 .unified_registry()
21365 .get("mlx/qwen3-4b:4bit")
21366 .unwrap()
21367 .clone();
21368 escaping.id = "user/escape".into();
21369 escaping.name = "../../outside".into();
21370 std::fs::create_dir_all(root_path.join("outside")).unwrap();
21371 engine
21372 .unified_registry_mut()
21373 .register_project_model(escaping);
21374 let plan = engine.retire_plan_in("user/escape", &hub).unwrap();
21375 assert!(plan.projections.is_empty(), "{plan:?}");
21376 assert!(plan
21377 .refusals
21378 .iter()
21379 .any(|r| matches!(r, retire::RetireRefusal::UnsafePath { .. })));
21380
21381 let guard = crate::download::acquire_model_lock("vllm-mlx/gemma-4-12b-it").await;
21383 let plan = engine
21384 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
21385 .unwrap();
21386 assert!(
21387 plan.refusals.contains(&retire::RetireRefusal::Downloading),
21388 "{plan:?}"
21389 );
21390 assert_eq!(
21392 plan.also_retires,
21393 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
21394 );
21395 drop(guard);
21396 let plan = engine
21397 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
21398 .unwrap();
21399 assert!(!plan.refusals.contains(&retire::RetireRefusal::Downloading));
21400 }
21401
21402 #[cfg(unix)]
21405 #[test]
21406 fn discovered_rows_are_retired_alone_and_their_files_are_kept() {
21407 let root = tempfile::tempdir().unwrap();
21408 let root_path = root.path().canonicalize().unwrap();
21409 let hub = root_path.join("hub");
21410 let models = root_path.join("weights");
21411 std::fs::create_dir_all(&hub).unwrap();
21412 let mut engine = InferenceEngine::new(InferenceConfig {
21413 state_root: root_path.join("state"),
21414 models_dir: models.clone(),
21415 ..InferenceConfig::default()
21416 });
21417 let base = engine
21418 .unified_registry()
21419 .get("mlx/qwen3-4b:4bit")
21420 .unwrap()
21421 .clone();
21422 for (id, name) in [("disk/one", "One-MLX"), ("disk/two", "Two-MLX")] {
21423 let mut row = base.clone();
21424 row.id = id.into();
21425 row.name = name.into();
21426 row.source = ModelSource::Mlx {
21427 hf_repo: String::new(),
21428 hf_weight_file: None,
21429 };
21430 engine.unified_registry_mut().register_project_model(row);
21431 std::fs::create_dir_all(models.join(name)).unwrap();
21432 std::fs::write(models.join(name).join("model.safetensors"), b"user").unwrap();
21433 }
21434 let plan = engine.retire_plan_in("disk/one", &hub).unwrap();
21435 assert!(plan.also_retires.is_empty(), "{plan:?}");
21436 assert!(plan.projections.is_empty(), "{plan:?}");
21437 assert_eq!(plan.user_content, vec![models.join("One-MLX")]);
21438 assert_eq!(plan.freed_bytes, 0);
21439 }
21440
21441 static RETIRE_LOCKS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
21445
21446 #[cfg(unix)]
21450 fn plain_mlx_id(engine: &InferenceEngine) -> String {
21451 let protected = engine.protected_models(&engine.unified_registry(), true);
21452 [
21453 "mlx/qwen3-1.7b:3bit",
21454 "mlx/qwen3-0.6b:6bit",
21455 "mlx/qwen3-30b-a3b:4bit",
21456 ]
21457 .into_iter()
21458 .find(|id| !protected.contains_key(*id))
21459 .unwrap()
21460 .to_string()
21461 }
21462
21463 #[cfg(unix)]
21465 fn hub_repo(hub: &Path, repo: &str, files: &[(&str, usize)]) -> PathBuf {
21466 let dir = hf_cache::repo_dir_in(hub, repo);
21467 std::fs::create_dir_all(dir.join("snapshots/s")).unwrap();
21468 std::fs::create_dir_all(dir.join("blobs")).unwrap();
21469 for (i, (path, size)) in files.iter().enumerate() {
21470 let blob = dir.join("blobs").join(format!("b{i}"));
21471 std::fs::write(&blob, vec![0u8; *size]).unwrap();
21472 let entry = dir.join("snapshots/s").join(path);
21473 std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
21474 std::os::unix::fs::symlink(&blob, &entry).unwrap();
21475 }
21476 dir
21477 }
21478
21479 fn isolated_engine(root: &Path) -> (InferenceEngine, PathBuf, PathBuf) {
21480 let hub = root.join("hub");
21481 let models = root.join("weights");
21482 std::fs::create_dir_all(&hub).unwrap();
21483 std::fs::create_dir_all(&models).unwrap();
21484 let engine = InferenceEngine::new(InferenceConfig {
21485 state_root: root.join("state"),
21486 models_dir: models.clone(),
21487 ..InferenceConfig::default()
21488 });
21489 (engine, hub, models)
21490 }
21491
21492 #[tokio::test]
21498 async fn a_swapped_signed_catalog_is_one_generation_for_every_reader() {
21499 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21500 let root = tempfile::tempdir().unwrap();
21501 let (mut engine, _hub, _models) = isolated_engine(root.path());
21502 let builtin = crate::registry::builtin_catalog()
21503 .into_iter()
21504 .find(|m| !m.is_local())
21505 .expect("a remote builtin row");
21506 let mut added = builtin.clone();
21507 added.id = "signed/swapped-in".into();
21508 added.name = "swapped-in".into();
21509 added.tags.retain(|t| t != "builtin");
21510 let mut redefined = builtin.clone();
21511 redefined.context_length = builtin.context_length + 1;
21512 let mut kept = added.clone();
21514 kept.id = "project/registered-in-memory".into();
21515 kept.name = "registered-in-memory".into();
21516 assert!(engine
21517 .unified_registry_mut()
21518 .register_project_model(kept.clone()));
21519
21520 let before = engine.unified_registry();
21521 let (first, _key) = crate::catalog::signed_test_catalog(
21522 crate::catalog::CatalogDoc {
21523 revoked: Vec::new(),
21524 version: 2,
21525 models: vec![added.clone(), redefined],
21526 },
21527 7,
21528 );
21529 assert_eq!(engine.swap_signed_catalog(first), 2);
21531 assert!(engine.list_schemas().iter().any(|m| m.id == added.id));
21532 assert!(engine.unified_registry().get(&added.id).is_some());
21533 assert_eq!(
21534 engine.model_schema("swapped-in").map(|m| m.id),
21535 Some(added.id.clone())
21536 );
21537 assert_eq!(
21538 engine
21539 .unified_registry()
21540 .get(&builtin.id)
21541 .unwrap()
21542 .context_length,
21543 builtin.context_length,
21544 "a signed row never redefines a compiled builtin's identity"
21545 );
21546 assert!(
21547 before.get(&added.id).is_none(),
21548 "an older generation is unchanged"
21549 );
21550 assert!(engine.unified_registry().get(&kept.id).is_some());
21551
21552 let (second, _key) = crate::catalog::signed_test_catalog(
21553 crate::catalog::CatalogDoc {
21554 revoked: Vec::new(),
21555 version: 3,
21556 models: vec![],
21557 },
21558 7,
21559 );
21560 assert_eq!(engine.swap_signed_catalog(second), 0);
21561 assert!(
21562 engine.unified_registry().get(&added.id).is_none(),
21563 "withdrawn"
21564 );
21565 assert!(engine.model_schema("swapped-in").is_none());
21566 assert!(engine.unified_registry().get(&builtin.id).is_some());
21567 assert!(engine.unified_registry().get(&kept.id).is_some());
21568 }
21569
21570 #[tokio::test]
21573 async fn a_withdrawn_row_a_lane_defaults_to_is_kept_deprecated() {
21574 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21575 let root = tempfile::tempdir().unwrap();
21576 let (engine, _hub, _models) = isolated_engine(root.path());
21577 let mut row = crate::registry::builtin_catalog()
21578 .into_iter()
21579 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
21580 .unwrap();
21581 row.id = "signed/lane-default".into();
21582 row.name = "lane-default".into();
21583 let catalog = |models, version| {
21584 crate::catalog::signed_test_catalog(
21585 crate::catalog::CatalogDoc {
21586 revoked: Vec::new(),
21587 version,
21588 models,
21589 },
21590 13,
21591 )
21592 .0
21593 };
21594 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
21595 engine
21596 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
21597 .unwrap();
21598 engine.swap_signed_catalog(catalog(vec![], 3));
21599 let kept = engine
21600 .unified_registry()
21601 .get(&row.id)
21602 .cloned()
21603 .expect("kept");
21604 assert!(kept.deprecated);
21605 }
21606
21607 #[tokio::test]
21610 async fn a_row_withdrawn_mid_pull_is_kept_and_recorded() {
21611 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21612 let root = tempfile::tempdir().unwrap();
21613 let (engine, _hub, _models) = isolated_engine(root.path());
21614 let mut row = crate::registry::builtin_catalog()
21615 .into_iter()
21616 .find(|m| !m.is_local())
21617 .unwrap();
21618 row.id = "signed/mid-pull".into();
21619 row.name = "mid-pull".into();
21620 let catalog = |models, version| {
21621 crate::catalog::signed_test_catalog(
21622 crate::catalog::CatalogDoc {
21623 revoked: Vec::new(),
21624 version,
21625 models,
21626 },
21627 17,
21628 )
21629 .0
21630 };
21631 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
21632 let retained = catalog::retained_path(&root.path().join("state"));
21633 engine
21634 .with_pinned_registry(async {
21635 engine.swap_signed_catalog(catalog(vec![], 3));
21636 assert!(engine.current_registry().get(&row.id).is_none());
21637 assert!(!retained.exists(), "nothing was kept yet");
21638 engine.keep_pulled_row_if_withdrawn("mid-pull");
21639 })
21640 .await;
21641 let kept = engine
21642 .unified_registry()
21643 .get(&row.id)
21644 .cloned()
21645 .expect("kept");
21646 assert!(kept.deprecated);
21647 let recorded = std::fs::read_to_string(&retained).unwrap();
21648 assert!(recorded.contains("signed/mid-pull"), "{recorded}");
21649 }
21650
21651 #[tokio::test]
21655 async fn the_scheduled_refresh_loads_only_a_newer_verified_catalog() {
21656 use std::io::{Read, Write};
21657 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21658 let root = tempfile::tempdir().unwrap();
21659 let (engine, _hub, _models) = isolated_engine(root.path());
21660 let mut row = crate::registry::builtin_catalog()
21661 .into_iter()
21662 .find(|m| !m.is_local())
21663 .unwrap();
21664 row.id = "signed/served".into();
21665 row.name = "served".into();
21666 let (published, key) = crate::catalog::signed_test_catalog(
21667 crate::catalog::CatalogDoc {
21668 revoked: Vec::new(),
21669 version: 5,
21670 models: vec![row],
21671 },
21672 31,
21673 );
21674 let serving = std::sync::Arc::new(std::sync::Mutex::new(published.envelope()));
21675 let served = serving.clone();
21676 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
21677 let url = format!("http://{}/catalog.json", listener.local_addr().unwrap());
21678 std::thread::spawn(move || {
21679 for stream in listener.incoming() {
21680 let Ok(mut stream) = stream else { return };
21681 let mut request = [0u8; 2048];
21682 let n = stream.read(&mut request).unwrap_or(0);
21683 let request = String::from_utf8_lossy(&request[..n]);
21684 let envelope = served.lock().unwrap().clone();
21685 let body = if request.starts_with("GET /catalog.json.sig ") {
21686 envelope.signature.clone()
21687 } else {
21688 envelope.signed_body.clone()
21689 };
21690 let _ = write!(
21691 stream,
21692 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
21693 body.len(),
21694 body
21695 );
21696 }
21697 });
21698 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(Some(1)));
21699 assert!(engine.unified_registry().get("signed/served").is_some());
21700 assert_eq!(
21701 engine.refresh_catalog_from(&url, &key).await,
21702 Ok(None),
21703 "nothing newer is not an error"
21704 );
21705 let status_path = catalog::status_path(&engine.config.state_root);
21707 let status = catalog::load_status(&status_path);
21708 assert_eq!(status.source.as_deref(), Some(url.as_str()));
21709 assert_eq!(status.verified_version, Some(5));
21710 assert!(status.last_verified_at.is_some() && status.failing_since.is_none());
21711 let (_, other_key) = crate::catalog::signed_test_catalog(
21712 crate::catalog::CatalogDoc {
21713 revoked: Vec::new(),
21714 version: 1,
21715 models: vec![],
21716 },
21717 32,
21718 );
21719 assert!(engine.refresh_catalog_from(&url, &other_key).await.is_err());
21720 let status = catalog::load_status(&status_path);
21723 assert!(status.failing_since.is_some(), "{status:?}");
21724 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Rejected));
21725 assert!(matches!(
21726 engine.catalog_freshness(),
21727 catalog::CatalogFreshness::Current { .. }
21728 ));
21729
21730 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
21733 assert!(catalog::load_status(&status_path).failing_since.is_none());
21734 let (older, _) = crate::catalog::signed_test_catalog(
21735 crate::catalog::CatalogDoc {
21736 revoked: Vec::new(),
21737 version: 4,
21738 models: vec![],
21739 },
21740 31,
21741 );
21742 *serving.lock().unwrap() = older.envelope();
21743 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
21744 let status = catalog::load_status(&status_path);
21745 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Older));
21746 assert!(
21747 status
21748 .last_error
21749 .as_deref()
21750 .is_some_and(|e| e.contains("older than the cached v5")),
21751 "{status:?}"
21752 );
21753
21754 let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
21756 let gone = format!("http://{}/catalog.json", closed.local_addr().unwrap());
21757 drop(closed);
21758 assert!(engine.refresh_catalog_from(&gone, &key).await.is_err());
21759 assert_eq!(
21760 catalog::load_status(&status_path).last_failure,
21761 Some(catalog::CheckFailure::Unreachable)
21762 );
21763 }
21764
21765 #[cfg(unix)]
21769 #[tokio::test]
21770 async fn a_revoked_installed_model_is_unlinked_and_not_rediscovered() {
21771 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21772 let root = tempfile::tempdir().unwrap();
21773 let root_path = root.path().canonicalize().unwrap();
21774 let (engine, _hub, models) = isolated_engine(&root_path);
21775 let mut row = crate::registry::builtin_catalog()
21776 .into_iter()
21777 .find(|m| matches!(m.source, ModelSource::Local { .. }))
21778 .expect("a downloadable local row");
21779 row.id = "signed/revoked-local".into();
21780 row.name = "signed-revoked-local".into();
21781 row.tags.retain(|t| t != "builtin");
21782 let state = root_path.join("state");
21783 let path = catalog::cache_path(&state);
21784 let doc = |models, revoked, version| catalog::CatalogDoc {
21785 version,
21786 models,
21787 revoked,
21788 };
21789 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 61);
21790 engine
21791 .adopt_verified_catalog(&path, &v2, &key)
21792 .await
21793 .unwrap();
21794
21795 let shared = root_path.join("hub-snapshot");
21797 std::fs::create_dir_all(&shared).unwrap();
21798 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
21799 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
21800 std::os::unix::fs::symlink(&shared, models.join(&row.name)).unwrap();
21801 engine.adopt_model_into_car(&row.id).await.unwrap();
21802
21803 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 61);
21804 engine
21805 .adopt_verified_catalog(&path, &v3, &key)
21806 .await
21807 .unwrap();
21808 assert!(engine.unified_registry().get(&row.id).is_none());
21809 assert!(
21810 std::fs::symlink_metadata(models.join(&row.name)).is_err(),
21811 "CAR's linkage to revoked weights is gone"
21812 );
21813 assert!(
21814 shared.join("model.gguf").exists(),
21815 "the bytes are left to the orphan rules"
21816 );
21817 let restarted =
21818 crate::registry::UnifiedRegistry::new_with_state_root(state, models.clone());
21819 assert!(
21820 restarted.list().iter().all(|m| m.name != row.name),
21821 "nothing on disk brings it back"
21822 );
21823 }
21824
21825 #[cfg(unix)]
21831 #[tokio::test]
21832 async fn revoked_models_without_receipts_never_come_back_and_keep_their_bytes() {
21833 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21834 let root = tempfile::tempdir().unwrap();
21835 let root_path = root.path().canonicalize().unwrap();
21836 let (engine, _hub, models) = isolated_engine(&root_path);
21837 let template = crate::registry::builtin_catalog()
21838 .into_iter()
21839 .find(|m| matches!(m.source, ModelSource::Local { .. }))
21840 .expect("a downloadable local row");
21841 let signed = |id: &str, name: &str| {
21842 let mut m = template.clone();
21843 m.id = id.into();
21844 m.name = name.into();
21845 m.tags.retain(|t| t != "builtin");
21846 m
21847 };
21848 let lazy = signed("signed/lazy", "signed-lazy");
21849 let copied = signed("signed/copied", "signed-copied");
21850 let state = root_path.join("state");
21851 let path = catalog::cache_path(&state);
21852 let doc = |models, revoked, version| catalog::CatalogDoc {
21853 version,
21854 models,
21855 revoked,
21856 };
21857 let (v2, key) =
21858 catalog::signed_test_catalog(doc(vec![lazy.clone(), copied.clone()], vec![], 2), 62);
21859 engine
21860 .adopt_verified_catalog(&path, &v2, &key)
21861 .await
21862 .unwrap();
21863
21864 let shared = root_path.join("hub-snapshot");
21865 std::fs::create_dir_all(&shared).unwrap();
21866 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
21867 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
21868 std::os::unix::fs::symlink(&shared, models.join(&lazy.name)).unwrap();
21869 let copy = models.join(&copied.name);
21870 std::fs::create_dir_all(©).unwrap();
21871 std::fs::write(copy.join("model.gguf"), b"copied weights").unwrap();
21872 std::fs::write(copy.join("tokenizer.json"), b"{}").unwrap();
21873
21874 let (v3, _) = catalog::signed_test_catalog(
21875 doc(vec![], vec![lazy.id.clone(), copied.id.clone()], 3),
21876 62,
21877 );
21878 engine
21879 .adopt_verified_catalog(&path, &v3, &key)
21880 .await
21881 .unwrap();
21882 assert!(
21883 std::fs::symlink_metadata(models.join(&lazy.name)).is_err(),
21884 "links unlinked"
21885 );
21886 assert!(
21887 copy.join("model.gguf").exists(),
21888 "a copy's bytes are never deleted"
21889 );
21890 let restarted =
21891 crate::registry::UnifiedRegistry::new_with_state_root(state.clone(), models.clone());
21892 assert!(
21893 restarted
21894 .list()
21895 .iter()
21896 .all(|m| m.name != lazy.name && m.name != copied.name),
21897 "neither is rediscovered"
21898 );
21899
21900 let want = vec![retire::RevokedCopy {
21901 model_id: copied.id.clone(),
21902 path: copy.clone(),
21903 bytes: (b"copied weights".len() + b"{}".len()) as u64,
21904 }];
21905 assert_eq!(retire::revoked_copies(&state, &models), want);
21906 let hub = root_path.join("hub");
21907 assert_eq!(
21908 engine
21909 .portfolio_in(&hub, &HardwareInfo::detect())
21910 .await
21911 .revoked_copies,
21912 want
21913 );
21914 let opts = doctor::DoctorOptions {
21915 deep: false,
21916 repair: false,
21917 };
21918 let report = doctor::diagnose_at_isolated(&state, &models, &hub, &opts);
21919 assert_eq!(report.revoked_copies, want);
21920 let without = doctor::DoctorReport {
21921 revoked_copies: Vec::new(),
21922 ..report.clone()
21923 };
21924 assert_eq!(
21925 report.is_healthy(),
21926 without.is_healthy(),
21927 "informational only"
21928 );
21929
21930 let outside = root_path.join("outside");
21933 std::fs::create_dir_all(&outside).unwrap();
21934 std::fs::write(outside.join("private"), b"user data").unwrap();
21935 let names_path = catalog::revoked_names_path(&state);
21936 let mut names = catalog::load_revoked_names(&names_path);
21937 names.insert("../outside".into(), copied.id.clone());
21938 catalog::save_revoked_names(&names_path, &names).unwrap();
21939 assert_eq!(retire::revoked_copies(&state, &models), want);
21940
21941 std::fs::remove_dir_all(©).unwrap();
21943 std::fs::create_dir_all(©).unwrap();
21944 std::os::unix::fs::symlink(shared.join("model.gguf"), copy.join("model.gguf")).unwrap();
21945 assert!(retire::revoked_copies(&state, &models).is_empty());
21946 }
21947
21948 #[tokio::test]
21953 async fn agents_pinning_a_revoked_model_are_reported_by_id_or_name() {
21954 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21955 let root = tempfile::tempdir().unwrap();
21956 let root_path = root.path().canonicalize().unwrap();
21957 let (engine, _hub, _models) = isolated_engine(&root_path);
21958 let mut row = crate::registry::builtin_catalog()
21959 .into_iter()
21960 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
21961 .unwrap();
21962 row.id = "signed/revoked-pinned".into();
21963 row.name = "revoked-pinned".into();
21964 let registry = engine.current_registry();
21965 let builtin = crate::registry::builtin_catalog()
21966 .into_iter()
21967 .find(|m| {
21968 !m.is_local()
21969 && registry.find_by_name(&m.name).map(|f| f.id.as_str()) == Some(m.id.as_str())
21970 })
21971 .unwrap();
21972 let fine = crate::registry::builtin_catalog()
21973 .into_iter()
21974 .find(|m| m.id != builtin.id && !m.is_local())
21975 .unwrap();
21976 let path = catalog::cache_path(&root_path.join("state"));
21977 let doc = |models, revoked, version| catalog::CatalogDoc {
21978 version,
21979 models,
21980 revoked,
21981 };
21982 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 65);
21983 engine
21984 .adopt_verified_catalog(&path, &v2, &key)
21985 .await
21986 .unwrap();
21987 let pins = std::collections::BTreeMap::from([
21988 (row.id.clone(), vec!["by-id".to_string()]),
21989 (
21990 row.name.clone(),
21991 vec!["monthly".to_string(), "digest".to_string()],
21992 ),
21993 (builtin.name.clone(), vec!["weekly".to_string()]),
21994 (fine.id.clone(), vec!["untouched".to_string()]),
21995 ]);
21996 engine.set_external_pins(pins.clone());
21997 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
21998
21999 let (v3, _) = catalog::signed_test_catalog(
22000 doc(vec![], vec![row.id.clone(), builtin.id.clone()], 3),
22001 65,
22002 );
22003 engine
22004 .adopt_verified_catalog(&path, &v3, &key)
22005 .await
22006 .unwrap();
22007 let expect = |agents: &[&str], pin: &str, model_id: &str| portfolio::RevokedPin {
22008 pin: pin.to_string(),
22009 model_id: model_id.to_string(),
22010 agents: agents.iter().map(|a| a.to_string()).collect(),
22011 };
22012 let mut want = vec![
22013 expect(&["by-id"], &row.id, &row.id),
22014 expect(&["digest", "monthly"], &row.name, &row.id),
22015 expect(&["weekly"], &builtin.name, &builtin.id),
22016 ];
22017 want.sort_by(|a, b| a.pin.cmp(&b.pin));
22018 assert_eq!(engine.revoked_pins(&engine.current_registry()), want);
22019 let state = root_path.join("state");
22022 assert!(
22023 catalog::load_revoked_names(&catalog::revoked_models_path(&state))
22024 .contains_key(&row.name)
22025 );
22026 assert!(
22027 !catalog::load_revoked_names(&catalog::revoked_names_path(&state))
22028 .contains_key(&row.name)
22029 );
22030 let hub = root_path.join("hub");
22032 let hw = HardwareInfo::detect();
22033 assert_eq!(engine.portfolio_in(&hub, &hw).await.revoked_pins, want);
22034 assert_eq!(engine.concierge_status(false).await.revoked_pins, want);
22035
22036 drop(engine);
22039 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
22040 let (restarted, _hub, _models) = isolated_engine(&root_path);
22041 std::env::remove_var("CAR_CATALOG_PUBKEY");
22042 restarted.set_external_pins(pins);
22043 assert_eq!(restarted.revoked_pins(&restarted.current_registry()), want);
22044 }
22045
22046 #[tokio::test]
22050 async fn a_pin_by_a_name_a_live_row_holds_is_not_reported() {
22051 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22052 let root = tempfile::tempdir().unwrap();
22053 let root_path = root.path().canonicalize().unwrap();
22054 let (engine, _hub, _models) = isolated_engine(&root_path);
22055 let base = crate::registry::builtin_catalog()
22056 .into_iter()
22057 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22058 .unwrap();
22059 let named = |id: &str| {
22060 let mut row = base.clone();
22061 row.id = id.into();
22062 row.name = "shared-name".into();
22063 row
22064 };
22065 let (old, fresh) = (named("signed/old"), named("signed/fresh"));
22066 let path = catalog::cache_path(&root_path.join("state"));
22067 let doc = |models, revoked, version| catalog::CatalogDoc {
22068 version,
22069 models,
22070 revoked,
22071 };
22072 let (v2, key) = catalog::signed_test_catalog(doc(vec![old.clone()], vec![], 2), 66);
22073 engine
22074 .adopt_verified_catalog(&path, &v2, &key)
22075 .await
22076 .unwrap();
22077 engine.set_external_pins(std::collections::BTreeMap::from([(
22078 "shared-name".to_string(),
22079 vec!["monthly".to_string()],
22080 )]));
22081
22082 let (v3, _) =
22083 catalog::signed_test_catalog(doc(vec![fresh.clone()], vec![old.id.clone()], 3), 66);
22084 engine
22085 .adopt_verified_catalog(&path, &v3, &key)
22086 .await
22087 .unwrap();
22088 assert!(
22089 engine.revoked_pins(&engine.current_registry()).is_empty(),
22090 "the name belongs to a live row"
22091 );
22092
22093 let (v4, _) = catalog::signed_test_catalog(doc(vec![], vec![old.id.clone()], 4), 66);
22096 engine
22097 .adopt_verified_catalog(&path, &v4, &key)
22098 .await
22099 .unwrap();
22100 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
22101 }
22102
22103 #[tokio::test]
22106 async fn a_revoked_lane_default_is_cleared_and_recorded() {
22107 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22108 let root = tempfile::tempdir().unwrap();
22109 let root_path = root.path().canonicalize().unwrap();
22110 let (engine, _hub, _models) = isolated_engine(&root_path);
22111 let mut row = crate::registry::builtin_catalog()
22112 .into_iter()
22113 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22114 .unwrap();
22115 row.id = "signed/revoked-default".into();
22116 row.name = "revoked-default".into();
22117 let other = crate::registry::builtin_catalog()
22118 .into_iter()
22119 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Code))
22120 .unwrap();
22121 let path = catalog::cache_path(&root_path.join("state"));
22122 let doc = |models, revoked, version| catalog::CatalogDoc {
22123 version,
22124 models,
22125 revoked,
22126 };
22127 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 63);
22128 engine
22129 .adopt_verified_catalog(&path, &v2, &key)
22130 .await
22131 .unwrap();
22132 engine
22133 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22134 .unwrap();
22135 engine
22136 .set_lane_default(None, crate::intent::UseCase::Coding, &other.id)
22137 .unwrap();
22138
22139 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 63);
22140 engine
22141 .adopt_verified_catalog(&path, &v3, &key)
22142 .await
22143 .unwrap();
22144 assert_eq!(
22145 engine.lane_default(None, crate::intent::UseCase::Assistant),
22146 None
22147 );
22148 assert_eq!(
22149 engine.lane_default(None, crate::intent::UseCase::Coding),
22150 Some(other.id.clone()),
22151 "a lane on another model is untouched"
22152 );
22153 let cleared = engine
22154 .concierge_actions(0)
22155 .into_iter()
22156 .find(|a| a.kind == action_ledger::ConciergeActionKind::RevokedLaneCleared)
22157 .expect("the clearing is recorded");
22158 assert_eq!(cleared.model_id, row.id);
22159 assert!(cleared.detail.contains("revoked"), "{}", cleared.detail);
22160 let now = std::time::SystemTime::now()
22162 .duration_since(std::time::UNIX_EPOCH)
22163 .unwrap()
22164 .as_secs();
22165 let shown = engine
22166 .recent_concierge_actions(now)
22167 .into_iter()
22168 .find(|a| a.kind == "lane_cleared")
22169 .expect("the clearing is in recent actions");
22170 assert_eq!(shown.subject, row.id);
22171 assert_eq!(shown.undo, None);
22172 let refused = engine
22174 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22175 .unwrap_err();
22176 assert!(refused.contains("revoked"), "{refused}");
22177 }
22178
22179 #[tokio::test]
22182 async fn a_revocation_cached_before_a_restart_still_clears_its_lane() {
22183 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22184 let root = tempfile::tempdir().unwrap();
22185 let root_path = root.path().canonicalize().unwrap();
22186 let mut row = crate::registry::builtin_catalog()
22187 .into_iter()
22188 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22189 .unwrap();
22190 row.id = "signed/revoked-while-down".into();
22191 row.name = "revoked-while-down".into();
22192 let state = root_path.join("state");
22193 let path = catalog::cache_path(&state);
22194 let doc = |models, revoked, version| catalog::CatalogDoc {
22195 version,
22196 models,
22197 revoked,
22198 };
22199 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 64);
22200 {
22201 let (engine, _hub, _models) = isolated_engine(&root_path);
22202 engine
22203 .adopt_verified_catalog(&path, &v2, &key)
22204 .await
22205 .unwrap();
22206 engine
22207 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22208 .unwrap();
22209 }
22210 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 64);
22212 catalog::save_verified(&path, &v3).unwrap();
22213 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
22214 let (engine, _hub, _models) = isolated_engine(&root_path);
22215 std::env::remove_var("CAR_CATALOG_PUBKEY");
22216 assert_eq!(
22217 engine.lane_default(None, crate::intent::UseCase::Assistant),
22218 Some(row.id.clone())
22219 );
22220 engine.enforce_catalog_revocations().await;
22221 assert_eq!(
22222 engine.lane_default(None, crate::intent::UseCase::Assistant),
22223 None
22224 );
22225 }
22226
22227 #[tokio::test]
22230 async fn a_request_reads_one_registry_generation_across_a_swap() {
22231 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22232 let root = tempfile::tempdir().unwrap();
22233 let (engine, _hub, _models) = isolated_engine(root.path());
22234 let mut added = crate::registry::builtin_catalog()
22235 .into_iter()
22236 .find(|m| !m.is_local())
22237 .unwrap();
22238 added.id = "signed/mid-request".into();
22239 added.name = "mid-request".into();
22240 let (catalog, _key) = crate::catalog::signed_test_catalog(
22241 crate::catalog::CatalogDoc {
22242 revoked: Vec::new(),
22243 version: 2,
22244 models: vec![added],
22245 },
22246 9,
22247 );
22248 engine
22249 .with_pinned_registry(async {
22250 assert!(engine.model_schema("mid-request").is_none());
22251 engine.swap_signed_catalog(catalog);
22252 assert!(
22253 engine.model_schema("mid-request").is_none(),
22254 "the request's own generation is unchanged"
22255 );
22256 assert!(engine
22257 .current_registry()
22258 .get("signed/mid-request")
22259 .is_some());
22260 })
22261 .await;
22262 assert!(engine.model_schema("mid-request").is_some());
22263 }
22264
22265 #[tokio::test]
22268 async fn a_refreshed_catalog_loads_live_and_a_stale_one_changes_nothing() {
22269 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22270 let root = tempfile::tempdir().unwrap();
22271 let (engine, _hub, _models) = isolated_engine(root.path());
22272 let template = crate::registry::builtin_catalog()
22273 .into_iter()
22274 .find(|m| !m.is_local())
22275 .unwrap();
22276 let row = |id: &str| {
22277 let mut m = template.clone();
22278 m.id = id.into();
22279 m.name = id.replace('/', "-");
22280 m
22281 };
22282 let path = crate::catalog::cache_path(&root.path().join("state"));
22283 let (v3, key) = crate::catalog::signed_test_catalog(
22284 crate::catalog::CatalogDoc {
22285 revoked: Vec::new(),
22286 version: 3,
22287 models: vec![row("signed/v3")],
22288 },
22289 11,
22290 );
22291 let (v2, _) = crate::catalog::signed_test_catalog(
22292 crate::catalog::CatalogDoc {
22293 revoked: Vec::new(),
22294 version: 2,
22295 models: vec![row("signed/v2")],
22296 },
22297 11,
22298 );
22299 assert_eq!(engine.adopt_verified_catalog(&path, &v3, &key).await, Ok(1));
22300 assert!(engine.unified_registry().get("signed/v3").is_some());
22301 assert!(engine
22302 .adopt_verified_catalog(&path, &v2, &key)
22303 .await
22304 .is_err());
22305 assert!(engine.unified_registry().get("signed/v3").is_some());
22306 assert!(engine.unified_registry().get("signed/v2").is_none());
22307 }
22308
22309 #[cfg(unix)]
22313 #[tokio::test]
22314 async fn retiring_a_whole_repo_model_frees_it_and_leaves_nothing_behind() {
22315 let _serial = RETIRE_LOCKS.lock().await;
22316 let root = tempfile::tempdir().unwrap();
22317 let root_path = root.path().canonicalize().unwrap();
22318 let (engine, hub, models) = isolated_engine(&root_path);
22319 let id = plain_mlx_id(&engine);
22320 let id = id.as_str();
22321 let schema = engine.unified_registry().get(id).unwrap().clone();
22322 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22323 unreachable!()
22324 };
22325 let repo = hub_repo(
22326 &hub,
22327 hf_repo,
22328 &[("config.json", 10), ("model.safetensors", 500)],
22329 );
22330 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22331
22332 let dry = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22333 assert!(!dry.executed);
22334 assert!(repo.exists(), "a dry run deletes nothing");
22335
22336 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22337 assert!(outcome.executed, "{outcome:?}");
22338 assert!(outcome.errors.is_empty(), "{outcome:?}");
22339 assert!(!repo.exists());
22340 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
22341 let leftovers: Vec<_> = std::fs::read_dir(&hub)
22342 .unwrap()
22343 .filter_map(Result::ok)
22344 .collect();
22345 assert!(leftovers.is_empty(), "no quarantine left: {leftovers:?}");
22346 let journals = engine
22347 .model_management_store()
22348 .management_state_dir()
22349 .join("hub-removals");
22350 assert!(std::fs::read_dir(&journals).map_or(true, |mut d| d.next().is_none()));
22351 assert!(engine
22352 .concierge_actions(10)
22353 .iter()
22354 .any(|a| a.kind == action_ledger::ConciergeActionKind::Retire && a.model_id == id));
22355
22356 let again = engine.retire_plan_in(id, &hub).unwrap();
22357 assert!(
22358 again.deletions.is_empty() && again.projections.is_empty(),
22359 "{again:?}"
22360 );
22361 }
22362
22363 #[cfg(unix)]
22366 #[tokio::test]
22367 async fn retiring_a_gguf_model_leaves_the_users_checkout_alone() {
22368 let _serial = RETIRE_LOCKS.lock().await;
22369 let root = tempfile::tempdir().unwrap();
22370 let root_path = root.path().canonicalize().unwrap();
22371 let (engine, hub, _models) = isolated_engine(&root_path);
22372 let schema = engine
22373 .unified_registry()
22374 .all()
22375 .find(|m| m.id == "qwen/qwen3-8b:q4_k_m")
22376 .unwrap()
22377 .clone();
22378 let ModelSource::Local {
22379 hf_repo,
22380 hf_filename,
22381 tokenizer_repo,
22382 } = &schema.source
22383 else {
22384 unreachable!()
22385 };
22386 let quant = hub_repo(&hub, hf_repo, &[(hf_filename.as_str(), 300)]);
22387 let base = hub_repo(
22388 &hub,
22389 tokenizer_repo,
22390 &[("tokenizer.json", 7), ("model.safetensors", 900)],
22391 );
22392
22393 let outcome = engine
22394 .retire_model_in(&schema.id, &hub, false, None)
22395 .await
22396 .unwrap();
22397 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22398 assert!(std::fs::symlink_metadata(quant.join("snapshots/s").join(hf_filename)).is_err());
22399 assert!(!quant.join("blobs/b0").exists());
22400 assert!(base.join("snapshots/s/tokenizer.json").exists());
22401 assert!(base.join("snapshots/s/model.safetensors").exists());
22402 assert_eq!(std::fs::metadata(base.join("blobs/b1")).unwrap().len(), 900);
22403 }
22404
22405 #[cfg(unix)]
22409 #[tokio::test]
22410 async fn retirement_refuses_a_changed_plan_and_a_running_download() {
22411 let _serial = RETIRE_LOCKS.lock().await;
22412 let root = tempfile::tempdir().unwrap();
22413 let root_path = root.path().canonicalize().unwrap();
22414 let (engine, hub, _models) = isolated_engine(&root_path);
22415 let id = plain_mlx_id(&engine);
22416 let id = id.as_str();
22417 let ModelSource::Mlx { hf_repo, .. } =
22418 engine.unified_registry().get(id).unwrap().source.clone()
22419 else {
22420 unreachable!()
22421 };
22422 let repo = hub_repo(&hub, &hf_repo, &[("w", 10)]);
22423 let reviewed = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22424 assert!(!reviewed.plan.digest.is_empty());
22425
22426 std::fs::write(repo.join("blobs/late"), vec![0u8; 99]).unwrap();
22427 let changed = engine
22428 .retire_model_in(id, &hub, false, Some(&reviewed.plan.digest))
22429 .await
22430 .unwrap();
22431 assert!(!changed.executed);
22432 assert!(changed
22433 .plan
22434 .refusals
22435 .contains(&retire::RetireRefusal::StateChanged));
22436 assert!(repo.join("blobs/late").exists() && repo.join("blobs/b0").exists());
22437
22438 let busy = crate::download::try_acquire_model_lock(id).unwrap();
22439 let blocked = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22440 assert!(!blocked.executed);
22441 assert!(blocked
22442 .plan
22443 .refusals
22444 .contains(&retire::RetireRefusal::Downloading));
22445 assert!(repo.exists());
22446 drop(busy);
22447
22448 let current = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22449 let done = engine
22450 .retire_model_in(id, &hub, false, Some(¤t.plan.digest))
22451 .await
22452 .unwrap();
22453 assert!(done.executed && done.errors.is_empty(), "{done:?}");
22454 assert!(!repo.exists());
22455 }
22456
22457 #[cfg(unix)]
22460 #[tokio::test]
22461 async fn a_shared_auxiliary_does_not_make_twins() {
22462 let _serial = RETIRE_LOCKS.lock().await;
22463 let root = tempfile::tempdir().unwrap();
22464 let root_path = root.path().canonicalize().unwrap();
22465 let (mut engine, hub, _models) = isolated_engine(&root_path);
22466 let ltx = engine
22467 .unified_registry()
22468 .get("mlx/ltx-2.3:q4")
22469 .unwrap()
22470 .clone();
22471 let ModelSource::Mlx {
22472 hf_repo: ltx_repo, ..
22473 } = ltx.source.clone()
22474 else {
22475 unreachable!()
22476 };
22477 let mut second = ltx.clone();
22478 second.id = "user/ltx-q8".into();
22479 second.name = "ltx-q8".into();
22480 second.source = ModelSource::Mlx {
22481 hf_repo: "someone/ltx-q8".into(),
22482 hf_weight_file: None,
22483 };
22484 engine.unified_registry_mut().register_project_model(second);
22485 let ltx_dir = hub_repo(&hub, <x_repo, &[("config.json", 5), ("w", 50)]);
22486 hub_repo(&hub, "someone/ltx-q8", &[("config.json", 5), ("w", 70)]);
22487 hub_repo(
22488 &hub,
22489 crate::registry::LTX_TEXT_ENCODER,
22490 &[("config.json", 5), ("w", 90)],
22491 );
22492 let plan = engine.retire_plan_in("mlx/ltx-2.3:q4", &hub).unwrap();
22493 assert!(plan.also_retires.is_empty(), "{plan:?}");
22494 assert!(
22495 plan.deletions.iter().all(|d| !matches!(
22496 d,
22497 retire::HubDeletion::Repo { repo, .. } if repo == "someone/ltx-q8"
22498 )),
22499 "{plan:?}"
22500 );
22501 assert!(ltx_dir.exists());
22502 }
22503
22504 #[cfg(unix)]
22506 #[tokio::test]
22507 async fn twins_sharing_a_repo_retire_together() {
22508 let _serial = RETIRE_LOCKS.lock().await;
22509 let root = tempfile::tempdir().unwrap();
22510 let root_path = root.path().canonicalize().unwrap();
22511 let (engine, hub, _models) = isolated_engine(&root_path);
22512 let id = "mlx/gemma-4-12b-it:4bit";
22513 let ModelSource::Mlx { hf_repo, .. } =
22514 engine.unified_registry().get(id).unwrap().source.clone()
22515 else {
22516 unreachable!()
22517 };
22518 let repo = hub_repo(&hub, &hf_repo, &[("config.json", 5), ("w", 50)]);
22519 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22520 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22521 assert_eq!(
22522 outcome.plan.also_retires,
22523 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
22524 );
22525 assert!(!repo.exists());
22526 let retired: Vec<String> = engine
22527 .concierge_actions(10)
22528 .into_iter()
22529 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
22530 .map(|a| a.model_id)
22531 .collect();
22532 assert!(retired.contains(&id.to_string()), "{retired:?}");
22533 assert!(
22534 retired.contains(&"vllm-mlx/gemma-4-12b-it".to_string()),
22535 "{retired:?}"
22536 );
22537
22538 let again = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22540 assert!(!again.executed);
22541 assert!(again
22542 .plan
22543 .refusals
22544 .contains(&retire::RetireRefusal::NothingToRetire));
22545 assert_eq!(
22546 engine
22547 .concierge_actions(10)
22548 .iter()
22549 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
22550 .count(),
22551 2
22552 );
22553 }
22554
22555 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22560 #[tokio::test]
22561 async fn a_receipted_projection_retires_and_the_row_stays_fetchable() {
22562 let _serial = RETIRE_LOCKS.lock().await;
22563 let root = tempfile::tempdir().unwrap();
22564 let root_path = root.path().canonicalize().unwrap();
22565 let (engine, hub, models) = isolated_engine(&root_path);
22566 let id = plain_mlx_id(&engine);
22567 let id = id.as_str();
22568 let schema = engine.unified_registry().get(id).unwrap().clone();
22569 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22570 unreachable!()
22571 };
22572 let repo = hub_repo(
22573 &hub,
22574 hf_repo,
22575 &[("config.json", 10), ("model.safetensors", 500)],
22576 );
22577 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22578 engine.adopt_model_into_car(id).await.unwrap();
22579 let store = engine.model_management_store();
22580 assert!(store.receipt_path(id).exists());
22581
22582 let plan = engine.retire_plan_in(id, &hub).unwrap();
22583 assert!(
22584 plan.projections
22585 .iter()
22586 .any(|p| p.receipt_for.as_deref() == Some(id)),
22587 "{plan:?}"
22588 );
22589 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22590 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22591 assert!(!repo.exists());
22592 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
22593 assert!(!store.receipt_path(id).exists());
22594 assert!(!store.tombstone_path(id).exists(), "retired, not disabled");
22595 assert!(store.car_enabled(id).unwrap());
22596 let intents = store.management_state_dir().join("retiring");
22597 assert!(std::fs::read_dir(&intents).map_or(true, |mut d| d.next().is_none()));
22598 }
22599
22600 #[cfg(unix)]
22602 #[test]
22603 fn a_projection_holding_a_real_file_is_not_removed() {
22604 let root = tempfile::tempdir().unwrap();
22605 let models = root.path().canonicalize().unwrap().join("weights");
22606 let projection = models.join("Some-MLX");
22607 std::fs::create_dir_all(&projection).unwrap();
22608 std::os::unix::fs::symlink("/nowhere", projection.join("link")).unwrap();
22609 std::fs::write(projection.join("notes.txt"), b"mine").unwrap();
22610 assert!(retire::remove_links_only_tree(&projection, &models).is_err());
22611 assert!(std::fs::symlink_metadata(projection.join("link")).is_ok());
22612 assert!(projection.join("notes.txt").exists());
22613 let elsewhere = root.path().canonicalize().unwrap().join("elsewhere");
22615 std::fs::create_dir_all(&elsewhere).unwrap();
22616 assert!(retire::remove_links_only_tree(&elsewhere, &models).is_err());
22617 assert!(elsewhere.exists());
22618 }
22619
22620 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22625 #[tokio::test]
22626 async fn portfolio_keeps_defaults_and_waits_for_evidence() {
22627 let _serial = RETIRE_LOCKS.lock().await;
22628 let root = tempfile::tempdir().unwrap();
22629 let root_path = root.path().canonicalize().unwrap();
22630 let (engine, hub, models) = isolated_engine(&root_path);
22631 let hw = HardwareInfo {
22632 total_ram_mb: 64 * 1024,
22633 ..HardwareInfo::detect()
22634 };
22635 let protected = engine.protected_models(&engine.unified_registry(), true);
22638 let idle_id = [
22639 "mlx/qwen3-30b-a3b:4bit",
22640 "mlx/qwen3-1.7b:3bit",
22641 "mlx/qwen3-0.6b:6bit",
22642 ]
22643 .into_iter()
22644 .find(|id| !protected.contains_key(*id))
22645 .unwrap();
22646 let mut installed = Vec::new();
22647 for id in [idle_id, "mlx/qwen3-8b:4bit"] {
22648 let schema = engine.unified_registry().get(id).unwrap().clone();
22649 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22650 unreachable!()
22651 };
22652 let repo = hub_repo(
22653 &hub,
22654 hf_repo,
22655 &[("config.json", 10), ("model.safetensors", 500)],
22656 );
22657 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
22658 .unwrap();
22659 installed.push(id);
22660 }
22661 engine.lane_defaults_cache.write().unwrap().set(
22662 None,
22663 crate::intent::UseCase::Assistant,
22664 "mlx/qwen3-8b:4bit".to_string(),
22665 1,
22666 );
22667
22668 let fresh = engine.portfolio_in(&hub, &hw).await;
22669 let ids: Vec<&str> = fresh.models.iter().map(|m| m.model_id.as_str()).collect();
22670 for id in &installed {
22671 assert!(ids.contains(id), "{ids:?}");
22672 }
22673 let verdict = |p: &portfolio::Portfolio, id: &str| {
22674 p.models.iter().find(|m| m.model_id == id).unwrap().clone()
22675 };
22676 assert_eq!(
22677 verdict(&fresh, "mlx/qwen3-8b:4bit").verdict,
22678 portfolio::Verdict::Protected
22679 );
22680 let small = verdict(&fresh, idle_id);
22681 assert_eq!(small.verdict, portfolio::Verdict::Unknown, "{small:?}");
22682 assert_eq!(small.action, portfolio::PortfolioAction::Keep);
22683
22684 engine
22686 .model_management_store()
22687 .set_usage_tracking_since_for_test(1);
22688 let aged = engine.portfolio_in(&hub, &hw).await;
22689 let small = verdict(&aged, idle_id);
22690 assert_eq!(small.verdict, portfolio::Verdict::Idle, "{small:?}");
22691 assert_eq!(small.action, portfolio::PortfolioAction::Retire);
22692 assert!(aged.reclaimable_bytes >= 510, "{aged:?}");
22693 let digest = small.retire.unwrap().digest;
22694 let done = engine
22695 .retire_model_in(idle_id, &hub, false, Some(&digest))
22696 .await
22697 .unwrap();
22698 assert!(done.executed && done.errors.is_empty(), "{done:?}");
22699 assert_eq!(
22700 verdict(&aged, "mlx/qwen3-8b:4bit").verdict,
22701 portfolio::Verdict::Protected
22702 );
22703
22704 let pinned = plain_mlx_id(&engine);
22707 engine.set_external_pins(std::collections::BTreeMap::from([(
22708 pinned.clone(),
22709 vec!["monthly".to_string()],
22710 )]));
22711 let plan = engine.retire_plan_in(&pinned, &hub).unwrap();
22712 assert!(
22713 plan.refusals.iter().any(|r| matches!(
22714 r,
22715 retire::RetireRefusal::Protected { why } if why.contains("monthly")
22716 )),
22717 "{plan:?}"
22718 );
22719 engine.set_external_pins(Default::default());
22720
22721 let tool = engine
22723 .retire_plan_in(LOCAL_TOOL_MODEL_PULL_ID, &hub)
22724 .unwrap();
22725 assert!(
22726 tool.refusals
22727 .iter()
22728 .any(|r| matches!(r, retire::RetireRefusal::Protected { .. })),
22729 "{tool:?}"
22730 );
22731 }
22732
22733 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22740 #[tokio::test]
22741 async fn maintenance_retires_under_pressure_and_respects_keep_and_churn() {
22742 use crate::update_prefs::UpdatePolicy;
22743 let _serial = RETIRE_LOCKS.lock().await;
22744 let root = tempfile::tempdir().unwrap();
22745 let root_path = root.path().canonicalize().unwrap();
22746 let (engine, hub, _models) = isolated_engine(&root_path);
22747 let hw = HardwareInfo {
22748 total_ram_mb: 64 * 1024,
22749 ..HardwareInfo::detect()
22750 };
22751 let id = plain_mlx_id(&engine);
22752 let ModelSource::Mlx { hf_repo, .. } =
22753 engine.unified_registry().get(&id).unwrap().source.clone()
22754 else {
22755 unreachable!()
22756 };
22757 let layout = || hub_repo(&hub, &hf_repo, &[("config.json", 10), ("w", 500)]);
22758 let repo = layout();
22759 engine
22760 .model_management_store()
22761 .set_usage_tracking_since_for_test(1);
22762 let gb = 1_000_000_000;
22763 let roomy = Some(maintenance::DiskSpace {
22764 free_bytes: 500 * gb,
22765 total_bytes: 1_000 * gb,
22766 });
22767 let tight = Some(maintenance::DiskSpace {
22768 free_bytes: 50 * gb,
22769 total_bytes: 1_000 * gb,
22770 });
22771 let none = crate::nudge::NudgeState::default();
22772
22773 let calm = engine
22774 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22775 .await;
22776 assert!(calm.retired.is_none(), "{calm:?}");
22777 assert!(calm.plan.held.iter().any(|h| h.model_id == id), "{calm:?}");
22778 assert!(repo.exists());
22779
22780 let notify = engine
22781 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, tight)
22782 .await;
22783 assert!(notify.retired.is_none());
22784 assert!(
22785 notify.plan.suggest.iter().any(|d| d.model_id == id),
22786 "{notify:?}"
22787 );
22788 assert!(repo.exists(), "Notify never acts");
22789
22790 let off = engine
22791 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Off, &none, tight)
22792 .await;
22793 assert_eq!(off.plan, maintenance::MaintenancePlan::default());
22794
22795 let mut kept = crate::nudge::NudgeState::default();
22796 kept.dismiss_labeled(
22797 maintenance::keep_key(&id),
22798 crate::concierge::DismissReason::Wrong,
22799 1,
22800 );
22801 let held = engine
22802 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &kept, tight)
22803 .await;
22804 assert!(held.retired.is_none());
22805 assert!(repo.exists(), "kept");
22806
22807 let pressed = engine
22808 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22809 .await;
22810 let retired = pressed.retired.as_ref().expect("retired under pressure");
22811 assert_eq!(retired.model_id, id);
22812 assert!(!repo.exists());
22813 let record = engine.portfolio_in(&hub, &hw).await.recent_actions;
22815 let entry = record
22816 .iter()
22817 .find(|a| a.kind == "retire" && a.subject == id)
22818 .expect("the retirement is on record");
22819 assert_eq!(
22820 entry.undo.as_deref(),
22821 Some(format!("car models pull {id}").as_str())
22822 );
22823 assert!(!entry.interrupted);
22824 let kinds: Vec<_> = engine
22825 .concierge_actions(10)
22826 .into_iter()
22827 .filter(|a| a.model_id == id)
22828 .map(|a| a.kind)
22829 .collect();
22830 assert!(kinds.contains(&action_ledger::ConciergeActionKind::RetireStarted));
22831 assert!(kinds.contains(&action_ledger::ConciergeActionKind::Retire));
22832
22833 let repo = layout();
22835 let again = engine
22836 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22837 .await;
22838 assert!(again.retired.is_none(), "{again:?}");
22839 assert!(again
22840 .plan
22841 .held
22842 .iter()
22843 .any(|h| h.model_id == id && h.why.contains("fetched back")));
22844 assert!(repo.exists());
22845
22846 let old = repo.join("blobs/aaaa.incomplete");
22848 let fresh = repo.join("blobs/bbbb.incomplete");
22849 std::fs::write(&old, b"half").unwrap();
22850 std::fs::write(&fresh, b"half").unwrap();
22851 let two_days_ago =
22852 std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 24 * 60 * 60);
22853 std::fs::File::options()
22854 .write(true)
22855 .open(&old)
22856 .unwrap()
22857 .set_modified(two_days_ago)
22858 .unwrap();
22859 let swept = engine
22860 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22861 .await;
22862 assert_eq!(swept.partials_discarded, vec![old.clone()], "{swept:?}");
22863 assert!(!old.exists() && fresh.exists());
22864 }
22865
22866 #[cfg(unix)]
22869 #[tokio::test]
22870 async fn maintenance_retires_an_orphan_car_fetched_only_under_pressure() {
22871 use crate::update_prefs::UpdatePolicy;
22872 let _serial = RETIRE_LOCKS.lock().await;
22873 let root = tempfile::tempdir().unwrap();
22874 let root_path = root.path().canonicalize().unwrap();
22875 let (engine, hub, models) = isolated_engine(&root_path);
22876 let hw = HardwareInfo::detect();
22877 let orphan = hub_repo(&hub, "dropped/model", &[("config.json", 10), ("w", 900)]);
22878 let theirs = hub_repo(
22879 &hub,
22880 "someone/checkout",
22881 &[("tokenizer.json", 5), ("w", 50)],
22882 );
22883 let fetched = |repo: &str, files: &[&str]| -> Vec<retire::Fetched> {
22884 files
22885 .iter()
22886 .map(|f| (repo.to_string(), f.to_string(), Some("s".to_string())))
22887 .collect()
22888 };
22889 retire::record_download(
22890 &root_path.join("state"),
22891 &hub,
22892 "old/row",
22893 &fetched("dropped/model", &["config.json", "w"]),
22894 );
22895 retire::record_download(
22896 &root_path.join("state"),
22897 &hub,
22898 "old/row",
22899 &fetched("someone/checkout", &["tokenizer.json"]),
22900 );
22901 let gb = 1_000_000_000;
22902 let none = crate::nudge::NudgeState::default();
22903 let roomy = Some(maintenance::DiskSpace {
22904 free_bytes: 500 * gb,
22905 total_bytes: 1_000 * gb,
22906 });
22907 let calm = engine
22908 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22909 .await;
22910 assert!(calm.orphan_retired.is_none());
22911 assert!(orphan.exists());
22912
22913 let tight = Some(maintenance::DiskSpace {
22914 free_bytes: 10 * gb,
22915 total_bytes: 1_000 * gb,
22916 });
22917 let link = models.join("Dropped-Model");
22920 std::os::unix::fs::symlink(orphan.join("snapshots/s"), &link).unwrap();
22921 let blocked = engine
22922 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22923 .await;
22924 assert!(blocked.orphan_retired.is_none(), "{blocked:?}");
22925 assert!(
22926 blocked
22927 .errors
22928 .iter()
22929 .any(|e| e.contains("resolves into it")),
22930 "{blocked:?}"
22931 );
22932 assert!(orphan.exists());
22933 std::fs::remove_file(&link).unwrap();
22934
22935 let pressed = engine
22936 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22937 .await;
22938 assert_eq!(
22939 pressed.orphan_retired.as_ref().map(|o| o.repo.as_str()),
22940 Some("dropped/model"),
22941 "{pressed:?}"
22942 );
22943 assert!(!orphan.exists());
22944 assert!(theirs.exists(), "a user's checkout is never touched");
22945 let again = engine
22946 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22947 .await;
22948 assert!(again.orphan_retired.is_none(), "{again:?}");
22949 assert!(theirs.exists());
22950 }
22951
22952 #[tokio::test]
22956 async fn maintenance_raises_an_unchosen_memory_policy_on_evidence() {
22957 use crate::resource_policy::{
22958 FileResourcePolicyRepository, ResourcePolicy, ResourcePolicyRepository, ResourceProfile,
22959 };
22960 use crate::update_prefs::UpdatePolicy;
22961 let _serial = RETIRE_LOCKS.lock().await;
22962 let root = tempfile::tempdir().unwrap();
22963 let root_path = root.path().canonicalize().unwrap();
22964 let hub = root_path.join("hub");
22965 std::fs::create_dir_all(&hub).unwrap();
22966 let engine = InferenceEngine::new(InferenceConfig {
22967 state_root: root_path.join("state"),
22968 models_dir: root_path.join("weights"),
22969 ..InferenceConfig::default()
22970 });
22971 let hw = HardwareInfo {
22972 total_ram_mb: 64 * 1024,
22973 ..HardwareInfo::detect()
22974 };
22975 let big = engine
22977 .unified_registry()
22978 .all()
22979 .find(|m| {
22980 m.is_local()
22981 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::everyday())).fit
22982 == recommend::ModelFitStatus::TooBig
22983 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::local_focused())).fit
22984 == recommend::ModelFitStatus::Fits
22985 })
22986 .expect("a row only Local-focused admits")
22987 .id
22988 .clone();
22989 let ledger = engine
22990 .config
22991 .state_models_dir()
22992 .join("outcome_ledger.jsonl");
22993 std::fs::create_dir_all(ledger.parent().unwrap()).unwrap();
22994 let now = std::time::SystemTime::now()
22995 .duration_since(std::time::UNIX_EPOCH)
22996 .unwrap()
22997 .as_secs();
22998 let lines: String = (0..3)
22999 .map(|i| {
23000 format!(
23001 "{}\n",
23002 serde_json::json!({
23003 "trace_id": format!("t{i}"), "model_id": big, "task": "generate",
23004 "routing_reason": "explicit model", "latency_ms": 0,
23005 "input_tokens": 0, "output_tokens": 0, "error": "refused for memory",
23006 "attribution": "resource_blocked",
23007 "timestamp": now - 60 - i * 24 * 60 * 60
23008 })
23009 )
23010 })
23011 .collect();
23012 std::fs::write(&ledger, lines).unwrap();
23013 let none = crate::nudge::NudgeState::default();
23014 let repository = FileResourcePolicyRepository::new(root_path.join("state"));
23015
23016 let unobserved = engine
23018 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23019 .await;
23020 assert!(unobserved.resource_policy.is_none(), "{unobserved:?}");
23021
23022 engine
23023 .model_management_store()
23024 .set_usage_tracking_since_for_test(now - 4 * 24 * 60 * 60);
23025 let notify = engine
23026 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, None)
23027 .await;
23028 let suggested = notify.resource_policy.expect("suggested under Notify");
23029 assert!(!suggested.applied);
23030 assert!(!repository.path().exists(), "Notify saves nothing");
23031
23032 let auto = engine
23033 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23034 .await;
23035 assert!(
23036 auto.resource_policy.as_ref().is_some_and(|p| p.applied),
23037 "{auto:?}"
23038 );
23039 assert_eq!(repository.load().unwrap(), ResourcePolicy::local_focused());
23040 assert_eq!(
23041 repository.concierge_marker(),
23042 Some(ResourcePolicy::local_focused())
23043 );
23044 assert_eq!(
23045 engine.active_local_resource_policy().policy.profile,
23046 ResourceProfile::LocalFocused
23047 );
23048
23049 let small = engine
23052 .unified_registry()
23053 .all()
23054 .find(|m| {
23055 m.is_local()
23056 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::everyday())).fit
23057 == recommend::ModelFitStatus::Fits
23058 })
23059 .unwrap()
23060 .id
23061 .clone();
23062 let later = std::time::SystemTime::now()
23063 .duration_since(std::time::UNIX_EPOCH)
23064 .unwrap()
23065 .as_secs()
23066 + 1;
23067 let mut text = std::fs::read_to_string(&ledger).unwrap();
23068 for i in 0..3 {
23069 text.push_str(&format!(
23070 "{}\n",
23071 serde_json::json!({
23072 "trace_id": format!("live{i}"), "model_id": small, "task": "generate",
23073 "routing_reason": "explicit model", "latency_ms": 0,
23074 "input_tokens": 0, "output_tokens": 0, "error": "not enough memory free",
23075 "attribution": "resource_blocked", "timestamp": later
23076 })
23077 ));
23078 }
23079 std::fs::write(&ledger, text).unwrap();
23080 let reverted = engine
23081 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23082 .await;
23083 let undo = reverted.resource_policy.expect("reverted");
23084 assert!(
23085 undo.applied && undo.to == ResourceProfile::Everyday,
23086 "{undo:?}"
23087 );
23088 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
23089 assert!(repository.concierge_choice().unwrap().reverted);
23090 let settled = engine
23091 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23092 .await;
23093 assert!(settled.resource_policy.is_none(), "{settled:?}");
23094
23095 repository.save(&ResourcePolicy::everyday()).unwrap();
23097 repository.clear_concierge_marker().unwrap();
23098 engine.apply_local_resource_policy(ResourcePolicy::everyday());
23099 let chosen = engine
23100 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23101 .await;
23102 assert!(chosen.resource_policy.is_none(), "{chosen:?}");
23103 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
23104 }
23105
23106 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
23112 #[tokio::test]
23113 async fn a_curated_upgrade_switches_the_lane_that_used_the_old_model() {
23114 let _serial = RETIRE_LOCKS.lock().await;
23115 let root = tempfile::tempdir().unwrap();
23116 let root_path = root.path().canonicalize().unwrap();
23117 let (engine, hub, models) = isolated_engine(&root_path);
23118 let from = plain_mlx_id(&engine);
23119 let to = "mlx/qwen3-8b:4bit".to_string();
23120 for id in [&from, &to] {
23121 let schema = engine.unified_registry().get(id).unwrap().clone();
23122 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23123 unreachable!()
23124 };
23125 let repo = hub_repo(
23126 &hub,
23127 hf_repo,
23128 &[("config.json", 10), ("model.safetensors", 500)],
23129 );
23130 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
23131 .unwrap();
23132 }
23133 let finding = |source| crate::upgrade::UpgradeFinding {
23134 from_id: from.clone(),
23135 from_name: from.clone(),
23136 to_id: to.clone(),
23137 to_name: to.clone(),
23138 reason: "newer".into(),
23139 trust_tier: crate::schema::TrustTier::Curated,
23140 source,
23141 target_pullable: true,
23142 };
23143 let now = 1;
23144 let none = crate::nudge::NudgeState::default();
23145
23146 let idle = engine
23148 .apply_one_curated_upgrade_from(
23149 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23150 &hub,
23151 &none,
23152 now,
23153 )
23154 .await
23155 .unwrap();
23156 assert!(idle.is_none());
23157
23158 engine
23159 .set_lane_default(None, crate::intent::UseCase::Assistant, &from)
23160 .unwrap();
23161 let community = engine
23163 .apply_one_curated_upgrade_from(
23164 vec![finding(crate::upgrade::UpgradeSource::Upstream)],
23165 &hub,
23166 &none,
23167 now,
23168 )
23169 .await
23170 .unwrap();
23171 assert!(community.is_none());
23172 let mut dismissed = crate::nudge::NudgeState::default();
23173 dismissed.dismiss(&format!("{from}=>{to}"));
23174 assert!(engine
23175 .apply_one_curated_upgrade_from(
23176 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23177 &hub,
23178 &dismissed,
23179 now,
23180 )
23181 .await
23182 .unwrap()
23183 .is_none());
23184
23185 let upgraded = engine
23186 .apply_one_curated_upgrade_from(
23187 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23188 &hub,
23189 &none,
23190 now,
23191 )
23192 .await
23193 .unwrap()
23194 .expect("upgraded");
23195 assert_eq!(upgraded.lanes, vec!["assistant".to_string()]);
23196 assert_eq!(
23197 engine
23198 .lane_default(None, crate::intent::UseCase::Assistant)
23199 .as_deref(),
23200 Some(to.as_str())
23201 );
23202 let record = engine.recent_concierge_actions(now);
23203 let entry = record
23204 .iter()
23205 .find(|a| a.kind == "upgrade")
23206 .expect("on record");
23207 assert_eq!(
23208 entry.undo.as_deref(),
23209 Some("car concierge rollback assistant")
23210 );
23211 assert_eq!(entry.use_case.as_deref(), Some("assistant"));
23212 assert_eq!(entry.prior.as_deref(), Some(from.as_str()));
23213 }
23214
23215 #[tokio::test]
23218 async fn concierge_ask_without_a_question_states_the_facts_without_a_model() {
23219 let _serial = RETIRE_LOCKS.lock().await;
23220 let root = tempfile::tempdir().unwrap();
23221 let engine = InferenceEngine::new(InferenceConfig {
23222 state_root: root.path().join("state"),
23223 models_dir: root.path().join("weights"),
23224 ..InferenceConfig::default()
23225 });
23226 let answer = engine.concierge_ask(None).await.unwrap();
23227 assert!(answer.model_used.is_none());
23228 assert_eq!(answer.answer, answer.summary);
23229 assert!(
23230 answer.summary.contains("Installed local models:"),
23231 "{}",
23232 answer.summary
23233 );
23234 assert!(answer.summary.contains("Memory policy for local models:"));
23235 }
23236
23237 #[cfg(unix)]
23239 #[tokio::test]
23240 async fn a_refused_retirement_deletes_nothing() {
23241 let _serial = RETIRE_LOCKS.lock().await;
23242 let root = tempfile::tempdir().unwrap();
23243 let root_path = root.path().canonicalize().unwrap();
23244 let (engine, hub, _models) = isolated_engine(&root_path);
23245 let id = "mlx/qwen3-4b:4bit";
23246 let schema = engine.unified_registry().get(id).unwrap().clone();
23247 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23248 unreachable!()
23249 };
23250 let repo = hub_repo(&hub, hf_repo, &[("w", 10)]);
23251 engine.lane_defaults_cache.write().unwrap().set(
23252 None,
23253 crate::intent::UseCase::Assistant,
23254 id.to_string(),
23255 1,
23256 );
23257 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23258 assert!(!outcome.executed);
23259 assert!(repo.exists());
23260 }
23261
23262 #[test]
23265 fn retire_plan_refuses_a_model_stored_outside_the_hub() {
23266 let root = tempfile::tempdir().unwrap();
23267 let engine = InferenceEngine::new(InferenceConfig {
23268 state_root: root.path().join("state"),
23269 models_dir: root.path().join("weights"),
23270 ..InferenceConfig::default()
23271 });
23272 let whisper = engine
23273 .unified_registry()
23274 .all()
23275 .find(|m| matches!(m.source, ModelSource::WhisperCpp { .. }))
23276 .expect("a whisper.cpp row")
23277 .id
23278 .clone();
23279 let plan = engine.retire_plan_in(&whisper, root.path()).unwrap();
23280 assert!(
23281 matches!(
23282 &plan.refusals[..],
23283 [retire::RetireRefusal::Unsupported { .. }]
23284 ),
23285 "{plan:?}"
23286 );
23287 }
23288
23289 #[test]
23291 fn downloads_are_recorded_per_hub() {
23292 let root = tempfile::tempdir().unwrap();
23293 let fetched = [(
23294 "renamed/repo".to_string(),
23295 "model.safetensors".to_string(),
23296 Some("r1".to_string()),
23297 )];
23298 let hub = root.path().join("hub");
23299 retire::record_download(root.path(), &hub, "old/row", &fetched);
23300 let recorded = retire::recorded_files(root.path(), &hub);
23301 assert_eq!(
23302 recorded.get("renamed/repo"),
23303 Some(&retire::Owned::Files(std::collections::BTreeSet::from([
23304 retire::OwnedFile {
23305 path: "model.safetensors".into(),
23306 revision: Some("r1".into()),
23307 }
23308 ])))
23309 );
23310 assert!(retire::recorded_files(root.path(), Path::new("/elsewhere")).is_empty());
23311 }
23312
23313 #[cfg(unix)]
23318 #[tokio::test]
23319 async fn a_failed_ledger_write_leaves_the_profiles_untouched_and_retries_once() {
23320 use std::os::unix::fs::PermissionsExt;
23321 let root = tempfile::tempdir().unwrap();
23322 let state = root.path().join("state");
23323 let models = state.join("models");
23324 std::fs::create_dir_all(&models).unwrap();
23325 let ledger = models.join("outcome_ledger.jsonl");
23326 let real = "mlx/qwen3-4b:4bit";
23327 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
23328 let mut seed = OutcomeTracker::new();
23329 seed.import_profiles(profiles);
23330 seed.save_to_file(&models.join("outcome_profiles.json"))
23331 .unwrap();
23332 let engine = InferenceEngine::new(InferenceConfig {
23333 state_root: state.clone(),
23334 models_dir: root.path().join("weights"),
23335 ..InferenceConfig::default()
23336 });
23337 let before = profile_json(&*engine.outcome_tracker().read().await, real);
23338 let dirty_before = engine.outcome_tracker().read().await.is_dirty();
23339 assert!(
23340 engine
23341 .outcome_tracker()
23342 .read()
23343 .await
23344 .profile(real)
23345 .unwrap()
23346 .fail_count
23347 > 1
23348 );
23349
23350 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o555)).unwrap();
23352 let failed = engine.reconcile_outcome_ledger().await;
23353 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o755)).unwrap();
23354 assert!(failed.is_err(), "the write was refused");
23355 {
23356 let tracker = engine.outcome_tracker();
23357 let tracker = tracker.read().await;
23358 assert_eq!(profile_json(&tracker, real), before, "profiles untouched");
23359 assert_eq!(
23360 tracker.is_dirty(),
23361 dirty_before,
23362 "the failed reconcile left nothing new for the periodic save to persist"
23363 );
23364 }
23365
23366 engine.reconcile_outcome_ledger().await.unwrap();
23368 engine.reconcile_outcome_ledger().await.unwrap();
23369 let p = engine
23370 .outcome_tracker()
23371 .read()
23372 .await
23373 .profile(real)
23374 .unwrap()
23375 .clone();
23376 assert_eq!(
23377 p.fail_count, 1,
23378 "only the genuine failure remains, taken back once"
23379 );
23380 }
23381
23382 #[test]
23385 fn engine_construction_never_rewrites_the_ledger() {
23386 let root = tempfile::tempdir().unwrap();
23387 let state = root.path().join("state");
23388 let models = state.join("models");
23389 std::fs::create_dir_all(&models).unwrap();
23390 let ledger = models.join("outcome_ledger.jsonl");
23391 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
23392 let before = std::fs::read(&ledger).unwrap();
23393 let _engine = InferenceEngine::new(InferenceConfig {
23394 state_root: state,
23395 models_dir: root.path().join("weights"),
23396 ..InferenceConfig::default()
23397 });
23398 assert_eq!(std::fs::read(&ledger).unwrap(), before);
23399 }
23400
23401 #[test]
23402 fn dispatch_outcome_guards_keep_real_failures_and_existing_rejections_distinct() {
23403 let model_id = "test/model";
23404
23405 let real_failure = InferenceError::InferenceFailed("provider returned HTTP 500".into());
23406 let mut tracker = OutcomeTracker::new();
23407 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
23408 record_dispatch_failure(&mut tracker, &trace, &real_failure);
23409 let profile = tracker.profile(model_id).expect("real failure profile");
23410 assert_eq!(profile.success_count, 0);
23411 assert_eq!(profile.fail_count, 1);
23412 let receipt = tracker.drain_ledger().pop().unwrap();
23413 assert_eq!(receipt.success, Some(false));
23414 assert!(error_counts_against_circuit_breaker(&real_failure));
23415 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
23416 breakers.record_failure(model_id);
23417 assert_eq!(
23418 breakers.state(model_id),
23419 Some(crate::routing_ext::CircuitState::Open)
23420 );
23421
23422 for rejection in [
23423 InferenceError::GatewayUnconfigured {
23424 provider: "parslee".into(),
23425 namespace: "parslee/openrouter/".into(),
23426 status: 503,
23427 message: "not configured".into(),
23428 },
23429 InferenceError::ContentRefused {
23430 provider: "parslee".into(),
23431 kind: Some("invalid_request_error".into()),
23432 code: Some("content_policy_violation".into()),
23433 message: "content refused".into(),
23434 },
23435 ] {
23436 let mut tracker = OutcomeTracker::new();
23437 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
23438 record_dispatch_failure(&mut tracker, &trace, &rejection);
23439 assert!(
23440 tracker.profile(model_id).is_none(),
23441 "existing rejection changed model profile: {rejection}"
23442 );
23443 let receipt = tracker.drain_ledger().pop().unwrap();
23444 assert_eq!(receipt.success, None, "{rejection}");
23445 assert_eq!(receipt.quality, None, "{rejection}");
23446 assert!(!error_counts_against_circuit_breaker(&rejection));
23447 }
23448 }
23449
23450 #[tokio::test(flavor = "current_thread")]
23463 async fn openrouter_out_of_credits_does_not_degrade_the_model() {
23464 use wiremock::matchers::{method, path};
23465 use wiremock::{Mock, MockServer, ResponseTemplate};
23466
23467 let _credential_scope = crate::openrouter::test_credential_scope();
23468 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
23469 let server = MockServer::start().await;
23470 Mock::given(method("POST"))
23471 .and(path("/v1/chat/completions"))
23472 .respond_with(
23473 ResponseTemplate::new(402)
23474 .set_body_string(r#"{"error":{"code":402,"message":"Insufficient credits"}}"#),
23475 )
23476 .mount(&server)
23477 .await;
23478
23479 let tmp = TempDir::new().unwrap();
23480 let models_dir = tmp.path().join("models");
23481 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
23482 let mut schema = crate::openrouter::curated_schemas()
23483 .into_iter()
23484 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
23485 .unwrap();
23486 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
23487 *endpoint = server.uri();
23488 }
23489 let model_id = schema.id.clone();
23490 engine.unified_registry_mut().register_project_model(schema);
23491
23492 let err = engine
23493 .generate_tracked(GenerateRequest {
23494 prompt: "bill me".into(),
23495 model: Some(model_id.clone()),
23496 params: GenerateParams {
23497 strict_model: true,
23498 ..Default::default()
23499 },
23500 ..Default::default()
23501 })
23502 .await
23503 .expect_err("402 must not succeed");
23504
23505 match &err {
23506 InferenceError::ProviderAccount {
23507 provider, status, ..
23508 } => {
23509 assert_eq!(status, &402);
23510 assert_eq!(provider, "openrouter");
23511 }
23512 other => panic!("expected ProviderAccount, got {other:?}"),
23513 }
23514 assert!(
23515 !error_counts_against_circuit_breaker(&err),
23516 "an account rejection must not feed the per-model breaker"
23517 );
23518
23519 let tracker_handle = engine.outcome_tracker();
23521 let tracker = tracker_handle.read().await;
23522 let profile = tracker.profile(&model_id).cloned();
23523 assert!(
23524 profile.as_ref().is_none_or(|p| p.fail_count == 0),
23525 "account rejection degraded the model profile: {profile:?}"
23526 );
23527 drop(tracker);
23528
23529 let ledger = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
23533 let entry = ledger
23534 .iter()
23535 .find(|e| e.model_id == model_id)
23536 .expect("the attempt must still produce a receipt");
23537 assert!(
23538 entry.success.is_none() && entry.quality.is_none(),
23539 "receipt must record the attempt without a verdict: {entry:?}"
23540 );
23541
23542 assert!(
23544 engine
23545 .adaptive_router
23546 .circuit_breakers
23547 .lock()
23548 .unwrap()
23549 .state(&model_id)
23550 .is_none(),
23551 "account rejection must not create breaker state for the model"
23552 );
23553 }
23554
23555 #[tokio::test(flavor = "current_thread")]
23556 async fn openrouter_stream_account_error_is_unattributed_and_never_completes_successfully() {
23557 use wiremock::matchers::{method, path};
23558 use wiremock::{Mock, MockServer, ResponseTemplate};
23559
23560 let _credential_scope = crate::openrouter::test_credential_scope();
23561 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
23562 let server = MockServer::start().await;
23563 Mock::given(method("POST"))
23564 .and(path("/v1/chat/completions"))
23565 .respond_with(ResponseTemplate::new(200).set_body_raw(
23566 "data: {\"error\":{\"code\":402,\"message\":\"private balance details\"}}\n\n",
23567 "text/event-stream",
23568 ))
23569 .mount(&server)
23570 .await;
23571
23572 let tmp = TempDir::new().unwrap();
23573 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23574 let mut schema = crate::openrouter::curated_schemas()
23575 .into_iter()
23576 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
23577 .unwrap();
23578 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
23579 *endpoint = server.uri();
23580 }
23581 let model_id = schema.id.clone();
23582 engine.unified_registry_mut().register_project_model(schema);
23583
23584 let mut handle = engine
23585 .generate_tracked_stream(GenerateRequest {
23586 prompt: "fail after headers".into(),
23587 model: Some(model_id.clone()),
23588 ..Default::default()
23589 })
23590 .await
23591 .unwrap();
23592 let mut events = Vec::new();
23593 while let Some(event) = handle.events.recv().await {
23594 events.push(event);
23595 }
23596 assert!(matches!(
23597 events.as_slice(),
23598 [StreamEvent::Error(message)] if message == "OpenRouter account is out of credits"
23599 ));
23600 assert!(!events
23601 .iter()
23602 .any(|event| matches!(event, StreamEvent::Done { .. })));
23603
23604 let mut receipts = Vec::new();
23607 for _ in 0..50 {
23608 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
23609 if !receipts.is_empty() {
23610 break;
23611 }
23612 tokio::task::yield_now().await;
23613 }
23614 let [receipt] = receipts.as_slice() else {
23615 panic!("expected one receipt, got {receipts:?}");
23616 };
23617 assert_eq!(receipt.success, None);
23618 assert_eq!(
23619 receipt.attribution,
23620 Some(crate::outcome::UnattributedReason::Account)
23621 );
23622 assert!(
23623 engine
23624 .outcome_tracker()
23625 .read()
23626 .await
23627 .profile(&model_id)
23628 .is_none(),
23629 "an account rejection must not touch the model's profile"
23630 );
23631 }
23632
23633 #[tokio::test(flavor = "current_thread")]
23636 async fn tracked_stream_dropped_by_the_caller_is_cancelled_not_a_failure() {
23637 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23638 let tmp = TempDir::new().unwrap();
23639 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23640 pin_test_local_admission_memory(&mut engine);
23641 let model_id = install_small_local_fixture(&engine);
23642 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
23643
23644 let stream = engine
23645 .generate_tracked_stream(GenerateRequest {
23646 prompt: "interrupted".into(),
23647 model: Some(model_id.clone()),
23648 params: GenerateParams {
23649 strict_model: true,
23650 ..Default::default()
23651 },
23652 ..Default::default()
23653 })
23654 .await
23655 .unwrap();
23656 drop(stream);
23657
23658 let mut receipts = Vec::new();
23659 for _ in 0..200 {
23660 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
23661 if !receipts.is_empty() {
23662 break;
23663 }
23664 tokio::task::yield_now().await;
23665 }
23666 crate::offload::set_local_offload(None);
23667 let [receipt] = receipts.as_slice() else {
23668 panic!("expected one receipt, got {receipts:?}");
23669 };
23670 assert_eq!(receipt.success, None);
23671 assert_eq!(
23672 receipt.attribution,
23673 Some(crate::outcome::UnattributedReason::Cancelled)
23674 );
23675 assert!(engine
23676 .outcome_tracker()
23677 .read()
23678 .await
23679 .profile(&model_id)
23680 .is_none_or(|p| p.fail_count == 0));
23681 }
23682
23683 #[test]
23687 fn an_apple_guardrail_decline_does_not_end_the_fallback_chain() {
23688 let refusal = |provider: &str| InferenceError::ContentRefused {
23689 provider: provider.into(),
23690 kind: Some("guardrailViolation".into()),
23691 code: None,
23692 message: "declined".into(),
23693 };
23694 assert!(!error_ends_fallback_chain(&refusal("apple")));
23695 assert!(error_ends_fallback_chain(&refusal("parslee")));
23696 assert!(!error_counts_against_circuit_breaker(&refusal("apple")));
23697 }
23698
23699 #[tokio::test(flavor = "current_thread")]
23700 async fn tracked_stream_without_done_records_failure_not_success() {
23701 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23702 let tmp = TempDir::new().unwrap();
23703 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23704 pin_test_local_admission_memory(&mut engine);
23705 let model_id = install_small_local_fixture(&engine);
23706 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: false })));
23707
23708 let mut stream = engine
23709 .generate_tracked_stream(GenerateRequest {
23710 prompt: "must not count as success".into(),
23711 model: Some(model_id.clone()),
23712 params: GenerateParams {
23713 strict_model: true,
23714 ..Default::default()
23715 },
23716 ..Default::default()
23717 })
23718 .await
23719 .unwrap();
23720 while stream.events.recv().await.is_some() {}
23721 crate::offload::set_local_offload(None);
23722
23723 for _ in 0..50 {
23724 if engine
23725 .outcome_tracker()
23726 .read()
23727 .await
23728 .profile(&model_id)
23729 .is_some_and(|profile| profile.fail_count == 1)
23730 {
23731 break;
23732 }
23733 tokio::task::yield_now().await;
23734 }
23735 let profile = engine
23736 .outcome_tracker()
23737 .read()
23738 .await
23739 .profile(&model_id)
23740 .cloned()
23741 .unwrap();
23742 assert_eq!(profile.fail_count, 1);
23743 assert_eq!(profile.success_count, 0);
23744 }
23745
23746 #[tokio::test(flavor = "current_thread")]
23747 async fn google_vertex_terminal_matrix_records_only_deliberate_finishes_as_success() {
23748 use wiremock::matchers::{method, path};
23749 use wiremock::{Mock, MockServer, ResponseTemplate};
23750
23751 let _provider_env = crate::openrouter::test_environment_scope_async().await;
23752 let _env = ENV_MUTEX.lock().await;
23753 unsafe { std::env::set_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY", "matrix-key") };
23754
23755 for (protocol, reason, should_succeed) in [
23756 (schema::ApiProtocol::Google, "STOP", true),
23757 (schema::ApiProtocol::Google, "SAFETY", false),
23758 (schema::ApiProtocol::VertexAi, "MAX_TOKENS", true),
23759 (
23760 schema::ApiProtocol::VertexAi,
23761 "MALFORMED_FUNCTION_CALL",
23762 false,
23763 ),
23764 ] {
23765 let server = MockServer::start().await;
23766 let expected_path = match protocol {
23767 schema::ApiProtocol::Google => "/v1beta/models/gemini-test:streamGenerateContent",
23768 schema::ApiProtocol::VertexAi => {
23769 "/publishers/google/models/gemini-test:streamGenerateContent"
23770 }
23771 _ => unreachable!(),
23772 };
23773 Mock::given(method("POST"))
23774 .and(path(expected_path))
23775 .respond_with(ResponseTemplate::new(200).set_body_raw(
23776 format!(
23777 "data: {{\"candidates\":[{{\"content\":{{\"parts\":[{{\"text\":\"matrix\"}}]}},\"finishReason\":\"{reason}\"}}]}}\n\n"
23778 ),
23779 "text/event-stream",
23780 ))
23781 .mount(&server)
23782 .await;
23783
23784 let id = format!("test/{protocol:?}-{reason}");
23785 let tmp = TempDir::new().unwrap();
23786 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23787 engine.register_model(remote_stream_fixture_schema(
23788 &id,
23789 server.uri(),
23790 protocol,
23791 "CAR_GOOGLE_OUTCOME_MATRIX_KEY",
23792 ));
23793 let mut stream = engine
23794 .generate_tracked_stream(GenerateRequest {
23795 prompt: "matrix".into(),
23796 model: Some(id.clone()),
23797 params: GenerateParams {
23798 strict_model: true,
23799 ..Default::default()
23800 },
23801 ..Default::default()
23802 })
23803 .await
23804 .unwrap();
23805 let mut events = Vec::new();
23806 while let Some(event) = stream.events.recv().await {
23807 events.push(event);
23808 }
23809 for _ in 0..50 {
23810 if engine
23811 .outcome_tracker()
23812 .read()
23813 .await
23814 .profile(&id)
23815 .is_some_and(|profile| profile.total_calls == 1)
23816 {
23817 break;
23818 }
23819 tokio::task::yield_now().await;
23820 }
23821 let profile = engine
23822 .outcome_tracker()
23823 .read()
23824 .await
23825 .profile(&id)
23826 .cloned()
23827 .unwrap();
23828 assert_eq!(
23829 (profile.success_count, profile.fail_count),
23830 if should_succeed { (1, 0) } else { (0, 1) },
23831 "{protocol:?}/{reason}: {events:?}"
23832 );
23833 }
23834
23835 unsafe { std::env::remove_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY") };
23836 }
23837
23838 #[tokio::test(flavor = "current_thread")]
23839 async fn remote_primary_stream_setup_failure_falls_back_to_installed_local_dispatch() {
23840 use wiremock::matchers::{method, path};
23841 use wiremock::{Mock, MockServer, ResponseTemplate};
23842
23843 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23844 let _env = ENV_MUTEX.lock().await;
23845 let server = MockServer::start().await;
23846 Mock::given(method("POST"))
23847 .and(path("/v1/chat/completions"))
23848 .respond_with(ResponseTemplate::new(503))
23849 .mount(&server)
23850 .await;
23851 unsafe { std::env::set_var("CAR_STREAM_FALLBACK_TEST_KEY", "fixture") };
23852
23853 let tmp = TempDir::new().unwrap();
23854 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23855 pin_test_local_admission_memory(&mut engine);
23856 let _local_id = install_small_local_fixture(&engine);
23857 let remote_id = "test/remote-primary";
23858 engine.register_model(ModelSchema {
23859 id: remote_id.into(),
23860 name: "remote-primary".into(),
23861 provider: "test".into(),
23862 family: "test".into(),
23863 version: "1".into(),
23864 capabilities: vec![ModelCapability::Generate],
23865 context_length: 8_192,
23866 max_output_tokens: Some(1_024),
23867 param_count: String::new(),
23868 quantization: None,
23869 performance: Default::default(),
23870 cost: Default::default(),
23871 source: ModelSource::RemoteApi {
23872 endpoint: server.uri(),
23873 api_key_env: "CAR_STREAM_FALLBACK_TEST_KEY".into(),
23874 api_key_envs: vec![],
23875 api_version: None,
23876 protocol: schema::ApiProtocol::OpenAiCompat,
23877 },
23878 tags: vec![],
23879 supported_params: vec![],
23880 public_benchmarks: vec![],
23881 trust_tier: TrustTier::Community,
23882 deprecated: false,
23883 available: true,
23884 weights_ready: true,
23885 });
23886 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
23887
23888 let mut stream = engine
23889 .generate_tracked_stream(GenerateRequest {
23890 prompt: "fall back".into(),
23891 model: Some(remote_id.into()),
23892 ..Default::default()
23893 })
23894 .await
23895 .expect("compatible installed local model should be dispatched");
23896 assert_ne!(stream.model_used, remote_id);
23897 assert!(
23898 engine
23899 .unified_registry()
23900 .get(&stream.model_used)
23901 .is_some_and(ModelSchema::is_local),
23902 "fallback must stay on a compatible local model: {}",
23903 stream.model_used
23904 );
23905 let mut saw_done = false;
23906 while let Some(event) = stream.events.recv().await {
23907 saw_done |= matches!(event, StreamEvent::Done { .. });
23908 }
23909 assert!(saw_done);
23910
23911 crate::offload::set_local_offload(None);
23912 unsafe { std::env::remove_var("CAR_STREAM_FALLBACK_TEST_KEY") };
23913 }
23914
23915 #[tokio::test(flavor = "current_thread")]
23916 async fn exact_model_id_rejects_display_name_without_dispatch() {
23917 let tmp = TempDir::new().unwrap();
23918 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23919 let mut first = remote_stream_fixture_schema(
23920 "test/ambiguous-a:1",
23921 "http://127.0.0.1:9".into(),
23922 schema::ApiProtocol::OpenAiCompat,
23923 "CAR_AMBIGUOUS_EXACT_KEY",
23924 );
23925 first.name = "Shared Alias".into();
23926 let mut second = first.clone();
23927 second.id = "test/ambiguous-b:1".into();
23928 engine.register_model(first);
23929 engine.register_model(second);
23930 let mut request = GenerateRequest {
23931 prompt: "must not dispatch".into(),
23932 ..Default::default()
23933 };
23934 pin_exact_model_id(&mut request, "Shared Alias".into()).unwrap();
23935
23936 let error = engine
23937 .generate_tracked(request)
23938 .await
23939 .expect_err("an exact-id pin must never resolve a display name");
23940 assert!(matches!(error, InferenceError::ModelNotFound(_)));
23941 }
23942
23943 #[tokio::test(flavor = "current_thread")]
23944 async fn exact_openai_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
23945 let _env = ENV_MUTEX.lock().await;
23946 assert_remote_model_identity_contract(
23947 schema::ApiProtocol::OpenAiCompat,
23948 "openai/newsroom-gpt-5.5-2026-04-23:test",
23949 "newsroom-gpt-5.5-2026-04-23-test",
23950 "CAR_OPENAI_IDENTITY_TEST_KEY",
23951 )
23952 .await;
23953 }
23954
23955 #[tokio::test(flavor = "current_thread")]
23956 async fn exact_anthropic_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
23957 let _env = ENV_MUTEX.lock().await;
23958 assert_remote_model_identity_contract(
23959 schema::ApiProtocol::Anthropic,
23960 "anthropic/newsroom-claude-opus-4-8:test",
23961 "newsroom-claude-opus-4-8-test",
23962 "CAR_ANTHROPIC_IDENTITY_TEST_KEY",
23963 )
23964 .await;
23965 }
23966
23967 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
23968 #[tokio::test(flavor = "current_thread")]
23969 async fn exact_model_id_nonstream_bypasses_mlx_equivalent_substitution() {
23970 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23971 let tmp = TempDir::new().unwrap();
23972 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23973 let exact_id = install_exact_pin_equivalent_fixture(&engine);
23974 let offload = Arc::new(ExactPinCaptureOffload::new());
23975 crate::offload::set_local_offload(Some(offload.clone()));
23976
23977 let mut request = GenerateRequest {
23978 prompt: "use only the immutable row I selected".into(),
23979 ..Default::default()
23980 };
23981 pin_exact_model_id(&mut request, exact_id.clone()).unwrap();
23982 let error = engine.generate_tracked(request).await.unwrap_err();
23983
23984 assert!(offload.dispatched_models().is_empty());
23985 assert!(error.to_string().contains(&exact_id), "{error}");
23986 assert!(
23987 error
23988 .to_string()
23989 .contains("MLX-equivalent substitution is disabled for exact pins"),
23990 "{error}"
23991 );
23992 crate::offload::set_local_offload(None);
23993 }
23994
23995 #[tokio::test(flavor = "current_thread")]
23996 async fn catalog_identity_mismatch_rejects_before_provider_dispatch() {
23997 use wiremock::matchers::{method, path};
23998 use wiremock::{Mock, MockServer, ResponseTemplate};
23999
24000 let _env = ENV_MUTEX.lock().await;
24001 let server = MockServer::start().await;
24002 Mock::given(method("POST"))
24003 .and(path("/v1/chat/completions"))
24004 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
24005 "choices": [{"message": {"content": "must not run"}, "finish_reason": "stop"}],
24006 })))
24007 .mount(&server)
24008 .await;
24009 unsafe { std::env::set_var("CAR_PRECONDITION_TEST_KEY", "fixture") };
24010
24011 let tmp = TempDir::new().unwrap();
24012 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24013 let model_id = "test/catalog-precondition:1";
24014 engine.register_model(remote_stream_fixture_schema(
24015 model_id,
24016 server.uri(),
24017 schema::ApiProtocol::OpenAiCompat,
24018 "CAR_PRECONDITION_TEST_KEY",
24019 ));
24020 let snapshot = engine.catalog_snapshot().unwrap();
24021 let row_digest = snapshot
24022 .model_by_exact_id(model_id)
24023 .unwrap()
24024 .row_digest
24025 .clone();
24026
24027 for (expected_row_digest, expected_catalog_revision) in [
24028 (
24029 Some("0".repeat(64)),
24030 Some(snapshot.catalog_revision.clone()),
24031 ),
24032 (Some(row_digest.clone()), Some("f".repeat(64))),
24033 ] {
24034 let mut request = GenerateRequest {
24035 prompt: "must not dispatch".into(),
24036 expected_row_digest,
24037 expected_catalog_revision,
24038 ..Default::default()
24039 };
24040 pin_exact_model_id(&mut request, model_id.into()).unwrap();
24041 let error = engine.generate_tracked(request).await.unwrap_err();
24042 assert!(matches!(
24043 error,
24044 InferenceError::CatalogPreconditionMismatch { .. }
24045 ));
24046 }
24047 assert!(
24048 server.received_requests().await.unwrap().is_empty(),
24049 "catalog identity mismatches must fail before provider dispatch"
24050 );
24051
24052 unsafe { std::env::remove_var("CAR_PRECONDITION_TEST_KEY") };
24053 }
24054
24055 #[tokio::test(flavor = "current_thread")]
24056 async fn exact_nonstream_identity_survives_thinking_retry() {
24057 use std::sync::atomic::{AtomicUsize, Ordering};
24058 use wiremock::matchers::{method, path};
24059 use wiremock::{Mock, MockServer, ResponseTemplate};
24060
24061 let _env = ENV_MUTEX.lock().await;
24062 let server = MockServer::start().await;
24063 let calls = Arc::new(AtomicUsize::new(0));
24064 let response_calls = calls.clone();
24065 Mock::given(method("POST"))
24066 .and(path("/v1/chat/completions"))
24067 .respond_with(move |_request: &wiremock::Request| {
24068 let attempt = response_calls.fetch_add(1, Ordering::SeqCst);
24069 let content = if attempt == 0 { "" } else { "recovered" };
24070 ResponseTemplate::new(200).set_body_json(serde_json::json!({
24071 "choices": [{
24072 "message": {"content": content},
24073 "finish_reason": if attempt == 0 { "length" } else { "stop" },
24074 }],
24075 "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
24076 }))
24077 })
24078 .mount(&server)
24079 .await;
24080 unsafe { std::env::set_var("CAR_IDENTITY_RETRY_KEY", "fixture") };
24081
24082 let tmp = TempDir::new().unwrap();
24083 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24084 let model_id = "test/identity-retry:1";
24085 engine.register_model(remote_stream_fixture_schema(
24086 model_id,
24087 server.uri(),
24088 schema::ApiProtocol::OpenAiCompat,
24089 "CAR_IDENTITY_RETRY_KEY",
24090 ));
24091 let snapshot = engine.catalog_snapshot().unwrap();
24092 let expected_row = snapshot.model_by_exact_id(model_id).unwrap();
24093 let mut request = GenerateRequest {
24094 prompt: "retry once".into(),
24095 expected_row_digest: Some(expected_row.row_digest.clone()),
24096 expected_catalog_revision: Some(snapshot.catalog_revision.clone()),
24097 ..Default::default()
24098 };
24099 pin_exact_model_id(&mut request, model_id.into()).unwrap();
24100
24101 let result = engine.generate_tracked(request).await.unwrap();
24102 assert_eq!(calls.load(Ordering::SeqCst), 2, "must exercise retry path");
24103 assert_eq!(result.stop_reason.as_deref(), Some("thinking_recovered"));
24104 assert_eq!(
24105 result.model_identity.requested_model_id.as_deref(),
24106 Some(model_id)
24107 );
24108 assert_eq!(result.model_identity.resolved_model_id, model_id);
24109 assert_eq!(result.model_identity.row_digest, expected_row.row_digest);
24110 assert_eq!(
24111 result.model_identity.catalog_revision,
24112 snapshot.catalog_revision
24113 );
24114
24115 unsafe { std::env::remove_var("CAR_IDENTITY_RETRY_KEY") };
24116 }
24117
24118 #[tokio::test(flavor = "current_thread")]
24119 async fn managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns() {
24120 if !crate::run_in_isolated_test_process(
24121 "tests::managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns",
24122 "CAR_MANAGED_REASONING_ROUNDTRIP_CHILD",
24123 ) {
24124 return;
24125 }
24126 let _home = crate::openrouter::StateRootScope::new();
24127 use wiremock::matchers::{header, method, path};
24128 use wiremock::{Mock, MockServer, ResponseTemplate};
24129
24130 let _provider_env = crate::openrouter::test_environment_scope_async().await;
24131 let _env = ENV_MUTEX.lock().await;
24132 let bearer = "managed-reasoning-roundtrip-bearer";
24133
24134 let server = MockServer::start().await;
24135 unsafe {
24136 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
24137 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
24138 }
24139 Mock::given(method("GET"))
24140 .and(path("/api/v1/organizations/me"))
24141 .and(header("authorization", format!("Bearer {bearer}")))
24142 .respond_with(
24143 ResponseTemplate::new(200)
24144 .set_body_json(serde_json::json!({"organizationId": "org-roundtrip"})),
24145 )
24146 .mount(&server)
24147 .await;
24148 Mock::given(method("GET"))
24149 .and(path("/connect/session"))
24150 .respond_with(
24151 ResponseTemplate::new(200)
24152 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
24153 )
24154 .mount(&server)
24155 .await;
24156 Mock::given(method("POST"))
24157 .and(path("/api/v1/orgs/org-roundtrip/inference/responses"))
24158 .respond_with(ResponseTemplate::new(200).set_body_raw(
24159 include_str!("../tests/fixtures/parslee-openrouter-reasoning-roundtrip.sse"),
24160 "text/event-stream",
24161 ))
24162 .expect(2)
24163 .mount(&server)
24164 .await;
24165
24166 let tmp = TempDir::new().unwrap();
24167 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24168 let schema = crate::openrouter::curated_schemas()
24169 .into_iter()
24170 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
24171 .unwrap();
24172 engine.unified_registry_mut().register_project_model(schema);
24173
24174 let first = engine
24175 .generate_tracked(GenerateRequest {
24176 prompt: "first".into(),
24177 model: Some("parslee/openrouter/frontier-general".into()),
24178 ..Default::default()
24179 })
24180 .await
24181 .expect("first managed turn");
24182 let expected_reasoning = serde_json::json!({
24183 "type": "reasoning",
24184 "id": "rs_car_roundtrip",
24185 "status": "completed",
24186 "summary": [{"type": "summary_text", "text": "safe summary"}],
24187 "encrypted_content": "opaque-encrypted-reasoning",
24188 });
24189 assert_eq!(
24190 first.provider_output_items,
24191 vec![expected_reasoning.clone()]
24192 );
24193
24194 let mut history = vec![Message::User {
24195 content: "first".into(),
24196 }];
24197 first.append_assistant_history(&mut history, first.tool_calls.clone());
24198 history.push(Message::User {
24199 content: "continue".into(),
24200 });
24201
24202 let second = engine
24203 .generate_tracked(GenerateRequest {
24204 prompt: String::new(),
24205 model: Some("parslee/openrouter/frontier-general".into()),
24206 messages: Some(history),
24207 ..Default::default()
24208 })
24209 .await
24210 .expect("second managed turn");
24211 assert_eq!(second.text, "first answer");
24212
24213 let requests = server.received_requests().await.unwrap();
24214 let posts: Vec<serde_json::Value> = requests
24215 .iter()
24216 .filter(|request| {
24217 request.method.as_str() == "POST"
24218 && request.url.path() == "/api/v1/orgs/org-roundtrip/inference/responses"
24219 })
24220 .map(|request| serde_json::from_slice(&request.body).unwrap())
24221 .collect();
24222 assert_eq!(posts.len(), 2);
24223 for body in &posts {
24224 assert_eq!(body["store"], false);
24225 assert_eq!(
24226 body["include"],
24227 serde_json::json!(["reasoning.encrypted_content"])
24228 );
24229 }
24230 let second_input = posts[1]["input"].as_array().unwrap();
24231 let reasoning_index = second_input
24232 .iter()
24233 .position(|item| item == &expected_reasoning)
24234 .expect("second request must replay the exact reasoning item");
24235 let assistant_index = second_input
24236 .iter()
24237 .position(|item| item["role"] == "assistant")
24238 .expect("second request assistant turn");
24239 let user_index = second_input
24240 .iter()
24241 .position(|item| item["content"] == "continue")
24242 .expect("second request user turn");
24243 assert!(
24244 reasoning_index < assistant_index && assistant_index < user_index,
24245 "provider output order must be reasoning, assistant text, then the next user turn"
24246 );
24247
24248 unsafe {
24249 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
24250 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
24251 }
24252 }
24253
24254 #[tokio::test(flavor = "current_thread")]
24255 async fn managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures() {
24256 if !crate::run_in_isolated_test_process(
24257 "tests::managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures",
24258 "CAR_MANAGED_PARTIAL_EOF_CHILD",
24259 ) {
24260 return;
24261 }
24262 let _home = crate::openrouter::StateRootScope::new();
24263 use wiremock::matchers::{header, method, path};
24264 use wiremock::{Mock, MockServer, ResponseTemplate};
24265
24266 let _provider_env = crate::openrouter::test_environment_scope_async().await;
24267 let _env = ENV_MUTEX.lock().await;
24268 let bearer = "managed-partial-outcome-bearer";
24269
24270 let server = MockServer::start().await;
24271 unsafe {
24272 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
24273 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
24274 }
24275 Mock::given(method("GET"))
24276 .and(path("/api/v1/organizations/me"))
24277 .and(header("authorization", format!("Bearer {bearer}")))
24278 .respond_with(
24279 ResponseTemplate::new(200)
24280 .set_body_json(serde_json::json!({"organizationId": "org-partial-outcome"})),
24281 )
24282 .mount(&server)
24283 .await;
24284 Mock::given(method("GET"))
24285 .and(path("/connect/session"))
24286 .respond_with(
24287 ResponseTemplate::new(200)
24288 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
24289 )
24290 .mount(&server)
24291 .await;
24292 Mock::given(method("POST"))
24293 .and(path("/api/v1/orgs/org-partial-outcome/inference/responses"))
24294 .respond_with(ResponseTemplate::new(200).set_body_raw(
24295 "event: response.output_text.delta\ndata: {\"delta\":\"partial must fail\"}\n\n",
24296 "text/event-stream",
24297 ))
24298 .expect(2)
24299 .mount(&server)
24300 .await;
24301
24302 let tmp = TempDir::new().unwrap();
24303 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24304 let model_id = "parslee/openrouter/frontier-general";
24305 let schema = crate::openrouter::curated_schemas()
24306 .into_iter()
24307 .find(|schema| schema.id == model_id)
24308 .unwrap();
24309 engine.unified_registry_mut().register_project_model(schema);
24310
24311 let buffered_error = engine
24312 .generate_tracked(GenerateRequest {
24313 prompt: "buffered".into(),
24314 model: Some(model_id.into()),
24315 params: GenerateParams {
24316 strict_model: true,
24317 ..Default::default()
24318 },
24319 ..Default::default()
24320 })
24321 .await
24322 .expect_err("buffered partial EOF must fail");
24323 assert!(
24324 buffered_error.to_string().contains("response.completed"),
24325 "unexpected buffered error: {buffered_error}"
24326 );
24327
24328 let mut stream = engine
24329 .generate_tracked_stream(GenerateRequest {
24330 prompt: "streamed".into(),
24331 model: Some(model_id.into()),
24332 params: GenerateParams {
24333 strict_model: true,
24334 ..Default::default()
24335 },
24336 ..Default::default()
24337 })
24338 .await
24339 .expect("HTTP streaming request starts");
24340 let mut events = Vec::new();
24341 while let Some(event) = stream.events.recv().await {
24342 events.push(event);
24343 }
24344 assert!(
24345 matches!(events.last(), Some(StreamEvent::Error(message)) if message.contains("response.completed"))
24346 );
24347 assert!(!events
24348 .iter()
24349 .any(|event| matches!(event, StreamEvent::Done { .. })));
24350
24351 for _ in 0..50 {
24352 if engine
24353 .outcome_tracker()
24354 .read()
24355 .await
24356 .profile(model_id)
24357 .is_some_and(|profile| profile.fail_count == 2)
24358 {
24359 break;
24360 }
24361 tokio::task::yield_now().await;
24362 }
24363 let profile = engine
24364 .outcome_tracker()
24365 .read()
24366 .await
24367 .profile(model_id)
24368 .cloned()
24369 .unwrap();
24370 assert_eq!(profile.fail_count, 2);
24371 assert_eq!(profile.success_count, 0);
24372
24373 unsafe {
24374 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
24375 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
24376 }
24377 }
24378
24379 #[tokio::test]
24380 async fn tokenize_rejects_known_remote_model_with_unsupported_mode() {
24381 let tmp = TempDir::new().unwrap();
24386 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24387 let remote_id = engine
24388 .list_schemas()
24389 .into_iter()
24390 .find(|s| !s.is_local())
24391 .map(|s| s.id)
24392 .expect("built-in catalog should include at least one remote model schema");
24393
24394 let err = engine
24395 .tokenize(&remote_id, "hello")
24396 .await
24397 .expect_err("remote tokenize must error");
24398 match err {
24399 InferenceError::UnsupportedMode { mode, backend, .. } => {
24400 assert_eq!(mode, "tokenize/detokenize");
24401 assert_eq!(backend, "remote");
24402 }
24403 other => panic!("expected UnsupportedMode, got {other:?}"),
24404 }
24405
24406 let err = engine
24407 .detokenize(&remote_id, &[1, 2, 3])
24408 .await
24409 .expect_err("remote detokenize must error");
24410 assert!(
24411 matches!(err, InferenceError::UnsupportedMode { .. }),
24412 "expected UnsupportedMode, got {err:?}"
24413 );
24414 }
24415
24416 #[test]
24417 fn unsupported_mode_does_not_trip_circuit_breaker() {
24418 let unsupported = InferenceError::UnsupportedMode {
24424 mode: "structured-output-json-schema",
24425 backend: "anthropic",
24426 reason: "not wired under the pinned API version",
24427 };
24428 assert!(!error_counts_against_circuit_breaker(&unsupported));
24429 assert!(error_counts_against_circuit_breaker(
24430 &InferenceError::InferenceFailed("API returned 500".into())
24431 ));
24432 assert!(error_counts_against_circuit_breaker(
24433 &InferenceError::InferenceFailed("API returned 429".into())
24434 ));
24435 }
24436
24437 #[test]
24445 fn a_content_refusal_does_not_trip_the_circuit_breaker() {
24446 let refused = InferenceError::ContentRefused {
24447 provider: "parslee".into(),
24448 kind: Some("invalid_request_error".into()),
24449 code: Some("content_policy_violation".into()),
24450 message: "content refused".into(),
24451 };
24452 assert!(!error_counts_against_circuit_breaker(&refused));
24453
24454 let rendered = refused.to_string();
24457 assert!(rendered.contains("content grounds"), "{rendered}");
24458 assert!(rendered.contains("content_policy_violation"), "{rendered}");
24459
24460 assert!(error_counts_against_circuit_breaker(
24462 &InferenceError::InferenceFailed("managed inference failed".into())
24463 ));
24464 }
24465
24466 #[test]
24476 fn a_content_refusal_ends_the_fallback_chain() {
24477 let refused = InferenceError::ContentRefused {
24478 provider: "parslee".into(),
24479 kind: Some("invalid_request_error".into()),
24480 code: Some("content_policy_violation".into()),
24481 message: "content refused".into(),
24482 };
24483 assert!(error_ends_fallback_chain(&refused));
24484
24485 for still_advances in [
24489 InferenceError::InferenceFailed("managed inference failed".into()),
24490 InferenceError::InferenceFailed("API returned 503".into()),
24491 InferenceError::ModelNotFound("parslee/reasoning".into()),
24492 InferenceError::UnsupportedMode {
24493 mode: "json_schema",
24494 backend: "anthropic",
24495 reason: "structured output is not supported by this protocol",
24496 },
24497 InferenceError::GatewayUnconfigured {
24498 provider: "parslee".into(),
24499 namespace: "parslee/openrouter/".into(),
24500 status: 503,
24501 message: "not configured".into(),
24502 },
24503 ] {
24504 assert!(
24505 !error_ends_fallback_chain(&still_advances),
24506 "must keep advancing the chain: {still_advances}"
24507 );
24508 }
24509 }
24510
24511 #[test]
24520 fn recovery_hints_do_not_rewrite_a_content_refusal() {
24521 let refused = InferenceError::ContentRefused {
24522 provider: "parslee".into(),
24523 kind: Some("invalid_request_error".into()),
24524 code: Some("content_policy_violation".into()),
24525 message: "blocked: 403 forbidden by the content filter".into(),
24527 };
24528 let out = apply_exhaustion_recovery_hint(refused);
24529 assert!(
24530 matches!(out, InferenceError::ContentRefused { .. }),
24531 "{out:?}"
24532 );
24533
24534 let signed_out = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
24536 "no credential for proprietary provider 'parslee'".into(),
24537 ));
24538 assert!(
24539 matches!(signed_out, InferenceError::InferenceFailed(ref m) if m.contains("car auth")),
24540 "{signed_out:?}"
24541 );
24542
24543 let transient = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
24545 "API returned 500".into(),
24546 ));
24547 assert_eq!(transient.to_string(), "inference failed: API returned 500");
24548 }
24549
24550 #[test]
24558 fn unconfigured_gateway_does_not_trip_circuit_breaker() {
24559 let unconfigured = InferenceError::GatewayUnconfigured {
24560 provider: "parslee".into(),
24561 namespace: "parslee/openrouter/".into(),
24562 status: 503,
24563 message: "OpenRouter inference is not configured on this Parslee environment.".into(),
24564 };
24565 assert!(!error_counts_against_circuit_breaker(&unconfigured));
24566 let rendered = unconfigured.to_string();
24570 assert!(rendered.contains("parslee/openrouter/"), "{rendered}");
24571 assert!(rendered.contains("not configured"), "{rendered}");
24572 }
24573
24574 #[test]
24580 fn namespace_drop_spares_siblings_outside_the_prefix() {
24581 let namespace = "parslee/openrouter/";
24582 let mut queue: std::collections::VecDeque<String> = [
24583 "parslee/openrouter/open-fast",
24584 "parslee/reasoning",
24585 "parslee/openrouter/frontier-general",
24586 "parslee/advisor",
24587 "anthropic/claude-opus-4-8:latest",
24588 ]
24589 .into_iter()
24590 .map(String::from)
24591 .collect();
24592
24593 queue.retain(|id| !id.starts_with(namespace));
24594
24595 assert_eq!(
24596 queue.iter().collect::<Vec<_>>(),
24597 vec![
24598 "parslee/reasoning",
24599 "parslee/advisor",
24600 "anthropic/claude-opus-4-8:latest"
24601 ],
24602 "only the unconfigured namespace may be dropped"
24603 );
24604 }
24605
24606 #[test]
24607 fn engine_loads_benchmark_priors_on_startup() {
24608 let _env = ENV_MUTEX.blocking_lock();
24609 let tmp = TempDir::new().unwrap();
24610 let priors_path = tmp.path().join("benchmark_priors.json");
24611 std::fs::write(
24612 &priors_path,
24613 serde_json::json!({
24614 "model_id": "qwen/qwen3-8b:q4_k_m",
24615 "overall_score": 0.88
24616 })
24617 .to_string(),
24618 )
24619 .unwrap();
24620
24621 unsafe {
24622 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
24623 }
24624
24625 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24626 let tracker = engine.outcome_tracker.blocking_read();
24627 let profile = tracker
24628 .profile("qwen/qwen3-8b:q4_k_m")
24629 .expect("benchmark prior should create a profile");
24630 assert!((profile.ema_quality - 0.88).abs() < 0.01);
24631
24632 unsafe {
24633 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
24634 }
24635 }
24636
24637 #[test]
24638 fn benchmark_priors_do_not_override_observed_profiles() {
24639 let _env = ENV_MUTEX.blocking_lock();
24640 let tmp = TempDir::new().unwrap();
24641 let models_dir = tmp.path().join("models");
24642 std::fs::create_dir_all(&models_dir).unwrap();
24643
24644 let observed = vec![ModelProfile {
24645 model_id: "qwen/qwen3-8b:q4_k_m".into(),
24646 total_calls: 12,
24647 success_count: 3,
24648 fail_count: 9,
24649 total_latency_ms: 1200,
24650 total_input_tokens: 0,
24651 total_output_tokens: 0,
24652 total_cache_read_input_tokens: 0,
24653 total_cache_creation_input_tokens: 0,
24654 task_stats: std::collections::HashMap::new(),
24655 ema_quality: 0.21,
24656 prior_sample_size: 0,
24657 quality_observations: 0,
24658 quality_per_1k_tokens: 0.0,
24659 updated_at: 1,
24660 }];
24661 std::fs::write(
24662 models_dir.join("outcome_profiles.json"),
24663 serde_json::to_string(&observed).unwrap(),
24664 )
24665 .unwrap();
24666
24667 let priors_path = tmp.path().join("benchmark_priors.json");
24668 std::fs::write(
24669 &priors_path,
24670 serde_json::json!({
24671 "model_id": "qwen/qwen3-8b:q4_k_m",
24672 "overall_score": 0.95
24673 })
24674 .to_string(),
24675 )
24676 .unwrap();
24677
24678 unsafe {
24679 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
24680 }
24681
24682 let engine = InferenceEngine::new(test_config(models_dir));
24683 let tracker = engine.outcome_tracker.blocking_read();
24684 let profile = tracker
24685 .profile("qwen/qwen3-8b:q4_k_m")
24686 .expect("observed profile should remain present");
24687 assert!((profile.ema_quality - 0.21).abs() < 0.01);
24688 assert_eq!(profile.total_calls, 12);
24689
24690 unsafe {
24691 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
24692 }
24693 }
24694
24695 #[test]
24696 fn speech_runtime_package_spec_defaults_and_overrides() {
24697 let _env = ENV_MUTEX.blocking_lock();
24698 unsafe {
24699 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
24700 }
24701 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.2");
24702
24703 unsafe {
24704 std::env::set_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC", "mlx-audio==0.4.1");
24705 }
24706 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.1");
24707
24708 unsafe {
24709 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
24710 }
24711 }
24712
24713 #[test]
24714 fn speech_runtime_spacy_model_spec_defaults_and_overrides() {
24715 let _env = ENV_MUTEX.blocking_lock();
24716 unsafe {
24717 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
24718 }
24719 assert!(
24720 speech_runtime_spacy_model_spec().starts_with("en-core-web-sm @ https://github.com/")
24721 );
24722
24723 unsafe {
24724 std::env::set_var(
24725 "CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC",
24726 "en-core-web-sm==3.8.0",
24727 );
24728 }
24729 assert_eq!(speech_runtime_spacy_model_spec(), "en-core-web-sm==3.8.0");
24730
24731 unsafe {
24732 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
24733 }
24734 }
24735
24736 #[test]
24737 fn kokoro_runtime_fallback_defaults_on() {
24738 unsafe {
24739 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
24740 }
24741 assert!(kokoro_runtime_fallback_enabled());
24742
24743 unsafe {
24744 std::env::set_var("CAR_SPEECH_KOKORO_FALLBACK", "false");
24745 }
24746 assert!(!kokoro_runtime_fallback_enabled());
24747
24748 unsafe {
24749 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
24750 }
24751 }
24752
24753 #[test]
24754 fn preferred_local_tts_wins_over_builtin_rank() {
24755 let tmp = TempDir::new().unwrap();
24756 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24757 engine.set_speech_policy(SpeechPolicy {
24758 prefer_local: true,
24759 allow_remote_fallback: false,
24760 preferred_local_stt: None,
24761 preferred_local_tts: Some("Kokoro-82M-6bit".into()),
24762 preferred_remote_stt: None,
24763 preferred_remote_tts: None,
24764 });
24765
24766 let schema = engine
24767 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
24768 .expect("preferred local TTS should resolve");
24769 #[cfg(not(target_os = "windows"))]
24776 assert_eq!(schema.name, "Kokoro-82M-6bit");
24777 #[cfg(target_os = "windows")]
24778 assert_eq!(schema.name, "Windows-Speech");
24779 }
24780
24781 #[test]
24782 fn preferred_discovered_vllm_mlx_model_wins_generate_routing() {
24783 let tmp = TempDir::new().unwrap();
24784 let mut config = test_config(tmp.path().join("models"));
24785 config.preferred_generation_model =
24786 Some("vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit".into());
24787 let mut engine = InferenceEngine::new(config);
24788 let schema = crate::vllm_mlx::to_model_schema(
24789 &crate::vllm_mlx::DiscoveredModel {
24790 id: "mlx-community/gemma-3n-E2B-it-lm-4bit".into(),
24791 owned_by: Some("mlx-community".into()),
24792 },
24793 "http://127.0.0.1:8001",
24794 );
24795 engine.register_model(schema);
24796
24797 let rt = tokio::runtime::Runtime::new().unwrap();
24798 let decision = rt.block_on(engine.route_adaptive("say hello in one sentence"));
24799 assert_eq!(
24800 decision.model_id,
24801 "vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit"
24802 );
24803 assert_eq!(decision.strategy, RoutingStrategy::Explicit);
24804 assert_eq!(decision.reason, "preferred generation model override");
24805 }
24806
24807 #[test]
24816 fn tools_request_on_non_tool_model_terminates_not_spins() {
24817 let tmp = TempDir::new().unwrap();
24818 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24819 let schema = crate::vllm_mlx::to_model_schema(
24821 &crate::vllm_mlx::DiscoveredModel {
24822 id: "test-org/embed-only-model".into(),
24823 owned_by: None,
24824 },
24825 "http://127.0.0.1:1", );
24827 let model_id = schema.id.clone();
24828 engine.register_model(schema);
24829
24830 let mut req = GenerateRequest {
24831 prompt: "call a tool".to_string(),
24832 model: Some(model_id),
24833 tools: Some(vec![serde_json::json!({
24834 "name": "noop", "description": "n", "parameters": {"type": "object"}
24835 })]),
24836 ..Default::default()
24837 };
24838 req.params.strict_model = true;
24839
24840 let rt = tokio::runtime::Runtime::new().unwrap();
24841 let out = rt.block_on(async {
24842 tokio::time::timeout(
24843 std::time::Duration::from_secs(10),
24844 engine.generate_tracked(req),
24845 )
24846 .await
24847 });
24848 let completed = out.expect("fallback loop must terminate, not spin");
24851 assert!(completed.is_err());
24852 }
24853
24854 fn fake_ready_speech_runtime(root: &Path) {
24865 managed_venv::seed_ready_venv(root);
24866 let runtime = SpeechRuntime::new(root.to_path_buf());
24867 for program in [&runtime.stt_program, &runtime.tts_program] {
24868 std::fs::create_dir_all(program.parent().expect("program has a parent")).unwrap();
24869 std::fs::write(program, b"").unwrap();
24870 }
24871 }
24872
24873 #[tokio::test]
24900 async fn prepare_speech_runtime_returns_the_root_doctor_reports() {
24901 let _env = ENV_MUTEX.lock().await;
24902 let tmp = TempDir::new().unwrap();
24903 let runtime_root = tmp.path().join("speech-runtime");
24904 fake_ready_speech_runtime(&runtime_root);
24905 unsafe {
24906 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
24907 }
24908
24909 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24910 let root = engine
24911 .prepare_speech_runtime()
24912 .await
24913 .expect("prepare should succeed against a ready runtime");
24914 let health = engine.speech_health();
24915
24916 assert_eq!(
24917 root, health.runtime.root,
24918 "prepare returned a different root than doctor reports"
24919 );
24920 assert!(
24921 root.exists(),
24922 "prepare returned {} but it does not exist",
24923 root.display()
24924 );
24925 assert!(
24926 health.runtime.installed,
24927 "doctor should report a ready runtime as installed"
24928 );
24929 assert_eq!(
24931 engine
24932 .prepare_speech_runtime()
24933 .await
24934 .expect("second prepare should succeed"),
24935 root
24936 );
24937
24938 unsafe {
24939 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
24940 }
24941 }
24942
24943 #[tokio::test]
24957 async fn prepare_speech_runtime_fails_when_bootstrap_fails() {
24958 let _env = ENV_MUTEX.lock().await;
24959 let tmp = TempDir::new().unwrap();
24960 let runtime_root = tmp.path().join("speech-runtime");
24961 assert!(!runtime_root.exists(), "precondition: root absent");
24962 unsafe {
24963 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
24964 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
24965 }
24966
24967 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24968 let err = engine.prepare_speech_runtime().await.err();
24969 let health = engine.speech_health();
24970
24971 assert!(
24972 err.is_some(),
24973 "a failed bootstrap must fail the install, not return a root"
24974 );
24975 assert!(
24976 !health.runtime.installed,
24977 "doctor must not claim a runtime the bootstrap never built"
24978 );
24979
24980 unsafe {
24981 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
24982 std::env::remove_var("CAR_SPEECH_PYTHON");
24983 }
24984 }
24985
24986 #[cfg(target_os = "linux")]
24999 #[tokio::test]
25000 async fn a_runtime_that_cannot_be_built_no_longer_blocks_the_whisper_install() {
25001 let _env = ENV_MUTEX.lock().await;
25002 let tmp = TempDir::new().unwrap();
25003
25004 let cached = tmp
25008 .path()
25009 .join(".tokhn")
25010 .join("whisper")
25011 .join("ggml-large-v3-turbo-q5_0.bin");
25012 std::fs::create_dir_all(cached.parent().unwrap()).unwrap();
25013 std::fs::write(&cached, b"stand-in for the ggml weights").unwrap();
25014
25015 let previous_home = std::env::var_os("HOME");
25016 unsafe {
25017 std::env::set_var("HOME", tmp.path());
25018 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", tmp.path().join("speech-runtime"));
25019 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
25023 }
25024
25025 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25026 let installed = engine.install_curated_speech().await;
25027
25028 let restore = || unsafe {
25029 match &previous_home {
25030 Some(value) => std::env::set_var("HOME", value),
25031 None => std::env::remove_var("HOME"),
25032 }
25033 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
25034 std::env::remove_var("CAR_SPEECH_PYTHON");
25035 };
25036 let installed = match installed {
25037 Ok(installed) => installed,
25038 Err(error) => {
25039 restore();
25040 panic!("a runtime that cannot be built must not abort the install: {error}");
25041 }
25042 };
25043 let runtime_installed = engine.speech_health().runtime.installed;
25044 restore();
25045
25046 assert!(
25047 !runtime_installed,
25048 "precondition: the bootstrap must actually have failed, or this \
25049 test would pass without exercising anything"
25050 );
25051 let whisper = installed
25052 .iter()
25053 .find(|report| report.hf_repo == "ggerganov/whisper.cpp")
25054 .unwrap_or_else(|| {
25055 panic!(
25056 "the cross-platform whisper model must still be installed, got: {installed:?}"
25057 )
25058 });
25059 assert_eq!(whisper.snapshot_path, cached);
25060 assert!(
25061 installed
25062 .iter()
25063 .all(|report| report.hf_repo == "ggerganov/whisper.cpp"),
25064 "MLX weights are Apple-only and must not be pulled here — over a \
25065 gigabyte of them, for models this host can never load; got: {installed:?}"
25066 );
25067 }
25068
25069 #[test]
25077 fn alias_resolves_to_canonical_id_for_outcome_keying() {
25078 let tmp = TempDir::new().unwrap();
25079 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25080
25081 let cases = [
25083 ("claude-sonnet-4-6", "anthropic/claude-sonnet-4-6:latest"),
25084 ("gpt-5.4", "openai/gpt-5.4:latest"),
25085 ("gemini-2.5-flash", "google/gemini-2.5-flash:latest"),
25086 ];
25087 for (alias, canonical) in cases {
25088 let resolved = engine
25091 .resolve_schema(alias)
25092 .map(|s| s.id)
25093 .unwrap_or_else(|| alias.to_string());
25094 assert_eq!(
25095 resolved, canonical,
25096 "alias `{alias}` must resolve to canonical `{canonical}` for outcome keying, got `{resolved}`"
25097 );
25098 assert_ne!(
25099 resolved, alias,
25100 "alias `{alias}` must NOT be recorded raw — that is the fragmentation bug"
25101 );
25102 }
25103 }
25104
25105 #[test]
25110 fn inference_result_serializes_with_full_shape() {
25111 use crate::tasks::generate::ToolCall;
25112 use std::collections::HashMap;
25113
25114 let mut args = HashMap::new();
25115 args.insert("path".to_string(), serde_json::json!("README.md"));
25116
25117 let result = InferenceResult {
25118 text: String::new(),
25119 bounding_boxes: Vec::new(),
25120 tool_calls: vec![ToolCall {
25121 id: None,
25122 name: "read_file".into(),
25123 arguments: args,
25124 }],
25125 trace_id: "trace-abc".into(),
25126 model_used: "test-model".into(),
25127 model_identity: InferenceModelIdentity {
25128 requested_model_id: Some("test/model:1".into()),
25129 resolved_model_id: "test/model:1".into(),
25130 row_digest: "a".repeat(64),
25131 catalog_revision: "b".repeat(64),
25132 },
25133 latency_ms: 1234,
25134 time_to_first_token_ms: Some(180),
25135 usage: Some(TokenUsage {
25136 prompt_tokens: 100,
25137 completion_tokens: 50,
25138 total_tokens: 150,
25139 context_window: 8192,
25140 ..Default::default()
25141 }),
25142 provider_output_items: Vec::new(),
25143 thinking: Vec::new(),
25144 stop_reason: Some("tool_use".into()),
25145 auth_fallback_from: None,
25146 local_last_resort: false,
25147 fallback_from: Vec::new(),
25148 };
25149
25150 let json = serde_json::to_value(&result).expect("serialize");
25151
25152 assert_eq!(json["stop_reason"].as_str(), Some("tool_use"));
25154
25155 assert_eq!(json["text"].as_str(), Some(""));
25157 assert_eq!(json["trace_id"].as_str(), Some("trace-abc"));
25158 assert_eq!(json["model_used"].as_str(), Some("test-model"));
25159 assert_eq!(json["requested_model_id"].as_str(), Some("test/model:1"));
25160 assert_eq!(json["resolved_model_id"].as_str(), Some("test/model:1"));
25161 assert_eq!(json["row_digest"].as_str().unwrap(), "a".repeat(64));
25162 assert_eq!(json["catalog_revision"].as_str().unwrap(), "b".repeat(64));
25163 assert_eq!(json["latency_ms"].as_u64(), Some(1234));
25164
25165 let tool_calls = json["tool_calls"].as_array().expect("tool_calls array");
25167 assert_eq!(tool_calls.len(), 1);
25168 assert_eq!(tool_calls[0]["name"].as_str(), Some("read_file"));
25169 assert_eq!(
25170 tool_calls[0]["arguments"]["path"].as_str(),
25171 Some("README.md")
25172 );
25173
25174 let usage = &json["usage"];
25176 assert_eq!(usage["prompt_tokens"].as_u64(), Some(100));
25177 assert_eq!(usage["completion_tokens"].as_u64(), Some(50));
25178 assert_eq!(usage["total_tokens"].as_u64(), Some(150));
25179 assert_eq!(usage["context_window"].as_u64(), Some(8192));
25180
25181 assert_eq!(json["time_to_first_token_ms"].as_u64(), Some(180));
25183 }
25184
25185 #[test]
25191 fn inference_result_top_level_keys_are_locked() {
25192 use std::collections::BTreeSet;
25193
25194 let result = InferenceResult {
25195 text: "anything".into(),
25196 bounding_boxes: Vec::new(),
25197 tool_calls: vec![],
25198 trace_id: "t".into(),
25199 model_used: "m".into(),
25200 model_identity: InferenceModelIdentity::default(),
25201 latency_ms: 0,
25202 time_to_first_token_ms: None,
25203 usage: None,
25204 provider_output_items: Vec::new(),
25205 thinking: Vec::new(),
25206 stop_reason: None,
25207 auth_fallback_from: None,
25208 local_last_resort: false,
25209 fallback_from: Vec::new(),
25210 };
25211
25212 let json = serde_json::to_value(&result).expect("serialize");
25213 let keys: BTreeSet<&str> = json
25214 .as_object()
25215 .expect("top-level object")
25216 .keys()
25217 .map(String::as_str)
25218 .collect();
25219
25220 let expected: BTreeSet<&str> = [
25221 "text",
25222 "tool_calls",
25223 "trace_id",
25224 "model_used",
25225 "requested_model_id",
25226 "resolved_model_id",
25227 "row_digest",
25228 "catalog_revision",
25229 "latency_ms",
25230 "time_to_first_token_ms",
25231 "usage",
25232 "stop_reason",
25233 ]
25234 .into_iter()
25235 .collect();
25236
25237 assert_eq!(
25238 keys, expected,
25239 "infer response top-level keys drifted -- update both the test \
25240 and the WebSocket protocol documentation if this is intentional"
25241 );
25242
25243 for key in &keys {
25245 assert!(
25246 !key.chars().any(|c| c.is_uppercase()) && !key.contains('-'),
25247 "key '{}' is not snake_case",
25248 key
25249 );
25250 }
25251 }
25252
25253 #[test]
25257 fn inference_result_serializes_plain_text_response() {
25258 let result = InferenceResult {
25259 text: "hello world".into(),
25260 bounding_boxes: Vec::new(),
25261 tool_calls: vec![],
25262 trace_id: "trace-xyz".into(),
25263 model_used: "test-model".into(),
25264 model_identity: InferenceModelIdentity::default(),
25265 latency_ms: 42,
25266 time_to_first_token_ms: None,
25267 usage: None,
25268 provider_output_items: Vec::new(),
25269 thinking: Vec::new(),
25270 stop_reason: None,
25271 auth_fallback_from: None,
25272 local_last_resort: false,
25273 fallback_from: Vec::new(),
25274 };
25275
25276 let json = serde_json::to_value(&result).expect("serialize");
25277 assert_eq!(json["text"], "hello world");
25278 assert!(json["stop_reason"].is_null());
25280 assert!(json["tool_calls"].is_array());
25281 assert_eq!(json["tool_calls"].as_array().unwrap().len(), 0);
25282 assert_eq!(json["model_used"], "test-model");
25283 assert!(json["usage"].is_null());
25284 assert!(json["time_to_first_token_ms"].is_null());
25287 }
25288
25289 #[test]
25290 fn append_assistant_history_preserves_responses_items_in_provider_order() {
25291 let reasoning = serde_json::json!({
25292 "type": "reasoning",
25293 "id": "rs_history",
25294 "status": "completed",
25295 "summary": [{"type": "summary_text", "text": "safe"}],
25296 "encrypted_content": "opaque",
25297 });
25298 let result: InferenceResult = serde_json::from_value(serde_json::json!({
25299 "text": "calling",
25300 "tool_calls": [{
25301 "id": "call_1",
25302 "name": "read_file",
25303 "arguments": {"path": "README.md"}
25304 }],
25305 "trace_id": "trace",
25306 "model_used": "gateway-alias",
25307 "resolved_model_id": "openrouter/anthropic/claude-sonnet-4.5",
25308 "local_last_resort": true,
25309 "latency_ms": 1,
25310 "provider_output_items": [reasoning.clone()],
25311 }))
25312 .unwrap();
25313 let mut history = vec![crate::tasks::generate::Message::User {
25314 content: "inspect".into(),
25315 }];
25316
25317 result.append_assistant_history(&mut history, result.tool_calls.clone());
25318
25319 assert!(matches!(
25320 &history[1],
25321 crate::tasks::generate::Message::ProviderOutputItems { protocol, items }
25322 if protocol == crate::protocol::OPENAI_RESPONSES_PROTOCOL
25323 && items == &vec![reasoning]
25324 ));
25325 assert!(matches!(
25326 &history[2],
25327 crate::tasks::generate::Message::Assistant {
25328 content,
25329 tool_calls,
25330 model_id,
25331 local_last_resort,
25332 ..
25333 } if content == "calling"
25334 && tool_calls[0].id.as_deref() == Some("call_1")
25335 && model_id.as_deref() == Some("openrouter/anthropic/claude-sonnet-4.5")
25336 && *local_last_resort
25337 ));
25338 }
25339
25340 #[test]
25341 fn append_assistant_history_leaves_personal_chat_history_unchanged() {
25342 let result: InferenceResult = serde_json::from_value(serde_json::json!({
25343 "text": "plain",
25344 "tool_calls": [],
25345 "trace_id": "trace",
25346 "model_used": "openrouter/openai/gpt-4.1-mini",
25347 "latency_ms": 1,
25348 }))
25349 .unwrap();
25350 let mut history = Vec::new();
25351
25352 result.append_assistant_history(&mut history, Vec::new());
25353
25354 assert_eq!(history.len(), 1);
25355 assert!(matches!(
25356 &history[0],
25357 crate::tasks::generate::Message::Assistant { content, .. } if content == "plain"
25358 ));
25359 }
25360
25361 #[test]
25373 fn generate_request_deserializes_intent_field_from_json_rpc_params() {
25374 use crate::intent::TaskHint;
25375 use crate::schema::ModelCapability;
25376
25377 let params = serde_json::json!({
25380 "prompt": "summarize this email",
25381 "intent": {
25382 "task": "chat",
25383 "prefer_local": true,
25384 "require": ["tool_use"],
25385 },
25386 });
25387
25388 let req: GenerateRequest =
25389 serde_json::from_value(params).expect("GenerateRequest deserialize");
25390
25391 let intent = req.intent.as_ref().expect("intent field deserialized");
25392 assert_eq!(intent.task, Some(TaskHint::Chat));
25393 assert!(intent.prefer_local);
25394 assert_eq!(intent.require, vec![ModelCapability::ToolUse]);
25395
25396 let back: serde_json::Value =
25400 serde_json::to_value(&req).expect("re-serialize GenerateRequest");
25401 assert_eq!(back["intent"]["task"], "chat");
25402 assert_eq!(back["intent"]["prefer_local"], true);
25403 assert_eq!(back["intent"]["require"][0], "tool_use");
25404
25405 let default_req: GenerateRequest = serde_json::from_value(serde_json::json!({
25410 "prompt": "x",
25411 "intent": {},
25412 }))
25413 .unwrap();
25414 let default_intent = default_req.intent.expect("present but empty");
25415 assert_eq!(default_intent.task, None);
25416 assert!(!default_intent.prefer_local);
25417 assert!(default_intent.require.is_empty());
25418
25419 let no_intent: GenerateRequest =
25422 serde_json::from_value(serde_json::json!({"prompt": "x"})).unwrap();
25423 assert!(no_intent.intent.is_none());
25424 }
25425
25426 #[test]
25427 fn rerank_prompt_matches_upstream_template_shape() {
25428 let p = rerank_prompt(
25429 "retrieve relevant passages",
25430 "who runs the treasury?",
25431 "doc x",
25432 );
25433 assert!(p.contains("<|im_start|>system"));
25434 assert!(p.contains("Note that the answer can only be \"yes\" or \"no\"."));
25435 assert!(p.contains("<|im_start|>user\n<Instruct>: retrieve relevant passages"));
25436 assert!(p.contains("<Query>: who runs the treasury?"));
25437 assert!(p.contains("<Document>: doc x<|im_end|>"));
25438 assert!(p.contains("<|im_start|>assistant\n<think>\n\n</think>\n\n"));
25439 }
25440
25441 #[test]
25442 fn rerank_score_yes_and_no_exactly() {
25443 assert_eq!(score_from_rerank_output("yes", "m"), 1.0);
25444 assert_eq!(score_from_rerank_output("no", "m"), 0.0);
25445 }
25446
25447 #[test]
25448 fn rerank_score_handles_case_leading_space_and_chat_sentinels() {
25449 assert_eq!(score_from_rerank_output(" Yes", "m"), 1.0);
25452 assert_eq!(score_from_rerank_output("\nno.", "m"), 0.0);
25453 assert_eq!(score_from_rerank_output("<|im_end|>yes", "m"), 1.0);
25454 }
25455
25456 #[test]
25457 fn rerank_score_scans_up_to_three_tokens() {
25458 assert_eq!(score_from_rerank_output("_bos_ yes", "m"), 1.0);
25461 }
25462
25463 #[test]
25464 fn rerank_score_unexpected_is_neutral() {
25465 assert_eq!(score_from_rerank_output("maybe", "m"), 0.5);
25468 assert_eq!(score_from_rerank_output("", "m"), 0.5);
25469 }
25470
25471 #[tokio::test]
25472 async fn pull_reuses_a_valid_directory_receipt_and_reports_removability() {
25473 let root = tempfile::tempdir().unwrap();
25474 let models_dir = root.path().join("models");
25475 std::fs::create_dir_all(&models_dir).unwrap();
25476 let engine = InferenceEngine::new(InferenceConfig {
25477 state_root: root.path().join("state"),
25478 models_dir: models_dir.clone(),
25479 ..InferenceConfig::default()
25480 });
25481 let schema = engine
25482 .unified_registry()
25483 .all()
25484 .find(|schema| matches!(schema.source, ModelSource::Local { .. }))
25485 .unwrap()
25486 .clone();
25487 let managed = models_dir.join(&schema.name);
25488 std::fs::create_dir_all(&managed).unwrap();
25489 std::fs::write(managed.join("model.gguf"), b"owned").unwrap();
25490 std::fs::write(managed.join("tokenizer.json"), b"{}").unwrap();
25491 engine
25492 .model_management
25493 .record_managed_artifact(
25494 &schema.id,
25495 model_source_identity(&schema),
25496 None,
25497 1,
25498 false,
25499 managed.clone(),
25500 )
25501 .unwrap();
25502 let row = engine
25503 .list_models_unified()
25504 .into_iter()
25505 .find(|row| row.id == schema.id)
25506 .unwrap();
25507 if model_management::directory_removal_supported() {
25508 assert!(engine.model_management.can_remove(&schema.id).unwrap());
25509 assert!(row.can_remove);
25510 assert_eq!(row.management_evidence.as_deref(), Some("install_receipt"));
25511 } else {
25512 assert!(!engine.model_management.can_remove(&schema.id).unwrap());
25513 assert!(!row.can_remove);
25514 assert_eq!(
25515 row.management_evidence.as_deref(),
25516 Some("install_receipt_directory_cleanup_unsupported")
25517 );
25518 }
25519
25520 let reused = engine.pull_model(&schema.id).await.unwrap();
25521 assert_eq!(reused, managed);
25522 }
25523}
25524
25525#[cfg(test)]
25526mod response_format_support_tests {
25527 use super::*;
25528
25529 #[test]
25533 fn rejection_reason_tracks_the_protocol_handler() {
25534 let root = tempfile::tempdir().unwrap();
25537 let engine = InferenceEngine::new(InferenceConfig {
25538 state_root: root.path().join("state"),
25539 models_dir: root.path().join("models"),
25540 ..InferenceConfig::default()
25541 });
25542 let rf = crate::tasks::generate::ResponseFormat::JsonObject;
25543 let models = engine.list_models_unified();
25544 let anthropic = models
25545 .iter()
25546 .find(|m| m.provider.eq_ignore_ascii_case("anthropic"));
25547 if let Some(m) = anthropic {
25548 let reason = engine
25549 .response_format_rejection_reason(&m.id, &rf)
25550 .expect("the Anthropic protocol rejects response_format");
25551 assert!(reason.contains("protocol rejects"), "{reason}");
25552 }
25553 let openrouter = models
25554 .iter()
25555 .find(|m| m.provider.eq_ignore_ascii_case("openrouter"));
25556 if let Some(m) = openrouter {
25557 assert_eq!(
25558 engine.response_format_rejection_reason(&m.id, &rf),
25559 None,
25560 "OpenRouter forwards the format upstream"
25561 );
25562 }
25563 assert!(
25564 anthropic.is_some() || openrouter.is_some(),
25565 "the builtin catalog should list at least one of the two providers this pins"
25566 );
25567 assert_eq!(
25568 engine.response_format_rejection_reason("no/such-model", &rf),
25569 None,
25570 "an unknown model is not a rejection"
25571 );
25572 }
25573}