1pub mod action_ledger;
26pub mod adaptive_router;
27pub mod aws_sigv4;
28pub mod backend;
29pub mod backend_cache;
30pub mod billing;
31pub mod calibration;
32pub mod catalog;
33pub mod catalog_identity;
34pub mod concierge;
35pub mod concierge_answer;
36pub mod discovery;
37pub mod doctor;
38pub mod download;
39pub mod handle;
40pub mod hardware;
41pub mod hf_cache;
42pub mod hf_schema;
43pub mod intent;
44pub mod key_pool;
45pub mod lane_defaults;
46pub mod maintenance;
47pub mod managed_venv;
48pub mod media_tokens;
49pub mod model_management;
50pub mod models;
51pub mod nudge;
52pub mod offload;
53pub mod openrouter;
54pub mod outcome;
55pub mod parslee_credential;
56pub mod portfolio;
57pub mod protocol;
58pub mod recommend;
59pub mod registry;
60pub mod remote;
61pub mod resource_policy;
62pub mod retire;
63pub mod router;
64pub mod routing_ext;
65pub mod runner;
66pub mod schema;
67pub mod scoreboard;
68pub mod search;
69pub mod service;
70pub mod stream;
71pub mod tasks;
72pub(crate) mod tls_client;
76pub mod uninstall;
77pub mod update_prefs;
78pub mod upgrade;
79pub mod usage_profile;
80pub mod vllm_mlx;
81pub mod vllm_pool;
82pub mod vllm_runtime;
83
84use std::path::{Path, PathBuf};
85use std::sync::Arc;
86use std::time::Instant;
87use std::time::{SystemTime, UNIX_EPOCH};
88
89use reqwest::multipart::{Form, Part};
90use serde::Serialize;
91use thiserror::Error;
92use tokio::io::AsyncReadExt;
93use tokio::process::Command;
94use tokio::sync::Mutex;
95use tokio::sync::RwLock;
96use tracing::{debug, instrument};
97
98pub use action_ledger::{ConciergeActionEntry, ConciergeActionKind};
100pub use adaptive_router::{
101 AdaptiveRouter, AdaptiveRoutingDecision, RoutingConfig, RoutingStrategy,
102};
103pub use catalog_identity::{CatalogModelRow, CatalogSnapshot};
104pub use concierge::{
105 decide_concierge, evaluate_concierge, ConciergeDecision, ConciergeMode, ConciergeStatus,
106 ConciergeSuggestion, DismissReason, DismissalRecord, ModelHealth,
107 DEFAULT_CONCIERGE_THROTTLE_SECS, DEFAULT_WATCHED_USE_CASES,
108};
109pub use download::{DownloadEvent, DownloadProgress, ProgressSink};
110pub use handle::InferenceHandle;
111pub use intent::{IntentHint, Privacy, QualityTier, TaskHint, TierWeights, UseCase, UseCaseRole};
112pub use key_pool::{KeyPool, KeyStats};
113pub use lane_defaults::{LaneDefault, LaneDefaults};
114pub use nudge::{NudgeDecision, NudgeState, UpgradeNudge};
115pub use outcome::{
116 prune_ledger, read_ledger, CodeOutcome, InferenceOutcome, InferenceTask, InferredOutcome,
117 ModelProfile, OutcomeLedgerEntry, OutcomeTracker,
118};
119pub use recommend::{
120 model_fit, platform_compatible, recommend, recommend_with_policy, FitStatus, ModelFit,
121 ModelFitStatus, Recommendation, RecommendationSet,
122};
123pub use resource_policy::{
124 estimate_model_memory, estimate_model_memory_with_measured_weights, evaluate_resources,
125 AcceleratorResourceBudget, EffectiveResourceBudget, FileResourcePolicyRepository,
126 LocalAdmissionCoordinator, LocalLoadPreflight, LocalLoadVerdict, ModelMemoryEstimate,
127 ModelResourceEvidence, ResourceEvaluation, ResourcePolicy, ResourcePolicyError,
128 ResourcePolicyLoadEvidence, ResourcePolicyLoadSource, ResourcePolicyRepository,
129 ResourceProfile, RECOMMENDATION_CONTEXT_TOKENS,
130};
131pub use update_prefs::{UpdateChannel, UpdatePolicy, UpdatePreferences};
132pub use upgrade::{HuggingFaceProbe, UpgradeFinding, UpgradeSource, UpstreamProbe};
133pub use usage_profile::{use_case_for_task, LaneUsage, UsageProfile};
134
135fn now_unix() -> u64 {
137 std::time::SystemTime::now()
138 .duration_since(std::time::UNIX_EPOCH)
139 .map(|d| d.as_secs())
140 .unwrap_or(0)
141}
142pub use offload::{
143 clear_remote_deadline, current_controlled_termination_token, current_inference_control_id,
144 current_local_offload, current_remote_deadline, ensure_not_controlled_terminated,
145 is_offload_worker, scope_inference_control_id, set_local_offload, set_remote_deadline,
146 ControlledTerminationToken, InferenceTerminationAck, LocalGenerationOffload,
147 LocalOffloadResult, LocalOffloadStream, LocalWorkerAdmission, LocalWorkerResidency,
148 RemoteDeadline,
149};
150pub use registry::{
151 is_builtin_subscription_billed, model_unavailable_reason, ModelFilter, ModelInfo,
152 ModelRuntimeRequirement, ModelUpgrade, UnifiedRegistry,
153};
154pub use remote::RemoteBackend;
155pub use routing_ext::{
156 CircuitBreaker, CircuitBreakerRegistry, CircuitState, ImplicitSignal, ImplicitSignalType,
157 RoutingMode, SpendControl, SpendLimitExceeded, SpendLimits, SpendStatus,
158};
159pub use runner::{
160 current_inference_runner, set_inference_runner, EventEmitter, InferenceRunner, RunnerError,
161 RunnerResult,
162};
163pub use schema::{
164 ApiProtocol, ApproxCost, BenchmarkScore, CostModel, ModelCapability, ModelSchema, ModelSource,
165 PerformanceEnvelope, ProprietaryAuth, ProprietaryProtocol, ProprietaryWire, QuantScheme,
166 Quantization, TrustTier, OPENAI_CODEX_SIGN_IN_HINT,
167};
168
169pub use adaptive_router::{RouterExclusion, RouterExclusionReason, TaskComplexity};
171#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
172pub use backend::CandleBackend;
173#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
174pub use backend::EmbeddingBackend;
175#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
176pub use backend::SwiftLmBackend;
177pub use hardware::HardwareInfo;
178pub use models::{ModelRegistry, ModelRole};
179pub use router::{ModelRouter, RoutingDecision};
180pub use stream::{StreamAccumulator, StreamEvent};
181pub use tasks::{
182 parse_boxes, BoundingBox, ClassifyRequest, ClassifyResult, ContentBlock, EmbedRequest,
183 GenerateImageRequest, GenerateImageResult, GenerateParams, GenerateRequest,
184 GenerateVideoRequest, GenerateVideoResult, GroundRequest, GroundResult, Message,
185 OptionProbabilities, Provenance, RerankRequest, RerankResult, RerankedDocument, ResponseFormat,
186 RoutingWorkload, SynthesizeRequest, SynthesizeResult, ThinkingMode, ToolCall,
187 TranscribeRequest, TranscribeResult, VideoMode,
188};
189#[derive(Debug, Clone, PartialEq, Eq)]
197pub struct InferenceRetryProgress {
198 pub model: String,
200 pub attempt: u32,
202 pub reason: &'static str,
204 pub backoff_ms: u64,
206}
207
208#[derive(Error, Debug)]
209pub enum InferenceError {
210 #[error("model not found: {0}")]
211 ModelNotFound(String),
212
213 #[error("no eligible model remains after required exclusions: {excluded_models}")]
217 NoEligibleModel { excluded_models: String },
218
219 #[error("model download failed: {0}")]
220 DownloadFailed(String),
221
222 #[error("inference failed: {0}")]
223 InferenceFailed(String),
224
225 #[error("classify: the model's reply {reply:?} {reason}")]
229 ClassifyNoAnswer { reply: String, reason: String },
230
231 #[error("classify: invalid labels: {0}")]
235 InvalidClassifyLabels(String),
236
237 #[error("catalog precondition mismatch: {detail}")]
242 CatalogPreconditionMismatch { detail: String },
243
244 #[error("controlled inference termination confirmed")]
249 ControlledTermination,
250
251 #[error(transparent)]
252 ModelManagement(#[from] model_management::ModelManagementError),
253
254 #[error("{recovery}")]
257 LocalResourceBlocked {
258 preflight: resource_policy::LocalLoadPreflight,
259 recovery: String,
260 },
261
262 #[error("transient remote failure after retries (status={status:?}): {message}")]
271 Transient {
272 status: Option<u16>,
273 message: String,
274 },
275
276 #[error("deadline exceeded: the caller's {applied_ms} ms infer deadline elapsed after {elapsed_ms} ms; last attempt: {last_error}")]
284 DeadlineExceeded {
285 applied_ms: u64,
286 elapsed_ms: u64,
287 last_error: String,
288 },
289
290 #[error("mode {mode} not implemented on backend {backend}: {reason}")]
295 UnsupportedMode {
296 mode: &'static str,
297 backend: &'static str,
298 reason: &'static str,
299 },
300
301 #[error("{provider} account rejected the request (HTTP {status}): {message}")]
316 ProviderAccount {
317 provider: String,
318 status: u16,
319 message: String,
320 },
321
322 #[error("inference failed: {message}")]
331 ProviderKeyMissing {
332 provider: String,
333 model: String,
334 env_vars: Vec<String>,
337 message: String,
340 },
341
342 #[error("no credential for proprietary provider '{provider}' (model {model}): {detail}")]
359 CredentialUnavailable {
360 provider: String,
361 model: String,
362 reason: CredentialFailure,
364 detail: String,
366 },
367
368 #[error("{provider} account has no workspace yet: {detail}")]
385 WorkspaceRequired {
386 provider: String,
388 detail: String,
390 },
391
392 #[error("{provider} refused this request on content grounds{}: {message}",
412 content_refusal_detail(.kind.as_deref(), .code.as_deref()))]
413 ContentRefused {
414 provider: String,
415 kind: Option<String>,
417 code: Option<String>,
418 message: String,
419 },
420
421 #[error("{provider} gateway has no upstream configured for '{namespace}' (HTTP {status}): {message}")]
439 GatewayUnconfigured {
440 provider: String,
441 namespace: String,
442 status: u16,
443 message: String,
444 },
445
446 #[error("tokenization error: {0}")]
447 TokenizationError(String),
448
449 #[error("device error: {0}")]
450 DeviceError(String),
451
452 #[error("io error: {0}")]
453 Io(#[from] std::io::Error),
454}
455
456impl From<resource_policy::LocalAdmissionError> for InferenceError {
457 fn from(error: resource_policy::LocalAdmissionError) -> Self {
458 let recovery = match error.preflight.verdict {
459 resource_policy::LocalLoadVerdict::DisabledByPolicy => {
460 "Local model loading is disabled by the 0 GB allocation. Increase Local Models RAM in Settings, or choose a remote model. Downloads remain available.".to_string()
461 }
462 resource_policy::LocalLoadVerdict::ExceedsConfiguredCeiling => format!(
463 "This model needs about {} MB for this request, beyond the configured {} MB local-model allocation. Increase the allocation or choose a smaller model.",
464 error.preflight.estimated_incremental_mb,
465 error.preflight.configured_ceiling_mb
466 ),
467 resource_policy::LocalLoadVerdict::InsufficientLiveMemory => format!(
468 "Not enough memory is free to start this model while preserving CAR's {} MB emergency reserve. Close memory-heavy apps or choose a smaller model.",
469 error.preflight.emergency_reserve_mb
470 ),
471 resource_policy::LocalLoadVerdict::LiveMemoryUnknown => {
472 "CAR could not measure live memory. The static allocation fits, but current safety is unknown.".to_string()
473 }
474 resource_policy::LocalLoadVerdict::ModelMaintenance => {
475 "This local model is being removed or maintained. Wait for that operation to finish, then retry.".to_string()
476 }
477 resource_policy::LocalLoadVerdict::PendingTeardown => {
478 "CAR is still confirming that the previous local model process exited. Wait for teardown to finish, then retry.".to_string()
479 }
480 resource_policy::LocalLoadVerdict::Allowed => error.to_string(),
481 };
482 Self::LocalResourceBlocked {
483 preflight: error.preflight,
484 recovery,
485 }
486 }
487}
488
489fn error_counts_against_circuit_breaker(e: &InferenceError) -> bool {
526 failure_attribution(e).is_none()
527}
528
529fn content_refusal_detail(kind: Option<&str>, code: Option<&str>) -> String {
535 match (kind, code) {
536 (Some(k), Some(c)) => format!(" (type={k}, code={c})"),
537 (Some(k), None) => format!(" (type={k})"),
538 (None, Some(c)) => format!(" (code={c})"),
539 (None, None) => String::new(),
540 }
541}
542
543fn record_dispatch_failure(tracker: &mut OutcomeTracker, trace_id: &str, error: &InferenceError) {
549 match failure_attribution(error) {
550 Some(reason) => tracker.record_not_model_fault(trace_id, &error.to_string(), reason),
551 None => tracker.record_failure(trace_id, &error.to_string()),
552 }
553}
554
555fn failure_attribution(error: &InferenceError) -> Option<outcome::UnattributedReason> {
561 use outcome::UnattributedReason as R;
562 match error {
563 InferenceError::UnsupportedMode { .. }
564 | InferenceError::CatalogPreconditionMismatch { .. }
565 | InferenceError::InvalidClassifyLabels { .. } => Some(R::Capability),
567 InferenceError::ProviderAccount { .. }
568 | InferenceError::ProviderKeyMissing { .. }
569 | InferenceError::CredentialUnavailable { .. }
570 | InferenceError::WorkspaceRequired { .. }
571 | InferenceError::GatewayUnconfigured { .. } => Some(R::Account),
572 InferenceError::ContentRefused { .. } => Some(R::ContentRefused),
573 InferenceError::LocalResourceBlocked { .. } => Some(R::ResourceBlocked),
574 InferenceError::ControlledTermination => Some(R::Cancelled),
575 InferenceError::ModelNotFound { .. }
576 | InferenceError::NoEligibleModel { .. }
577 | InferenceError::DownloadFailed { .. }
578 | InferenceError::InferenceFailed { .. }
579 | InferenceError::ClassifyNoAnswer { .. }
580 | InferenceError::ModelManagement { .. }
581 | InferenceError::Transient { .. }
582 | InferenceError::DeadlineExceeded { .. }
583 | InferenceError::TokenizationError { .. }
584 | InferenceError::DeviceError { .. }
585 | InferenceError::Io { .. } => None,
586 }
587}
588
589#[cfg(any(
595 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
596 all(target_os = "ios", target_arch = "aarch64")
597))]
598fn send_stream_failure_blocking(
599 tx: &tokio::sync::mpsc::Sender<stream::StreamEvent>,
600 error: &InferenceError,
601) {
602 if let Some(reason) = failure_attribution(error) {
603 let _ = tx.blocking_send(stream::StreamEvent::Attribution(reason));
604 }
605 let _ = tx.blocking_send(stream::StreamEvent::Error(error.to_string()));
606}
607
608fn error_ends_fallback_chain(e: &InferenceError) -> bool {
630 if matches!(e, InferenceError::ContentRefused { provider, .. } if provider == "apple") {
637 return false;
638 }
639 matches!(
640 e,
641 InferenceError::ContentRefused { .. }
642 | InferenceError::CatalogPreconditionMismatch { .. }
643 | InferenceError::ControlledTermination
644 | InferenceError::WorkspaceRequired { .. }
650 )
651}
652
653fn apply_exhaustion_recovery_hint(underlying: InferenceError) -> InferenceError {
664 if matches!(
665 underlying,
666 InferenceError::ContentRefused { .. }
667 | InferenceError::CatalogPreconditionMismatch { .. }
668 | InferenceError::ControlledTermination
669 | InferenceError::WorkspaceRequired { .. }
674 ) {
675 return underlying;
676 }
677 let underlying_str = underlying.to_string();
678 match no_backend_recovery_hint(&underlying_str)
679 .or_else(|| auth_expired_recovery_hint(&underlying_str))
680 {
681 Some(msg) => InferenceError::InferenceFailed(msg),
682 None => underlying,
683 }
684}
685
686const AUTH_LOGIN_MARKER: &str = "auth login";
687const AUTH_STORE_UNREADABLE_MARKER: &str = "credential store unreadable";
688const AUTH_ENV_MISSING_MARKER: &str = "credential environment variable missing";
689
690const AUTH_FAILURE_MESSAGE_MARKERS: &[&str] = &[
695 "no credential for proprietary",
696 AUTH_LOGIN_MARKER,
697 "session has expired",
698 "cannot read parslee credentials",
699 AUTH_STORE_UNREADABLE_MARKER,
700 AUTH_ENV_MISSING_MARKER,
701 "credential was rejected",
707 "repair its provider login",
708];
709
710#[derive(Debug, Clone, PartialEq, Eq)]
720struct RouteCredentialFailure {
721 summary: String,
722 source_error: String,
723}
724
725fn parslee_signed_out_route_failure() -> RouteCredentialFailure {
726 let summary = "Parslee login is absent — run `car auth login` before retrying".to_string();
727 RouteCredentialFailure {
728 source_error: summary.clone(),
729 summary,
730 }
731}
732
733fn route_credential_failure(
734 candidate: &str,
735 error: &InferenceError,
736 promote_missing_credential: bool,
737) -> Option<String> {
738 match error {
739 InferenceError::CredentialUnavailable {
740 provider,
741 reason,
742 detail,
743 ..
744 } => {
745 let provider_name = if provider.eq_ignore_ascii_case("parslee") {
746 "Parslee".to_string()
747 } else {
748 provider.clone()
749 };
750 let summary = match reason {
751 CredentialFailure::Expired { .. } => format!(
752 "{provider_name} login expired for `{candidate}` — run `car auth login`"
753 ),
754 CredentialFailure::SignedOut => format!(
755 "{provider_name} login is absent for `{candidate}` — run `car auth login`"
756 ),
757 CredentialFailure::StoreUnreadable => format!(
758 "{provider_name} {AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
759 ),
760 CredentialFailure::EnvVarMissing { .. } if !promote_missing_credential => {
761 return None;
762 }
763 CredentialFailure::EnvVarMissing { env_var } => format!(
764 "{provider_name} {AUTH_ENV_MISSING_MARKER}: `{env_var}` for explicitly requested `{candidate}` — {detail}"
765 ),
766 CredentialFailure::RaceRetryable => return None,
769 };
770 Some(summary)
771 }
772 InferenceError::ProviderAccount {
773 provider, status, ..
774 } if matches!(*status, 401 | 403) => {
775 if provider.eq_ignore_ascii_case("parslee") {
776 Some(format!(
777 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
778 ))
779 } else {
780 Some(format!(
781 "{provider} credential was rejected for `{candidate}` (HTTP {status})"
782 ))
783 }
784 }
785 _ => {
786 let rendered = error.to_string();
787 let lower = rendered.to_ascii_lowercase();
788 if is_auth_rejection_message(&rendered) {
789 if candidate
790 .split_once('/')
791 .is_some_and(|(provider, _)| provider.eq_ignore_ascii_case("parslee"))
792 {
793 Some(format!(
794 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
795 ))
796 } else {
797 Some(format!(
798 "credential expired or was rejected for `{candidate}` — repair its provider login"
799 ))
800 }
801 } else if lower.contains(AUTH_STORE_UNREADABLE_MARKER)
802 || (promote_missing_credential && lower.contains("keychain lookup failed"))
803 {
804 Some(format!(
805 "{AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
806 ))
807 } else {
808 None
809 }
810 }
811 }
812}
813
814fn record_route_credential_failure(
815 slot: &mut Option<RouteCredentialFailure>,
816 candidate: &str,
817 error: &InferenceError,
818 promote_missing_credential: bool,
819) {
820 if let Some(summary) = route_credential_failure(candidate, error, promote_missing_credential) {
821 *slot = Some(RouteCredentialFailure {
826 summary,
827 source_error: error.to_string(),
828 });
829 }
830}
831
832fn chain_includes_parslee_route<'a>(
837 mut resolve: impl FnMut(&str) -> Option<&'a ModelSchema>,
838 chain: &[String],
839) -> bool {
840 chain.iter().any(|candidate| {
841 resolve(candidate).is_some_and(|schema| schema.provider.eq_ignore_ascii_case("parslee"))
842 })
843}
844
845fn apply_route_failure_context(
865 underlying: InferenceError,
866 credential: Option<&RouteCredentialFailure>,
867) -> InferenceError {
868 if matches!(
869 underlying,
870 InferenceError::ContentRefused { .. }
871 | InferenceError::CatalogPreconditionMismatch { .. }
872 | InferenceError::ControlledTermination
873 | InferenceError::WorkspaceRequired { .. }
881 ) {
882 return underlying;
883 }
884 let Some(credential) = credential else {
885 return apply_exhaustion_recovery_hint(underlying);
886 };
887 let rendered = underlying.to_string();
888 let summary = credential.summary.as_str();
889 let setup_hint = no_backend_recovery_hint(&rendered);
890 let connective = if rendered == credential.source_error {
891 "provider detail"
892 } else {
893 "fallback then failed"
894 };
895 let augment = |field: String| match &setup_hint {
896 Some(hint) => format!("{summary}; {hint}"),
898 None => format!("{summary}; {connective}: {field}"),
899 };
900 match underlying {
901 InferenceError::InferenceFailed(message) => {
902 InferenceError::InferenceFailed(augment(message))
903 }
904 InferenceError::ProviderKeyMissing {
905 provider,
906 model,
907 env_vars,
908 message,
909 } => InferenceError::ProviderKeyMissing {
910 provider,
911 model,
912 env_vars,
913 message: augment(message),
914 },
915 InferenceError::Transient { status, message } => InferenceError::Transient {
916 status,
917 message: augment(message),
918 },
919 InferenceError::ProviderAccount {
920 provider,
921 status,
922 message,
923 } => InferenceError::ProviderAccount {
924 provider,
925 status,
926 message: augment(message),
927 },
928 InferenceError::CredentialUnavailable {
929 provider,
930 model,
931 reason,
932 detail,
933 } => InferenceError::CredentialUnavailable {
934 provider,
935 model,
936 reason,
937 detail: augment(detail),
938 },
939 InferenceError::GatewayUnconfigured {
940 provider,
941 namespace,
942 status,
943 message,
944 } => InferenceError::GatewayUnconfigured {
945 provider,
946 namespace,
947 status,
948 message: augment(message),
949 },
950 InferenceError::LocalResourceBlocked {
951 preflight,
952 recovery,
953 } => InferenceError::LocalResourceBlocked {
954 preflight,
955 recovery: augment(recovery),
956 },
957 _ => InferenceError::InferenceFailed(augment(rendered)),
960 }
961}
962
963#[derive(Debug, Clone, PartialEq, Eq)]
972pub enum CredentialFailure {
973 Expired { expires_at: u64 },
980 SignedOut,
983 StoreUnreadable,
987 EnvVarMissing { env_var: String },
989 RaceRetryable,
992}
993
994#[derive(Debug, Clone, Copy, PartialEq, Eq)]
996pub enum Device {
997 Cpu,
998 Metal,
999 Cuda(usize), }
1001
1002impl Device {
1003 pub fn auto() -> Self {
1011 #[cfg(all(target_os = "macos", feature = "metal"))]
1012 {
1013 return Device::Metal;
1014 }
1015 #[cfg(all(
1016 any(target_os = "linux", target_os = "windows"),
1017 target_arch = "x86_64",
1018 not(car_skip_cuda)
1019 ))]
1020 {
1021 return Device::Cuda(0);
1022 }
1023 #[cfg(not(any(
1024 all(target_os = "macos", feature = "metal"),
1025 all(
1026 any(target_os = "linux", target_os = "windows"),
1027 target_arch = "x86_64",
1028 not(car_skip_cuda)
1029 )
1030 )))]
1031 {
1032 Device::Cpu
1033 }
1034 }
1035}
1036
1037#[derive(Debug, Clone)]
1039pub struct InferenceConfig {
1040 pub models_dir: std::path::PathBuf,
1046 pub state_root: std::path::PathBuf,
1055 pub device: Option<Device>,
1057 pub generation_model: String,
1059 pub preferred_generation_model: Option<String>,
1061 pub embedding_model: String,
1063 pub preferred_embedding_model: Option<String>,
1065 pub classification_model: String,
1067 pub preferred_classification_model: Option<String>,
1069}
1070
1071impl Default for InferenceConfig {
1072 fn default() -> Self {
1073 let models_dir = default_models_dir();
1090
1091 let hw = HardwareInfo::detect();
1092
1093 Self {
1094 models_dir,
1095 state_root: car_home::root_or_relative(),
1096 device: None,
1097 generation_model: hw.recommended_model,
1098 preferred_generation_model: None,
1099 embedding_model: "Qwen3-Embedding-0.6B".to_string(),
1100 preferred_embedding_model: None,
1101 classification_model: "Qwen3-0.6B".to_string(),
1102 preferred_classification_model: None,
1103 }
1104 }
1105}
1106
1107impl InferenceConfig {
1108 pub fn state_models_dir(&self) -> std::path::PathBuf {
1117 self.state_root.join("models")
1118 }
1119}
1120
1121pub fn default_models_dir() -> std::path::PathBuf {
1128 dirs_next()
1129 .unwrap_or_else(|| std::path::PathBuf::from("."))
1130 .join(".car")
1131 .join("models")
1132}
1133
1134fn dirs_next() -> Option<std::path::PathBuf> {
1135 std::env::var_os("HOME")
1139 .or_else(|| std::env::var_os("USERPROFILE"))
1140 .map(std::path::PathBuf::from)
1141}
1142
1143fn model_source_identity(schema: &ModelSchema) -> &str {
1144 match &schema.source {
1145 ModelSource::Local { hf_repo, .. }
1146 | ModelSource::Mlx { hf_repo, .. }
1147 | ModelSource::ManagedVllmMlx { hf_repo, .. } => hf_repo,
1148 ModelSource::WhisperCpp { model } | ModelSource::CodexCli { model } => model,
1149 _ => &schema.id,
1150 }
1151}
1152
1153#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1155pub struct TokenUsage {
1156 pub prompt_tokens: u64,
1166 pub completion_tokens: u64,
1168 pub total_tokens: u64,
1170 pub context_window: u64,
1172 #[serde(default)]
1177 pub cache_read_input_tokens: u64,
1178 #[serde(default)]
1183 pub cache_creation_input_tokens: u64,
1184}
1185
1186#[derive(
1188 Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq, schemars::JsonSchema,
1189)]
1190pub struct InferenceModelIdentity {
1191 #[serde(default)]
1194 pub requested_model_id: Option<String>,
1195 #[serde(default)]
1209 pub resolved_model_id: String,
1210 #[serde(default)]
1212 pub row_digest: String,
1213 #[serde(default)]
1215 pub catalog_revision: String,
1216}
1217
1218#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1220pub struct InferenceResult {
1221 pub text: String,
1223 pub tool_calls: Vec<crate::tasks::generate::ToolCall>,
1225 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1232 pub bounding_boxes: Vec<crate::tasks::grounding::BoundingBox>,
1233 pub trace_id: String,
1235 pub model_used: String,
1238 #[serde(flatten)]
1241 pub model_identity: InferenceModelIdentity,
1242 pub latency_ms: u64,
1244 #[serde(default)]
1256 pub time_to_first_token_ms: Option<u64>,
1257 pub usage: Option<TokenUsage>,
1276 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1289 pub provider_output_items: Vec<serde_json::Value>,
1290 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1298 pub thinking: Vec<crate::tasks::generate::ThinkingBlock>,
1299 #[serde(default)]
1315 pub stop_reason: Option<String>,
1316 #[serde(default, skip_serializing_if = "Option::is_none")]
1325 pub auth_fallback_from: Option<String>,
1326 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
1333 pub local_last_resort: bool,
1334 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1354 pub fallback_from: Vec<FallbackFrom>,
1355}
1356
1357#[derive(
1359 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1360)]
1361pub struct FallbackFrom {
1362 pub candidate: String,
1364 pub reason: FallbackReason,
1365}
1366
1367#[derive(
1375 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1376)]
1377#[serde(rename_all = "snake_case")]
1378pub enum FallbackReason {
1379 CredentialRejected,
1388 CredentialAbsent,
1391 RateLimited,
1394 QuotaExhausted,
1400 TimedOut,
1408 Failed,
1414}
1415
1416pub fn classify_fallback_reason(error: &InferenceError) -> FallbackReason {
1449 use FallbackReason as R;
1450 match error {
1451 InferenceError::ProviderAccount { status, .. } => match status {
1454 402 => R::QuotaExhausted,
1455 _ => R::CredentialRejected,
1456 },
1457 InferenceError::CredentialUnavailable { reason, .. } => match reason {
1459 CredentialFailure::Expired { .. } => R::CredentialRejected,
1461 CredentialFailure::SignedOut | CredentialFailure::EnvVarMissing { .. } => {
1463 R::CredentialAbsent
1464 }
1465 CredentialFailure::StoreUnreadable | CredentialFailure::RaceRetryable => R::Failed,
1468 },
1469 InferenceError::Transient {
1470 status: Some(429), ..
1471 } => R::RateLimited,
1472 InferenceError::Transient { status: None, .. } => R::Failed,
1479 InferenceError::Transient { .. } => R::Failed,
1480 InferenceError::DeadlineExceeded { .. } => R::TimedOut,
1483 InferenceError::InferenceFailed(msg)
1484 | InferenceError::ProviderKeyMissing { message: msg, .. } => classify_untyped_failure(msg),
1485 _ => R::Failed,
1486 }
1487}
1488
1489fn classify_untyped_failure(msg: &str) -> FallbackReason {
1497 use FallbackReason as R;
1498 if let Some(status) = parse_api_returned_status(msg) {
1501 return match status {
1502 401 | 403 => R::CredentialRejected,
1503 402 => R::QuotaExhausted,
1504 429 => R::RateLimited,
1505 408 | 504 => R::TimedOut,
1506 _ => R::Failed,
1507 };
1508 }
1509 let l = msg.to_ascii_lowercase();
1510 if l.contains("org lookup failed") {
1513 return if l.contains("http 401") || l.contains("http 403") {
1514 R::CredentialRejected
1515 } else if l.contains("http 429") {
1516 R::RateLimited
1517 } else {
1518 R::Failed
1519 };
1520 }
1521 if l.contains("authentication required")
1522 || l.contains("invalid_grant")
1523 || l.contains("token expired")
1524 {
1525 return R::CredentialRejected;
1526 }
1527 if l.contains("no credential for proprietary") || l.contains("no api key") {
1528 return R::CredentialAbsent;
1529 }
1530 if l.contains("too many requests") || l.contains("rate limit") {
1531 return R::RateLimited;
1532 }
1533 if l.contains("timed out") || l.contains("deadline exceeded") {
1534 return R::TimedOut;
1535 }
1536 R::Failed
1537}
1538
1539pub fn record_fallback_from(hops: &mut Vec<FallbackFrom>, candidate: &str, error: &InferenceError) {
1545 hops.push(FallbackFrom {
1546 candidate: candidate.to_string(),
1547 reason: classify_fallback_reason(error),
1548 });
1549}
1550
1551pub struct TrackedStream {
1558 pub model_used: String,
1560 pub trace_id: String,
1562 pub events: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
1564}
1565
1566struct AbortOnDropTask<T>(Option<tokio::task::JoinHandle<T>>);
1567
1568impl<T> AbortOnDropTask<T> {
1569 async fn join(mut self) -> Result<T, tokio::task::JoinError> {
1570 self.0.take().expect("owned task handle available").await
1571 }
1572}
1573
1574impl<T> Drop for AbortOnDropTask<T> {
1575 fn drop(&mut self) {
1576 if let Some(task) = self.0.take() {
1577 task.abort();
1578 }
1579 }
1580}
1581
1582tokio::task_local! {
1583 static PINNED_REGISTRY: (usize, Arc<UnifiedRegistry>);
1587}
1588
1589fn bound_model_identity(
1590 snapshot: &CatalogSnapshot,
1591 requested_model_id: Option<&str>,
1592 resolved_model_id: &str,
1593) -> Result<InferenceModelIdentity, InferenceError> {
1594 let row = snapshot.model_by_exact_id(resolved_model_id).ok_or_else(|| {
1595 InferenceError::InferenceFailed(format!(
1596 "resolved model `{resolved_model_id}` was absent from the catalog snapshot bound to this request"
1597 ))
1598 })?;
1599 Ok(InferenceModelIdentity {
1600 requested_model_id: requested_model_id.map(str::to_string),
1601 resolved_model_id: row.model.id.clone(),
1602 row_digest: row.row_digest.clone(),
1603 catalog_revision: snapshot.catalog_revision.clone(),
1604 })
1605}
1606
1607fn validate_expected_catalog_revision(
1608 req: &GenerateRequest,
1609 snapshot: &CatalogSnapshot,
1610) -> Result<(), InferenceError> {
1611 if let Some(expected) = req.expected_catalog_revision.as_deref() {
1612 if expected != snapshot.catalog_revision {
1613 return Err(InferenceError::CatalogPreconditionMismatch {
1614 detail: format!(
1615 "expected catalog revision {expected}, got {}",
1616 snapshot.catalog_revision
1617 ),
1618 });
1619 }
1620 }
1621 Ok(())
1622}
1623
1624fn validate_expected_catalog_row(
1625 req: &GenerateRequest,
1626 snapshot: &CatalogSnapshot,
1627 resolved_model_id: &str,
1628) -> Result<(), InferenceError> {
1629 let Some(expected) = req.expected_row_digest.as_deref() else {
1630 return Ok(());
1631 };
1632 let row = snapshot
1633 .model_by_exact_id(resolved_model_id)
1634 .ok_or_else(|| InferenceError::CatalogPreconditionMismatch {
1635 detail: format!(
1636 "resolved model `{resolved_model_id}` is absent from the bound catalog snapshot"
1637 ),
1638 })?;
1639 if expected != row.row_digest {
1640 return Err(InferenceError::CatalogPreconditionMismatch {
1641 detail: format!(
1642 "expected row digest {expected} for `{resolved_model_id}`, got {}",
1643 row.row_digest
1644 ),
1645 });
1646 }
1647 Ok(())
1648}
1649
1650const EXACT_MODEL_ID_PREFIX: &str = "\0car-exact-model-id:";
1651
1652pub fn pin_exact_model_id(req: &mut GenerateRequest, model_id: String) -> Result<(), String> {
1655 if req.model.is_some() {
1656 return Err("`model` and `model_id` are mutually exclusive".to_string());
1657 }
1658 if model_id.trim().is_empty() {
1659 return Err("`model_id` must be a non-empty immutable id".to_string());
1660 }
1661 req.model = Some(format!("{EXACT_MODEL_ID_PREFIX}{model_id}"));
1662 req.params.strict_model = true;
1663 Ok(())
1664}
1665
1666fn default_system_one_classifier<'a>(
1671 schemas: impl Iterator<Item = &'a ModelSchema>,
1672) -> Option<String> {
1673 schemas
1674 .filter(|schema| {
1675 schema.has_capability(ModelCapability::Classify)
1676 && schema.available_now()
1677 && matches!(
1678 &schema.source,
1679 ModelSource::Proprietary { protocol, .. }
1680 if protocol.wire == ProprietaryWire::SystemOne
1681 )
1682 })
1683 .map(|schema| schema.id.clone())
1684 .min()
1685}
1686
1687pub fn exact_pinned_model_id(req: &GenerateRequest) -> Option<&str> {
1688 req.model
1689 .as_deref()
1690 .and_then(|model| model.strip_prefix(EXACT_MODEL_ID_PREFIX))
1691}
1692
1693fn auto_thinking_budget(
1702 is_code_intent: bool,
1703 is_complex: bool,
1704 supports_thinking: bool,
1705) -> Option<usize> {
1706 if !supports_thinking {
1707 return None;
1708 }
1709 if is_code_intent {
1710 Some(24_000)
1711 } else if is_complex {
1712 Some(8_000)
1713 } else {
1714 None
1715 }
1716}
1717
1718fn is_explicit_code_intent(intent: Option<&intent::IntentHint>) -> bool {
1728 intent.and_then(|h| h.task) == Some(intent::TaskHint::Code)
1729}
1730
1731fn should_append_local_last_resort(chain_has_local: bool, strict_model: bool) -> bool {
1736 !chain_has_local && !strict_model
1737}
1738
1739fn resolved_max_tokens(requested: usize, schema: &ModelSchema) -> usize {
1763 if requested != crate::tasks::generate::DEFAULT_MAX_TOKENS
1764 || schema.decodes_in_process()
1765 || schema.is_codex_cli()
1766 {
1767 return requested;
1768 }
1769 schema.effective_max_output()
1770}
1771
1772const DEFAULT_LOCAL_DECODE_TIMEOUT_SECS: u64 = 300;
1785
1786pub const LOCAL_DECODE_TIMEOUT_STOP_REASON: &str = "local_decode_timeout";
1793
1794#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1802const LOCAL_DECODE_HEARTBEAT_SECS: u64 = 10;
1803
1804fn parse_decode_timeout(raw: Option<&str>) -> Option<std::time::Duration> {
1807 let secs = match raw {
1808 Some(v) => v
1809 .trim()
1810 .parse::<u64>()
1811 .unwrap_or(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS),
1812 None => DEFAULT_LOCAL_DECODE_TIMEOUT_SECS,
1813 };
1814 (secs > 0).then(|| std::time::Duration::from_secs(secs))
1815}
1816
1817#[cfg_attr(
1830 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
1831 allow(dead_code)
1832)]
1833fn deadline_exceeded(elapsed: std::time::Duration, timeout: Option<std::time::Duration>) -> bool {
1834 timeout.is_some_and(|limit| elapsed >= limit)
1835}
1836
1837#[cfg_attr(
1843 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
1844 allow(dead_code)
1845)]
1846fn heartbeat_due(
1847 elapsed: std::time::Duration,
1848 last: std::time::Duration,
1849 interval: std::time::Duration,
1850) -> bool {
1851 elapsed.saturating_sub(last) >= interval
1852}
1853
1854#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1858enum EmptyPassAction {
1859 RetryWithoutThinking,
1862 FailDecodeCeiling,
1866 Accept,
1868}
1869
1870fn classify_empty_pass(
1878 recover: bool,
1879 stop_reason: Option<&str>,
1880 text: &str,
1881 tool_calls_empty: bool,
1882) -> EmptyPassAction {
1883 if !text.trim().is_empty() || !tool_calls_empty {
1884 return EmptyPassAction::Accept;
1885 }
1886 if stop_reason == Some(LOCAL_DECODE_TIMEOUT_STOP_REASON) {
1887 return EmptyPassAction::FailDecodeCeiling;
1888 }
1889 if recover {
1890 return EmptyPassAction::RetryWithoutThinking;
1891 }
1892 EmptyPassAction::Accept
1893}
1894
1895fn local_decode_timeout() -> Option<std::time::Duration> {
1896 parse_decode_timeout(
1897 std::env::var("CAR_LOCAL_DECODE_TIMEOUT_SECS")
1898 .ok()
1899 .as_deref(),
1900 )
1901}
1902
1903impl InferenceResult {
1904 pub fn served_model_id(&self) -> &str {
1909 if self.model_identity.resolved_model_id.is_empty() {
1910 &self.model_used
1911 } else {
1912 &self.model_identity.resolved_model_id
1913 }
1914 }
1915
1916 pub fn has_tool_calls(&self) -> bool {
1918 !self.tool_calls.is_empty()
1919 }
1920
1921 pub fn was_truncated(&self) -> bool {
1933 matches!(
1934 self.stop_reason.as_deref(),
1935 Some(
1936 "length"
1937 | "max_tokens"
1938 | "max_output_tokens"
1939 | "MAX_TOKENS"
1940 | crate::LOCAL_DECODE_TIMEOUT_STOP_REASON
1941 )
1942 )
1943 }
1944
1945 pub fn append_assistant_history(
1954 &self,
1955 messages: &mut Vec<crate::tasks::generate::Message>,
1956 tool_calls: Vec<crate::tasks::generate::ToolCall>,
1957 ) {
1958 if !self.provider_output_items.is_empty() {
1959 messages.push(crate::tasks::generate::Message::ProviderOutputItems {
1960 protocol: crate::protocol::OPENAI_RESPONSES_PROTOCOL.to_string(),
1961 items: self.provider_output_items.clone(),
1962 });
1963 }
1964 messages.push(crate::tasks::generate::Message::Assistant {
1965 content: self.text.clone(),
1966 tool_calls,
1967 thinking: self.thinking.clone(),
1968 model_id: Some(self.served_model_id().to_string()),
1969 local_last_resort: self.local_last_resort,
1970 });
1971 }
1972}
1973
1974#[derive(Debug, Clone, Serialize)]
1975pub struct SpeechRuntimeHealth {
1976 pub root: PathBuf,
1977 pub installed: bool,
1978 pub python: PathBuf,
1979 pub stt_command: PathBuf,
1980 pub tts_command: PathBuf,
1981 pub configured_python: Option<String>,
1982 pub detected_python: Option<String>,
1983}
1984
1985#[derive(Debug, Clone, Serialize)]
1986pub struct SpeechModelHealth {
1987 pub id: String,
1988 pub name: String,
1989 pub provider: String,
1990 pub capability: ModelCapability,
1991 pub is_local: bool,
1992 pub available: bool,
1993 pub cached: bool,
1994 pub selected_by_default: bool,
1995 pub source: String,
1996}
1997
1998#[derive(Debug, Clone, Serialize)]
1999pub struct SpeechHealthReport {
2000 pub runtime: SpeechRuntimeHealth,
2001 pub local_models: Vec<SpeechModelHealth>,
2002 pub remote_models: Vec<SpeechModelHealth>,
2003 pub elevenlabs_configured: bool,
2004 pub prefer_local: bool,
2005 pub allow_remote_fallback: bool,
2006 pub preferred_local_stt: Option<String>,
2007 pub preferred_local_tts: Option<String>,
2008 pub preferred_remote_stt: Option<String>,
2009 pub preferred_remote_tts: Option<String>,
2010 pub local_stt_default: Option<String>,
2011 pub local_tts_default: Option<String>,
2012 pub remote_stt_default: Option<String>,
2013 pub remote_tts_default: Option<String>,
2014}
2015
2016#[derive(Debug, Clone, Serialize)]
2017pub struct ModelDefaultHealth {
2018 pub capability: ModelCapability,
2019 pub configured_model: String,
2020 pub available: bool,
2021 pub is_local: bool,
2022 pub provider: Option<String>,
2023}
2024
2025#[derive(Debug, Clone, Serialize)]
2026pub struct ModelProviderHealth {
2027 pub provider: String,
2028 pub configured: bool,
2029 pub local_models: usize,
2030 pub remote_models: usize,
2031 pub available_models: usize,
2032 pub capabilities: Vec<ModelCapability>,
2033}
2034
2035#[derive(Debug, Clone, Serialize)]
2036pub struct ModelCapabilityHealth {
2037 pub capability: ModelCapability,
2038 pub total_models: usize,
2039 pub available_models: usize,
2040 pub local_available_models: usize,
2041 pub remote_available_models: usize,
2042}
2043
2044#[derive(Debug, Clone, Serialize)]
2045pub struct RoutingScenarioHealth {
2046 pub name: String,
2047 pub workload: RoutingWorkload,
2048 pub task_family: String,
2049 pub has_tools: bool,
2050 pub has_vision: bool,
2051 pub prefer_local: bool,
2052 pub quality_first_cold_start: bool,
2053 pub bootstrap_min_task_observations: u64,
2054 pub bootstrap_quality_floor: f64,
2055 pub model_id: String,
2056 pub model_name: String,
2057 pub reason: String,
2058 pub strategy: RoutingStrategy,
2059}
2060
2061#[derive(Debug, Clone, Serialize)]
2062pub struct ModelBenchmarkPriorHealth {
2063 pub model_id: String,
2064 pub model_name: Option<String>,
2065 pub overall_score: f64,
2066 pub overall_latency_ms: Option<f64>,
2067 pub task_scores: std::collections::HashMap<String, f64>,
2068 pub task_latency_ms: std::collections::HashMap<String, f64>,
2069 pub source_path: PathBuf,
2070}
2071
2072#[derive(Debug, Clone, Serialize)]
2073pub struct ModelHealthReport {
2074 pub total_models: usize,
2075 pub available_models: usize,
2076 pub local_models: usize,
2077 pub remote_models: usize,
2078 pub defaults: Vec<ModelDefaultHealth>,
2079 pub providers: Vec<ModelProviderHealth>,
2080 pub capabilities: Vec<ModelCapabilityHealth>,
2081 pub routing_prefer_local: bool,
2082 pub routing_quality_first_cold_start: bool,
2083 pub routing_min_observations: u64,
2084 pub routing_bootstrap_min_task_observations: u64,
2085 pub routing_bootstrap_quality_floor: f64,
2086 pub routing_quality_weight: f64,
2087 pub routing_latency_weight: f64,
2088 pub routing_cost_weight: f64,
2089 pub routing_scenarios: Vec<RoutingScenarioHealth>,
2090 pub benchmark_priors: Vec<ModelBenchmarkPriorHealth>,
2091 pub speech: SpeechHealthReport,
2092}
2093
2094#[derive(Debug, Clone, Serialize)]
2095pub struct SpeechInstallReport {
2096 pub name: String,
2097 pub hf_repo: String,
2098 pub snapshot_path: PathBuf,
2099 pub files_downloaded: usize,
2100}
2101
2102#[derive(Debug, Clone, Serialize)]
2103pub struct SpeechSmokePathReport {
2104 pub path: String,
2105 pub tts_model: String,
2106 pub stt_model: String,
2107 pub audio_path: PathBuf,
2108 pub transcript: String,
2109}
2110
2111#[derive(Debug, Clone, Serialize, Default)]
2112pub struct SpeechSmokeReport {
2113 pub local: Option<SpeechSmokePathReport>,
2114 pub remote: Option<SpeechSmokePathReport>,
2115 pub skipped: Vec<String>,
2116}
2117
2118#[derive(Debug, Clone, Serialize, Default)]
2119pub struct SpeechPolicy {
2120 pub prefer_local: bool,
2121 pub allow_remote_fallback: bool,
2122 pub preferred_local_stt: Option<String>,
2123 pub preferred_local_tts: Option<String>,
2124 pub preferred_remote_stt: Option<String>,
2125 pub preferred_remote_tts: Option<String>,
2126}
2127
2128fn render_for_local_backend(mut req: GenerateRequest) -> GenerateRequest {
2137 let has_msgs = req.messages.as_ref().is_some_and(|m| !m.is_empty());
2138 let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
2139 if has_msgs || has_tools {
2140 req.prompt = tasks::generate::render_chat_prompt(&req);
2141 req.messages = None;
2142 req.tools = None;
2143 }
2144 req
2145}
2146
2147#[cfg(test)]
2148static UNRETIRED_RELEASE_WARNING_COUNT: std::sync::atomic::AtomicUsize =
2149 std::sync::atomic::AtomicUsize::new(0);
2150
2151#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2165fn foundation_models_usage(
2166 instructions: Option<&str>,
2167 prompt: &str,
2168 completion: &str,
2169 context_window: u64,
2170) -> Option<crate::TokenUsage> {
2171 let (prompt_tokens, completion_tokens) =
2172 crate::backend::foundation_models::count_tokens(instructions, prompt, completion)?;
2173 Some(crate::TokenUsage {
2174 prompt_tokens,
2175 completion_tokens,
2176 total_tokens: prompt_tokens + completion_tokens,
2177 context_window,
2178 ..Default::default()
2180 })
2181}
2182
2183pub struct InferenceEngine {
2184 pub config: InferenceConfig,
2185 unified: std::sync::RwLock<Arc<UnifiedRegistry>>,
2191 registry_write: std::sync::Mutex<()>,
2194 pub adaptive_router: AdaptiveRouter,
2196 pub outcome_tracker: Arc<RwLock<OutcomeTracker>>,
2198 last_outcome_flush: Arc<std::sync::Mutex<Option<Instant>>>,
2204 outcome_profiles_loaded: bool,
2208 ledger_io_lock: Arc<tokio::sync::Mutex<()>>,
2213 spend_limits: Arc<std::sync::RwLock<Option<SpendLimits>>>,
2219 lane_defaults_cache: Arc<std::sync::RwLock<crate::lane_defaults::LaneDefaults>>,
2224 external_pins: Arc<std::sync::RwLock<std::collections::BTreeMap<String, Vec<String>>>>,
2228 disk_pressured: Arc<std::sync::atomic::AtomicBool>,
2231 failed_upgrades: Arc<std::sync::Mutex<std::collections::HashMap<String, u64>>>,
2235 concierge_action_lock: Arc<tokio::sync::Mutex<()>>,
2241 concierge_action_seq: Arc<std::sync::atomic::AtomicU64>,
2245 remote_backend: RemoteBackend,
2247 model_management: model_management::ModelManagementStore,
2250 local_admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2252 _runtime_scope: Arc<ScopedInferenceRuntime>,
2257 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2258 resource_policy_evidence: Arc<std::sync::RwLock<resource_policy::ResourcePolicyLoadEvidence>>,
2262 model_budget: Arc<backend_cache::SharedModelBudget>,
2265 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2271 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2272 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2279 local_backends:
2280 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2281 pub registry: models::ModelRegistry,
2283 pub router: ModelRouter,
2284 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2285 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2286 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2287 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2288 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2289 speech_policy: SpeechPolicy,
2290 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2294}
2295
2296enum SpeechCandidateAdmission {
2297 Proceed(Option<resource_policy::LocalLoadReservation>),
2298 SkipBlocked(InferenceError),
2299 FailBlocked(InferenceError),
2300}
2301
2302struct ScopedInferenceRuntime {
2303 model_budget: Arc<backend_cache::SharedModelBudget>,
2304 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2305 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2306 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2307 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2308 local_backends:
2309 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2310 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2311 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2312 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2313 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2314 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2315 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2316 #[cfg(test)]
2317 load_probe: Arc<backend_cache::BackendCache<()>>,
2318}
2319
2320fn scoped_inference_runtime_registry() -> &'static std::sync::Mutex<
2321 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2322> {
2323 static REGISTRY: std::sync::OnceLock<
2324 std::sync::Mutex<
2325 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2326 >,
2327 > = std::sync::OnceLock::new();
2328 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
2329}
2330
2331fn configured_cache_budget_mb(default_mb: u64) -> u64 {
2332 std::env::var("CAR_INFERENCE_MODEL_CACHE_MB")
2333 .ok()
2334 .and_then(|value| value.parse::<u64>().ok())
2335 .unwrap_or(default_mb)
2336}
2337
2338fn scoped_inference_runtime(
2339 state_root: &Path,
2340 configured_ceiling_mb: u64,
2341 admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2342) -> Arc<ScopedInferenceRuntime> {
2343 let state_root = resource_policy::normalized_state_root_key(state_root);
2344 let mut registry = scoped_inference_runtime_registry()
2345 .lock()
2346 .unwrap_or_else(std::sync::PoisonError::into_inner);
2347 registry.retain(|_, runtime| runtime.strong_count() > 0);
2348 if let Some(runtime) = registry.get(&state_root).and_then(std::sync::Weak::upgrade) {
2349 runtime.model_budget.set_budget_bytes(
2350 configured_cache_budget_mb(configured_ceiling_mb).saturating_mul(1024 * 1024),
2351 );
2352 return runtime;
2353 }
2354
2355 let model_budget = backend_cache::SharedModelBudget::from_env_or(configured_ceiling_mb);
2356 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2357 let cache_idle = backend_cache::idle_ttl_from_env();
2358 let runtime = Arc::new(ScopedInferenceRuntime {
2359 model_budget: model_budget.clone(),
2360 resource_policy_generation: Arc::new(std::sync::atomic::AtomicU64::new(1)),
2361 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2362 text_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2363 model_budget.clone(),
2364 cache_idle,
2365 Some(admission.clone()),
2366 )),
2367 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2368 local_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2369 model_budget.clone(),
2370 cache_idle,
2371 Some(admission.clone()),
2372 )),
2373 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2374 backend: Arc::new(RwLock::new(std::collections::HashMap::new())),
2375 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2376 embedding_backend: Arc::new(RwLock::new(None)),
2377 speech_runtime: Arc::new(Mutex::new(None)),
2378 vllm_pool: Arc::new(
2382 vllm_pool::VllmServerPool::with_admission(
2383 std::time::Duration::from_secs(
2384 std::env::var("CAR_VLLM_IDLE_SECS")
2385 .ok()
2386 .and_then(|value| value.parse().ok())
2387 .unwrap_or(300),
2388 ),
2389 admission,
2390 )
2391 .with_process_records(state_root.join("run").join("vllm-mlx")),
2392 ),
2393 #[cfg(test)]
2394 load_probe: Arc::new(backend_cache::BackendCache::new(1024)),
2395 });
2396 registry.insert(state_root, Arc::downgrade(&runtime));
2397 runtime
2398}
2399
2400impl InferenceEngine {
2401 fn requires_local_admission(schema: &ModelSchema) -> bool {
2402 schema.is_car_managed_vllm_mlx() || Self::supports_worker_offload(schema)
2403 }
2404
2405 fn reserve_in_outer_dispatch(schema: &ModelSchema) -> bool {
2411 Self::requires_local_admission(schema) && !schema.is_car_managed_vllm_mlx()
2412 }
2413
2414 fn supports_worker_offload(schema: &ModelSchema) -> bool {
2415 matches!(
2416 schema.source,
2417 ModelSource::Local { .. } | ModelSource::Mlx { .. }
2418 )
2419 }
2420
2421 pub fn local_model_retention(&self, model_id: &str) -> backend_cache::BackendRetention {
2424 if self.local_admission.is_resident(model_id) {
2425 backend_cache::BackendRetention::Resident
2426 } else {
2427 backend_cache::BackendRetention::Transient
2428 }
2429 }
2430 pub fn local_model_preflight(
2432 &self,
2433 model_id: &str,
2434 context_tokens: usize,
2435 ) -> Result<resource_policy::LocalLoadPreflight, InferenceError> {
2436 let schema = self
2437 .resolve_schema(model_id)
2438 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
2439 self.ensure_model_enabled(&schema.id)?;
2440 Ok(self.local_admission.preflight(&schema, context_tokens))
2441 }
2442
2443 pub fn active_local_resource_policy(&self) -> resource_policy::ResourcePolicyLoadEvidence {
2450 let mut evidence = self
2451 .resource_policy_evidence
2452 .read()
2453 .unwrap_or_else(|poisoned| poisoned.into_inner())
2454 .clone();
2455 evidence.policy = self.local_admission.policy();
2456 evidence
2457 }
2458
2459 pub fn apply_local_resource_policy(&self, policy: resource_policy::ResourcePolicy) {
2463 let ceiling_mb = policy
2464 .effective_budget(HardwareInfo::detect().total_ram_mb)
2465 .configured_model_ceiling_mb;
2466 self.local_admission.set_policy(policy.clone());
2467 *self
2468 .resource_policy_evidence
2469 .write()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner()) =
2471 resource_policy::ResourcePolicyLoadEvidence {
2472 policy,
2473 source: resource_policy::ResourcePolicyLoadSource::Loaded,
2474 warning: None,
2475 };
2476 let cache_ceiling_mb = configured_cache_budget_mb(ceiling_mb);
2477 self.model_budget
2478 .set_budget_bytes(cache_ceiling_mb.saturating_mul(1024 * 1024));
2479 let generation = self
2480 .resource_policy_generation
2481 .fetch_add(1, std::sync::atomic::Ordering::AcqRel)
2482 .saturating_add(1);
2483 if let Some(offload) = crate::offload::current_local_offload() {
2484 offload.refresh_resource_policy(generation);
2485 }
2486 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2487 {
2488 self.text_backends.enforce_budget();
2489 self.local_backends.enforce_budget();
2490 }
2491 }
2492
2493 pub fn begin_local_model_maintenance(
2494 &self,
2495 model_id: &str,
2496 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2497 {
2498 self.local_admission.begin_model_maintenance(model_id)
2499 }
2500
2501 pub async fn prepare_local_model_removal(
2502 &self,
2503 model_id: &str,
2504 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2505 {
2506 let maintenance = self.local_admission.begin_model_maintenance(model_id)?;
2507 if let Some(offload) = crate::offload::current_local_offload() {
2508 if offload
2509 .resident_models()
2510 .await
2511 .iter()
2512 .any(|resident| resident == model_id)
2513 {
2514 let acknowledged = offload.release_model(model_id).await.map_err(|error| {
2515 resource_policy::ModelMaintenanceError::ReleaseFailed(error.to_string())
2516 })?;
2517 if !acknowledged
2518 || offload
2519 .resident_models()
2520 .await
2521 .iter()
2522 .any(|resident| resident == model_id)
2523 {
2524 return Err(
2525 resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
2526 model_id.to_string(),
2527 ),
2528 );
2529 }
2530 }
2531 }
2532 if self
2533 .vllm_pool
2534 .release_model_if_present(model_id)
2535 .await
2536 .is_err()
2537 {
2538 return Err(
2539 resource_policy::ModelMaintenanceError::ProcessReleaseUnacknowledged(
2540 model_id.to_string(),
2541 ),
2542 );
2543 }
2544 if !self.evict_local_model_if_idle(model_id) {
2545 return Err(resource_policy::ModelMaintenanceError::CacheReleaseBlocked(
2546 model_id.to_string(),
2547 ));
2548 }
2549 let allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2550 if !allocation_ids.is_empty() {
2551 return Err(resource_policy::ModelMaintenanceError::ResidualResidency {
2552 model_id: model_id.to_string(),
2553 allocation_ids,
2554 });
2555 }
2556 Ok(maintenance)
2557 }
2558
2559 pub fn evict_local_model_if_idle(&self, model_id: &str) -> bool {
2563 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2564 {
2565 let mut allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2566 if !allocation_ids.iter().any(|id| id == model_id) {
2567 allocation_ids.push(model_id.to_string());
2568 }
2569 allocation_ids.into_iter().all(|allocation_id| {
2570 [
2571 self.text_backends.evict_if_idle(&allocation_id),
2572 self.local_backends.evict_if_idle(&allocation_id),
2573 ]
2574 .into_iter()
2575 .all(|evicted| evicted)
2576 })
2577 }
2578 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2579 {
2580 let _ = model_id;
2581 let backend_released = self.backend.try_write().is_ok_and(|mut backends| {
2582 if backends.remove(model_id).is_some() {
2583 self.local_admission.mark_evicted(model_id);
2584 }
2585 true
2586 });
2587 backend_released
2591 && self
2592 .embedding_backend
2593 .try_write()
2594 .is_ok_and(|slot| slot.is_none())
2595 }
2596 }
2597
2598 fn reserve_local_request(
2599 &self,
2600 schema: &ModelSchema,
2601 context_tokens: usize,
2602 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2603 self.ensure_model_enabled(&schema.id)?;
2604 let activity_lease = self.model_management.acquire_lease(&schema.id)?;
2605 let mut reservation = self
2606 .local_admission
2607 .reserve(schema, context_tokens)
2608 .map_err(InferenceError::from)?;
2609 reservation.attach_activity_lease(activity_lease);
2610 Ok(reservation)
2611 }
2612
2613 async fn reserve_local_request_with_worker_retry(
2618 &self,
2619 schema: &ModelSchema,
2620 context_tokens: usize,
2621 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2622 let original_error = match self.reserve_local_request(schema, context_tokens) {
2623 Ok(reservation) => return Ok(reservation),
2624 Err(error) => error,
2625 };
2626 let first_verdict = match &original_error {
2627 InferenceError::LocalResourceBlocked { preflight, .. } => preflight.verdict.clone(),
2628 _ => return Err(original_error),
2629 };
2630 if first_verdict != resource_policy::LocalLoadVerdict::InsufficientLiveMemory {
2631 return Err(original_error);
2632 }
2633
2634 let Some(offload) = crate::offload::current_local_offload() else {
2635 tracing::info!(
2636 model = %schema.id,
2637 first_verdict = ?first_verdict,
2638 release_happened = false,
2639 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2640 "local admission worker release-and-retry decision"
2641 );
2642 return Err(original_error);
2643 };
2644 if !offload
2645 .resident_models()
2646 .await
2647 .iter()
2648 .any(|model_id| model_id == &schema.id)
2649 {
2650 tracing::info!(
2651 model = %schema.id,
2652 first_verdict = ?first_verdict,
2653 release_happened = false,
2654 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2655 "local admission worker release-and-retry decision"
2656 );
2657 return Err(original_error);
2658 }
2659 let Some(scoped_allocation_id) = offload.resident_allocation_id(&schema.id) else {
2660 tracing::debug!(
2661 model = %schema.id,
2662 "release-and-retry skipped because the offload exposes no allocation id"
2663 );
2664 return Err(original_error);
2665 };
2666 match offload.release_model(&schema.id).await {
2667 Ok(true) => {
2668 let offload_still_resident = offload
2672 .resident_models()
2673 .await
2674 .iter()
2675 .any(|model_id| model_id == &schema.id);
2676 let allocation_still_charged = self
2679 .local_admission
2680 .resident_allocation_ids(&schema.id)
2681 .iter()
2682 .any(|allocation_id| allocation_id == &scoped_allocation_id);
2683 if offload_still_resident || allocation_still_charged {
2684 tracing::warn!(
2688 model = %schema.id,
2689 allocation = %scoped_allocation_id,
2690 offload_still_resident,
2691 allocation_still_charged,
2692 "release acknowledged but allocation {} still charged/pending",
2693 scoped_allocation_id
2694 );
2695 #[cfg(test)]
2696 UNRETIRED_RELEASE_WARNING_COUNT
2697 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2698 return Err(original_error);
2699 }
2700 let second = self.reserve_local_request(schema, context_tokens);
2701 let second_verdict = match &second {
2702 Ok(_) => Some("admitted".to_string()),
2703 Err(InferenceError::LocalResourceBlocked { preflight, .. }) => {
2704 Some(format!("{:?}", preflight.verdict))
2705 }
2706 Err(_) => None,
2707 };
2708 tracing::info!(
2709 model = %schema.id,
2710 first_verdict = ?first_verdict,
2711 release_happened = true,
2712 second_verdict = ?second_verdict,
2713 "local admission worker release-and-retry decision"
2714 );
2715 second
2716 }
2717 Ok(false) | Err(_) => {
2718 tracing::info!(
2719 model = %schema.id,
2720 first_verdict = ?first_verdict,
2721 release_happened = false,
2722 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2723 "local admission worker release-and-retry decision"
2724 );
2725 Err(original_error)
2726 }
2727 }
2728 }
2729
2730 #[cfg(any(
2735 test,
2736 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))
2737 ))]
2738 fn reconcile_transient_local_allocation(
2739 reservation: &mut resource_policy::LocalLoadReservation,
2740 measured_weights_bytes: u64,
2741 ) -> Result<(), InferenceError> {
2742 reservation
2743 .reconcile_measured_weights(measured_weights_bytes)
2744 .map(|_| ())
2745 .map_err(InferenceError::from)
2746 }
2747
2748 fn prepare_worker_admission(
2749 &self,
2750 schema: &ModelSchema,
2751 reservation: &mut resource_policy::LocalLoadReservation,
2752 ) -> Result<crate::offload::LocalWorkerAdmission, InferenceError> {
2753 let installed = self.config.models_dir.join(&schema.name);
2754 let installed_weights_bytes = backend_cache::estimate_model_size(&installed);
2755 let measured_weights_bytes =
2762 installed_weights_bytes.max(reservation.reconciled_weights_bytes());
2763 reservation
2764 .reconcile_measured_weights(measured_weights_bytes)
2765 .map_err(InferenceError::from)?;
2766 Ok(crate::offload::LocalWorkerAdmission {
2767 policy: self.local_admission.policy(),
2768 policy_generation: self
2769 .resource_policy_generation
2770 .load(std::sync::atomic::Ordering::Acquire),
2771 state_root: resource_policy::normalized_state_root_key(&self.config.state_root),
2772 measured_weights_bytes,
2773 })
2774 }
2775
2776 async fn reconcile_worker_residency(
2777 offload: &dyn crate::offload::LocalGenerationOffload,
2778 expected_model_id: &str,
2779 residency: &crate::offload::LocalWorkerResidency,
2780 retention: backend_cache::BackendRetention,
2781 reservation: &mut resource_policy::LocalLoadReservation,
2782 ) -> Result<(), InferenceError> {
2783 if residency.model_id != expected_model_id {
2784 if retention == backend_cache::BackendRetention::Resident {
2785 let released = offload
2786 .release_model(&residency.model_id)
2787 .await
2788 .unwrap_or(false);
2789 if !released {
2790 let allocation_id = offload
2791 .resident_allocation_id(&residency.model_id)
2792 .unwrap_or_else(|| {
2793 resource_policy::worker_process_allocation_id(&residency.model_id)
2794 });
2795 reservation.publish_resident_weights_as(
2796 &allocation_id,
2797 residency.measured_weights_bytes,
2798 );
2799 }
2800 }
2801 return Err(InferenceError::InferenceFailed(format!(
2802 "local worker acknowledged model '{}' for requested '{}'",
2803 residency.model_id, expected_model_id
2804 )));
2805 }
2806 if retention == backend_cache::BackendRetention::Resident {
2807 let allocation_id = offload
2808 .resident_allocation_id(expected_model_id)
2809 .unwrap_or_else(|| {
2810 resource_policy::worker_process_allocation_id(expected_model_id)
2811 });
2812 reservation
2813 .publish_resident_weights_as(&allocation_id, residency.measured_weights_bytes);
2814 }
2815 Ok(())
2816 }
2817
2818 fn admit_speech_candidate(
2819 &self,
2820 schema: &ModelSchema,
2821 explicit: bool,
2822 ) -> SpeechCandidateAdmission {
2823 if !schema.is_local()
2824 || matches!(
2825 schema.source,
2826 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. }
2827 )
2828 {
2829 return SpeechCandidateAdmission::Proceed(None);
2830 }
2831 match self.reserve_local_request(schema, 0) {
2832 Ok(reservation) => SpeechCandidateAdmission::Proceed(Some(reservation)),
2833 Err(error) if explicit => SpeechCandidateAdmission::FailBlocked(error),
2834 Err(error) => SpeechCandidateAdmission::SkipBlocked(error),
2835 }
2836 }
2837
2838 fn hold_local_reservation_for_stream(
2839 mut source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
2840 reservation: resource_policy::LocalLoadReservation,
2841 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
2842 let (tx, rx) = tokio::sync::mpsc::channel(64);
2843 tokio::spawn(async move {
2844 let _reservation = reservation;
2845 while let Some(event) = source.recv().await {
2846 if tx.send(event).await.is_err() {
2847 break;
2848 }
2849 }
2850 });
2851 rx
2852 }
2853
2854 fn hold_optional_reservation_for_stream(
2855 source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
2856 reservation: Option<resource_policy::LocalLoadReservation>,
2857 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
2858 match reservation {
2859 Some(reservation) => Self::hold_local_reservation_for_stream(source, reservation),
2860 None => source,
2861 }
2862 }
2863
2864 pub fn set_spend_limits(&self, limits: Option<SpendLimits>) {
2870 *self.spend_limits.write().unwrap() = limits;
2871 }
2872
2873 fn preferred_model_for_capability(&self, capability: ModelCapability) -> Option<&str> {
2874 match capability {
2875 ModelCapability::Generate => self.config.preferred_generation_model.as_deref(),
2876 ModelCapability::Embed => self.config.preferred_embedding_model.as_deref(),
2877 ModelCapability::Classify => self.config.preferred_classification_model.as_deref(),
2878 _ => None,
2879 }
2880 }
2881
2882 fn request_has_tools(req: &GenerateRequest) -> bool {
2888 req.tools.as_ref().is_some_and(|t| !t.is_empty())
2889 }
2890
2891 fn request_needs_vision(req: &GenerateRequest) -> bool {
2892 req.images.as_ref().is_some_and(|images| !images.is_empty())
2893 || req.messages.as_ref().is_some_and(|messages| {
2894 messages
2895 .iter()
2896 .any(|msg| matches!(msg, Message::UserMultimodal { .. }))
2897 })
2898 }
2899
2900 #[allow(dead_code)] fn request_has_video(req: &GenerateRequest) -> bool {
2906 let images_have_video = req
2907 .images
2908 .as_ref()
2909 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_video));
2910 let messages_have_video = req.messages.as_ref().is_some_and(|messages| {
2911 messages.iter().any(|msg| match msg {
2912 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_video),
2913 _ => false,
2914 })
2915 });
2916 images_have_video || messages_have_video
2917 }
2918
2919 #[allow(dead_code)] fn request_has_audio(req: &GenerateRequest) -> bool {
2924 let images_have_audio = req
2925 .images
2926 .as_ref()
2927 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_audio));
2928 let messages_have_audio = req.messages.as_ref().is_some_and(|messages| {
2929 messages.iter().any(|msg| match msg {
2930 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_audio),
2931 _ => false,
2932 })
2933 });
2934 images_have_audio || messages_have_audio
2935 }
2936
2937 pub fn new(config: InferenceConfig) -> Self {
2938 let registry = models::ModelRegistry::new(config.models_dir.clone());
2939 let hw = HardwareInfo::detect();
2940 let policy_evidence =
2941 resource_policy::FileResourcePolicyRepository::new(config.state_root.clone())
2942 .load_with_evidence()
2943 .unwrap_or_else(|error| {
2944 tracing::warn!(%error, "failed to read local model resource policy; using Everyday");
2945 resource_policy::ResourcePolicyLoadEvidence {
2946 policy: resource_policy::ResourcePolicy::everyday(),
2947 source: resource_policy::ResourcePolicyLoadSource::CorruptDefault,
2948 warning: Some(format!(
2949 "The local-model resource policy could not be read ({error}); CAR used Everyday."
2950 )),
2951 }
2952 });
2953 let policy = policy_evidence.policy.clone();
2954 let effective_budget = policy.effective_budget(hw.total_ram_mb);
2955 let local_admission = resource_policy::scoped_local_admission_with_models_dir(
2956 &config.state_root,
2957 policy,
2958 hw.clone(),
2959 Some(config.models_dir.clone()),
2960 );
2961 let runtime_scope = scoped_inference_runtime(
2962 &config.state_root,
2963 effective_budget.configured_model_ceiling_mb,
2964 local_admission.clone(),
2965 );
2966 let router = ModelRouter::new(hw.clone());
2967 let unified_registry = UnifiedRegistry::new_with_state_root(
2968 config.state_root.clone(),
2969 config.models_dir.clone(),
2970 );
2971 let adaptive_router = AdaptiveRouter::with_default_config(hw);
2972 let mut tracker = OutcomeTracker::new();
2973 let profiles_path = config.state_models_dir().join("outcome_profiles.json");
2975 let outcome_profiles_loaded = match tracker.load_from_file(&profiles_path) {
2978 Ok(n) => {
2979 if n > 0 {
2980 tracing::info!(loaded = n, "loaded persisted model profiles");
2981 }
2982 true
2983 }
2984 Err(error) => error.kind() == std::io::ErrorKind::NotFound,
2985 };
2986 let mut benchmark_models_loaded = 0usize;
2987 for path in benchmark_priors_paths(&config.state_models_dir()) {
2988 match routing_ext::load_benchmark_priors(&path) {
2989 Ok(priors) if !priors.is_empty() => {
2990 benchmark_models_loaded += priors.len();
2991 routing_ext::apply_benchmark_priors(&mut tracker, &priors);
2992 tracing::info!(
2993 path = %path.display(),
2994 loaded = priors.len(),
2995 "loaded benchmark quality priors"
2996 );
2997 }
2998 Ok(_) => {}
2999 Err(error) => {
3000 tracing::warn!(path = %path.display(), %error, "failed to load benchmark priors");
3001 }
3002 }
3003 }
3004 if benchmark_models_loaded > 0 {
3005 tracing::info!(
3006 loaded = benchmark_models_loaded,
3007 "applied benchmark priors to cold-start routing"
3008 );
3009 }
3010 let outcome_tracker = Arc::new(RwLock::new(tracker));
3011
3012 let remote_backend = RemoteBackend::new();
3013 let model_management = model_management::ModelManagementStore::new(
3014 config.state_root.clone(),
3015 config.models_dir.clone(),
3016 );
3017
3018 let lane_defaults_path = config
3019 .state_root
3020 .join(crate::lane_defaults::LANE_DEFAULTS_FILE);
3021 let next_action_seq = crate::action_ledger::read_actions(
3022 &config
3023 .state_root
3024 .join(crate::action_ledger::ACTION_LEDGER_FILE),
3025 0,
3026 )
3027 .iter()
3028 .map(|a| a.seq)
3029 .max()
3030 .map(|m| m + 1)
3031 .unwrap_or(1);
3032 Self {
3033 config,
3034 unified: std::sync::RwLock::new(Arc::new(unified_registry)),
3035 registry_write: std::sync::Mutex::new(()),
3036 adaptive_router,
3037 outcome_tracker,
3038 last_outcome_flush: Arc::new(std::sync::Mutex::new(None)),
3039 outcome_profiles_loaded,
3040 ledger_io_lock: Arc::new(tokio::sync::Mutex::new(())),
3041 spend_limits: Arc::new(std::sync::RwLock::new(None)),
3042 lane_defaults_cache: Arc::new(std::sync::RwLock::new(crate::lane_defaults::load_from(
3043 &lane_defaults_path,
3044 ))),
3045 concierge_action_lock: Arc::new(tokio::sync::Mutex::new(())),
3046 external_pins: Default::default(),
3047 disk_pressured: Default::default(),
3048 failed_upgrades: Default::default(),
3049 concierge_action_seq: Arc::new(std::sync::atomic::AtomicU64::new(next_action_seq)),
3050 remote_backend,
3051 model_management,
3052 local_admission: local_admission.clone(),
3053 _runtime_scope: runtime_scope.clone(),
3054 resource_policy_generation: runtime_scope.resource_policy_generation.clone(),
3055 resource_policy_evidence: Arc::new(std::sync::RwLock::new(policy_evidence)),
3056 model_budget: runtime_scope.model_budget.clone(),
3057 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3058 text_backends: runtime_scope.text_backends.clone(),
3059 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3060 local_backends: runtime_scope.local_backends.clone(),
3061 registry,
3062 router,
3063 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3064 backend: runtime_scope.backend.clone(),
3065 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3066 embedding_backend: runtime_scope.embedding_backend.clone(),
3067 speech_runtime: runtime_scope.speech_runtime.clone(),
3068 speech_policy: SpeechPolicy {
3069 prefer_local: cfg!(all(
3070 target_os = "macos",
3071 target_arch = "aarch64",
3072 not(car_skip_mlx)
3073 )),
3074 allow_remote_fallback: true,
3075 preferred_local_stt: None,
3076 preferred_local_tts: None,
3077 preferred_remote_stt: None,
3078 preferred_remote_tts: None,
3079 },
3080 vllm_pool: runtime_scope.vllm_pool.clone(),
3081 }
3082 }
3083
3084 pub async fn init_key_pool(&self) {
3087 for schema in self.unified_registry().list() {
3089 if schema.is_remote() {
3090 self.remote_backend.register_model_keys(schema).await;
3091 }
3092 }
3093
3094 let stats_path = self.config.state_models_dir().join("key_pool_stats.json");
3096 if let Ok(n) = self.remote_backend.key_pool.load_stats(&stats_path).await {
3097 if n > 0 {
3098 tracing::info!(loaded = n, "loaded persisted key pool stats");
3099 }
3100 }
3101
3102 let total = self.remote_backend.key_pool.total_keys().await;
3103 if total > 0 {
3104 tracing::info!(keys = total, "key pool initialized");
3105 }
3106 }
3107
3108 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3111 async fn ensure_backend(
3112 &self,
3113 schema: &ModelSchema,
3114 reservation: &mut resource_policy::LocalLoadReservation,
3115 ) -> Result<(), InferenceError> {
3116 let read = self.backend.read().await;
3117 if read.contains_key(&schema.id) {
3118 return Ok(());
3119 }
3120 drop(read);
3121
3122 let mut write = self.backend.write().await;
3123 if write.contains_key(&schema.id) {
3124 return Ok(());
3125 }
3126
3127 let model_path = self.registry.ensure_model(&schema.name).await?;
3128 let mut measured = backend_cache::estimate_model_size(&model_path);
3129 reservation
3130 .reconcile_measured_weights(measured)
3131 .map_err(InferenceError::from)?;
3132 let device = self.config.device.unwrap_or_else(Device::auto);
3133 let backend = match CandleBackend::load(&model_path, device) {
3135 Ok(b) => b,
3136 Err(load_err) => {
3137 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3141 return Err(load_err);
3142 }
3143 tracing::warn!(
3144 model = %schema.id,
3145 error = %load_err,
3146 "candle backend load failed; purged corrupt cache files and re-pulling once"
3147 );
3148 let model_path = self.registry.ensure_model(&schema.name).await?;
3149 measured = backend_cache::estimate_model_size(&model_path);
3150 reservation
3151 .reconcile_measured_weights(measured)
3152 .map_err(InferenceError::from)?;
3153 CandleBackend::load(&model_path, device)?
3154 }
3155 };
3156 write.insert(schema.id.clone(), backend);
3157 reservation.publish_resident_weights(measured);
3158 Ok(())
3159 }
3160
3161 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3164 async fn ensure_embedding_backend(
3165 &self,
3166 reservation: &mut resource_policy::LocalLoadReservation,
3167 ) -> Result<(), InferenceError> {
3168 let read = self.embedding_backend.read().await;
3169 if read.is_some() {
3170 return Ok(());
3171 }
3172 drop(read);
3173
3174 let mut write = self.embedding_backend.write().await;
3175 if write.is_some() {
3176 return Ok(());
3177 }
3178
3179 let embedding_model = self
3180 .preferred_model_for_capability(ModelCapability::Embed)
3181 .unwrap_or(&self.config.embedding_model);
3182 let model_path = self.registry.ensure_model(embedding_model).await?;
3183 let mut measured = backend_cache::estimate_model_size(&model_path);
3184 reservation
3185 .reconcile_measured_weights(measured)
3186 .map_err(InferenceError::from)?;
3187 let device = self.config.device.unwrap_or_else(Device::auto);
3188 let backend = match EmbeddingBackend::load(&model_path, device) {
3190 Ok(b) => b,
3191 Err(load_err) => {
3192 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3193 return Err(load_err);
3194 }
3195 tracing::warn!(
3196 model = embedding_model,
3197 error = %load_err,
3198 "embedding backend load failed; purged corrupt cache files and re-pulling once"
3199 );
3200 let model_path = self.registry.ensure_model(embedding_model).await?;
3201 measured = backend_cache::estimate_model_size(&model_path);
3202 reservation
3203 .reconcile_measured_weights(measured)
3204 .map_err(InferenceError::from)?;
3205 EmbeddingBackend::load(&model_path, device)?
3206 }
3207 };
3208 *write = Some(backend);
3209 reservation.publish_resident_weights(measured);
3210 Ok(())
3211 }
3212
3213 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3216 async fn ensure_mlx_embedding_backend(&self) -> Result<String, InferenceError> {
3217 let embedding_model_name = self
3218 .preferred_model_for_capability(ModelCapability::Embed)
3219 .unwrap_or(&self.config.embedding_model)
3220 .to_string();
3221 let schema = self
3222 .resolve_schema(&embedding_model_name)
3223 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model_name.clone()))?
3224 .clone();
3225 Ok(schema.id)
3226 }
3227
3228 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3244 async fn load_backend_healing<T, F, RF, R>(
3245 schema_id: &str,
3246 model_dir: std::path::PathBuf,
3247 cache: &backend_cache::BackendCache<T>,
3248 size: u64,
3249 reservation: &mut resource_policy::LocalLoadReservation,
3250 loader: F,
3251 repull: RF,
3252 ) -> Result<
3253 (
3254 backend_cache::CachedBackend<T>,
3255 backend_cache::BackendRetention,
3256 ),
3257 InferenceError,
3258 >
3259 where
3260 T: Send + 'static,
3261 F: Fn(&Path) -> Result<T, InferenceError>,
3262 RF: FnOnce() -> R,
3263 R: std::future::Future<Output = Result<std::path::PathBuf, InferenceError>>,
3264 {
3265 match cache.get_or_load_admitted(schema_id, size, reservation, || loader(&model_dir)) {
3266 Ok(admitted) => Ok(admitted),
3267 Err(load_err) => {
3268 let purged = crate::download::purge_corrupt_cache_files(&model_dir);
3276 if purged == 0 {
3277 return Err(load_err);
3279 }
3280 tracing::warn!(
3281 model = schema_id,
3282 purged,
3283 error = %load_err,
3284 "backend load failed; purged corrupt cache files and re-pulling once"
3285 );
3286 let fresh_dir = repull().await?;
3287 let fresh_size = backend_cache::estimate_model_size(&fresh_dir);
3288 cache
3289 .get_or_load_admitted(schema_id, fresh_size, reservation, || loader(&fresh_dir))
3290 }
3291 }
3292 }
3293
3294 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3298 async fn ensure_text_backend(
3299 &self,
3300 schema: &ModelSchema,
3301 reservation: &mut resource_policy::LocalLoadReservation,
3302 ) -> Result<
3303 (
3304 backend_cache::CachedBackend<backend::SwiftLmBackend>,
3305 backend_cache::BackendRetention,
3306 ),
3307 InferenceError,
3308 > {
3309 if !Self::supports_native_mlx(schema) {
3310 return Err(InferenceError::InferenceFailed(format!(
3311 "native MLX backend does not support {} ({}) yet; use vLLM-MLX or add a family-specific MLX backend",
3312 schema.name, schema.family
3313 )));
3314 }
3315
3316 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3317 let size = backend_cache::estimate_model_size(&model_dir);
3318 if !reservation.authorizes_model(&schema.id) {
3319 return Err(InferenceError::InferenceFailed(format!(
3320 "local admission reservation does not authorize {}",
3321 schema.id
3322 )));
3323 }
3324 let role = if schema.has_capability(ModelCapability::Embed)
3334 && !schema.has_capability(ModelCapability::Generate)
3335 {
3336 backend::swift_lm::BackendRole::Embedding
3337 } else {
3338 backend::swift_lm::BackendRole::Text
3339 };
3340 let loader = move |dir: &Path| {
3341 let dir = dir.to_path_buf();
3342 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3343 backend::SwiftLmBackend::load_with_role(&dir, role)
3345 }))
3346 .map_err(|e| {
3347 InferenceError::InferenceFailed(format!(
3348 "MLX backend loading panicked (possible Metal/accelerate exception): {:?}",
3349 e
3350 ))
3351 })?
3352 };
3353 let registry = self.unified_registry();
3354 Self::load_backend_healing(
3355 &schema.id,
3356 model_dir,
3357 &self.text_backends,
3358 size,
3359 reservation,
3360 loader,
3361 || registry.redownload_local(&schema.id),
3362 )
3363 .await
3364 }
3365
3366 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3371 async fn ensure_local_backend(
3372 &self,
3373 schema: &ModelSchema,
3374 reservation: &mut resource_policy::LocalLoadReservation,
3375 ) -> Result<
3376 (
3377 backend_cache::CachedBackend<Box<dyn backend::local::LocalInferenceBackend>>,
3378 backend_cache::BackendRetention,
3379 ),
3380 InferenceError,
3381 > {
3382 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3383 let size = backend_cache::estimate_model_size(&model_dir);
3384 if !reservation.authorizes_model(&schema.id) {
3385 return Err(InferenceError::InferenceFailed(format!(
3386 "local admission reservation does not authorize {}",
3387 schema.id
3388 )));
3389 }
3390 let loader = |dir: &Path| {
3391 let dir = dir.to_path_buf();
3392 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3393 backend::SwiftLmBackend::load(&dir)
3403 .map(|b| Box::new(b) as Box<dyn backend::local::LocalInferenceBackend>)
3404 .map_err(|e| {
3405 InferenceError::InferenceFailed(format!(
3406 "mlx-swift-lm could not load model_type '{}' ({}): {e}",
3407 backend::local::describe_model_type(&dir),
3408 dir.display()
3409 ))
3410 })
3411 }))
3412 .map_err(|e| {
3413 InferenceError::InferenceFailed(format!(
3414 "local backend loading panicked (possible Metal/accelerate exception): {:?}",
3415 e
3416 ))
3417 })?
3418 };
3419 let registry = self.unified_registry();
3420 Self::load_backend_healing(
3421 &schema.id,
3422 model_dir,
3423 &self.local_backends,
3424 size,
3425 reservation,
3426 loader,
3427 || registry.redownload_local(&schema.id),
3428 )
3429 .await
3430 }
3431
3432 pub async fn reset_local_kv_cache(&self, model_id: &str) {
3443 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3447 {
3448 let Some(schema) = self.unified_registry().get(model_id).cloned() else {
3449 return;
3450 };
3451 if !matches!(
3455 schema.source,
3456 ModelSource::Mlx { .. } | ModelSource::Local { .. }
3457 ) {
3458 return;
3459 }
3460 if !self.local_backends.contains(&schema.id) {
3463 return;
3464 }
3465 let Ok(mut reservation) = self.reserve_local_request(&schema, 0) else {
3466 return;
3467 };
3468 if let Ok((handle, _retention)) =
3469 self.ensure_local_backend(&schema, &mut reservation).await
3470 {
3471 if let Ok(mut guard) = handle.lock() {
3472 guard.clear_kv_cache();
3473 }
3474 }
3475 }
3476 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3477 let _ = model_id;
3478 }
3479
3480 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3487 pub async fn warm_up<S: AsRef<str>>(
3488 &self,
3489 schema_ids: &[S],
3490 ) -> Vec<Result<(), InferenceError>> {
3491 let mut results = Vec::with_capacity(schema_ids.len());
3492 for id in schema_ids {
3493 let id = id.as_ref();
3494 let outcome: Result<(), InferenceError> = async {
3495 let schema = self.unified_registry().get(id).cloned().ok_or_else(|| {
3496 InferenceError::InferenceFailed(format!("warm_up: unknown schema id {id}"))
3497 })?;
3498 let mut reservation = self.reserve_local_request(&schema, 0)?;
3500 match schema.capabilities.first().copied() {
3501 Some(ModelCapability::ImageGeneration) => {
3502 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3507 }
3508 Some(ModelCapability::VideoGeneration) => {
3509 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3514 }
3515 Some(ModelCapability::TextToSpeech) => {
3516 }
3520 _ => {
3521 let _ = self.ensure_text_backend(&schema, &mut reservation).await?;
3522 }
3523 }
3524 Ok(())
3525 }
3526 .await;
3527 results.push(outcome);
3528 }
3529 results
3530 }
3531
3532 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3534 pub async fn warm_up<S: AsRef<str>>(
3535 &self,
3536 _schema_ids: &[S],
3537 ) -> Vec<Result<(), InferenceError>> {
3538 Vec::new()
3539 }
3540
3541 async fn vllm_live_schema(
3547 &self,
3548 schema: ModelSchema,
3549 reservation: Option<resource_policy::LocalLoadReservation>,
3550 context_tokens: usize,
3551 ) -> Result<(ModelSchema, Option<resource_policy::LocalLoadReservation>), InferenceError> {
3552 match &schema.source {
3553 ModelSource::ManagedVllmMlx { .. } => {}
3554 ModelSource::VllmMlx { .. } => return Ok((schema, None)),
3555 _ => return Ok((schema, None)),
3556 }
3557 let _dispatch = self.vllm_pool.acquire_dispatch(&schema.id).await;
3562 self.vllm_pool
3563 .wait_for_teardown(&schema.id, std::time::Duration::from_secs(5))
3564 .await
3565 .map_err(InferenceError::InferenceFailed)?;
3566 self.vllm_pool
3571 .reap_dead(&schema.id)
3572 .await
3573 .map_err(InferenceError::InferenceFailed)?;
3574 let mut reservation = match reservation {
3575 Some(reservation) if reservation.model_id() == schema.id => reservation,
3576 _ => self.reserve_local_request(&schema, context_tokens)?,
3577 };
3578 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3579 let model_name = model_dir.display().to_string();
3580 let measured = backend_cache::estimate_model_size(&model_dir)
3581 .max(reservation.reconciled_weights_bytes());
3582 reservation
3583 .reconcile_measured_weights(measured)
3584 .map_err(InferenceError::from)?;
3585 let endpoint = self
3587 .vllm_pool
3588 .ensure(&schema.id, &model_name, &reservation, &schema.family)
3591 .await
3592 .map_err(InferenceError::InferenceFailed)?;
3593 let allocation_id = resource_policy::vllm_process_allocation_id(&schema.id);
3594 reservation.publish_resident_weights_as(&allocation_id, measured);
3595 let mut schema = schema;
3596 schema.source = ModelSource::VllmMlx {
3597 endpoint,
3598 model_name,
3599 };
3600 Ok((schema, Some(reservation)))
3601 }
3602
3603 pub async fn evict_idle_vllm_servers(&self) -> usize {
3606 self.vllm_pool.evict_idle().await
3607 }
3608
3609 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3620 pub fn evict_idle_backends(&self) -> (usize, u64) {
3621 let mut entries = 0usize;
3622 let mut bytes = 0u64;
3623 for (n, b) in [
3624 self.text_backends.evict_idle(),
3625 self.local_backends.evict_idle(),
3626 ] {
3627 entries += n;
3628 bytes = bytes.saturating_add(b);
3629 }
3630 (entries, bytes)
3631 }
3632
3633 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3635 pub fn evict_idle_backends(&self) -> (usize, u64) {
3636 (0, 0)
3637 }
3638
3639 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3640 fn supports_native_mlx(schema: &ModelSchema) -> bool {
3641 matches!(schema.family.as_str(), "qwen3" | "qwen2.5-vl" | "qwen2-vl")
3642 }
3643
3644 fn resolve_schema(&self, name: &str) -> Option<ModelSchema> {
3648 let registry = self.unified_registry();
3649 registry
3650 .get(name)
3651 .or_else(|| registry.find_by_name(name))
3652 .cloned()
3653 }
3654
3655 fn resolve_schema_name_first(&self, name: &str) -> Option<ModelSchema> {
3658 let registry = self.unified_registry();
3659 registry
3660 .find_by_name(name)
3661 .or_else(|| registry.get(name))
3662 .cloned()
3663 }
3664
3665 pub fn unified_registry(&self) -> Arc<UnifiedRegistry> {
3670 let me = self as *const Self as usize;
3671 PINNED_REGISTRY
3672 .try_with(|(engine, pinned)| (*engine == me).then(|| pinned.clone()))
3673 .ok()
3674 .flatten()
3675 .unwrap_or_else(|| self.current_registry())
3676 }
3677
3678 fn current_registry(&self) -> Arc<UnifiedRegistry> {
3680 self.unified
3681 .read()
3682 .unwrap_or_else(std::sync::PoisonError::into_inner)
3683 .clone()
3684 }
3685
3686 async fn with_pinned_registry<F: std::future::Future>(&self, operation: F) -> F::Output {
3695 let me = self as *const Self as usize;
3696 let pinned_here = PINNED_REGISTRY
3697 .try_with(|(engine, _)| *engine == me)
3698 .unwrap_or(false);
3699 if pinned_here {
3700 operation.await
3701 } else {
3702 PINNED_REGISTRY
3703 .scope((me, self.current_registry()), operation)
3704 .await
3705 }
3706 }
3707
3708 pub fn unified_registry_mut(&mut self) -> &mut UnifiedRegistry {
3712 Arc::make_mut(
3713 self.unified
3714 .get_mut()
3715 .unwrap_or_else(std::sync::PoisonError::into_inner),
3716 )
3717 }
3718
3719 fn update_registry<R>(&self, edit: impl FnOnce(&mut UnifiedRegistry) -> R) -> R {
3725 let _writer = self
3726 .registry_write
3727 .lock()
3728 .unwrap_or_else(std::sync::PoisonError::into_inner);
3729 let mut next = (*self.current_registry()).clone();
3730 let out = edit(&mut next);
3731 *self
3732 .unified
3733 .write()
3734 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(next);
3735 out
3736 }
3737
3738 pub fn swap_signed_catalog(&self, catalog: catalog::VerifiedCatalog) -> usize {
3747 let origin = Arc::new(catalog.envelope());
3748 let revoked: std::collections::HashSet<String> =
3749 catalog.revoked().iter().cloned().collect();
3750 self.update_registry(|registry| {
3751 registry.refresh_availability();
3754 let mut keep: std::collections::HashSet<String> =
3756 self.protected_models(registry, true).into_keys().collect();
3757 keep.extend(
3758 registry
3759 .list()
3760 .into_iter()
3761 .filter(|m| {
3762 (m.is_local() && m.downloads_weights() && m.weights_ready)
3765 || matches!(self.model_management.load_receipt(&m.id), Ok(Some(_)))
3766 })
3767 .map(|m| m.id.clone()),
3768 );
3769 let loaded = registry.replace_signed_catalog(
3770 catalog.into_models(),
3771 Some(origin),
3772 &keep,
3773 &revoked,
3774 );
3775 registry.refresh_availability();
3776 self.persist_retained(registry);
3777 loaded
3778 })
3779 }
3780
3781 fn persist_retained(&self, registry: &UnifiedRegistry) {
3784 let path = catalog::retained_path(&self.config.state_root);
3785 if let Err(error) = catalog::save_retained(&path, ®istry.retained_rows()) {
3786 tracing::warn!(path = %path.display(), %error, "could not record retained catalog rows");
3787 }
3788 }
3789
3790 fn keep_pulled_row_if_withdrawn(&self, name: &str) {
3794 let Some(schema) = self.resolve_schema_name_first(name) else {
3795 return;
3796 };
3797 if self.current_registry().get(&schema.id).is_some() {
3798 return;
3799 }
3800 let Some(origin) = self.unified_registry().signed_origin_of(&schema.id) else {
3801 return;
3802 };
3803 self.update_registry(|registry| {
3804 registry.restore_retained(vec![(schema, origin)]);
3805 registry.refresh_availability();
3806 self.persist_retained(registry);
3807 });
3808 }
3809
3810 fn catalog_registry_snapshot(&self) -> UnifiedRegistry {
3811 self.catalog_registry_snapshot_with(false)
3812 }
3813
3814 fn catalog_registry_snapshot_with(&self, authoritative_credentials: bool) -> UnifiedRegistry {
3823 let mut registry = (*self.unified_registry()).clone();
3824 registry.prune_missing_on_disk_models();
3825 if authoritative_credentials {
3830 registry.refresh_availability_for_explicit_status();
3831 } else {
3832 registry.refresh_availability();
3833 }
3834 self.filter_disabled_local_models(&mut registry);
3835 registry
3836 }
3837
3838 fn filter_disabled_local_models(&self, registry: &mut UnifiedRegistry) {
3839 let disabled = registry
3840 .all()
3841 .filter(|schema| {
3842 schema.downloads_weights()
3843 && !self
3844 .model_management
3845 .car_enabled(&schema.id)
3846 .unwrap_or(false)
3847 })
3848 .map(|schema| schema.id.clone())
3849 .collect::<Vec<_>>();
3850 for model_id in disabled {
3851 registry.unregister(&model_id);
3852 }
3853 }
3854
3855 async fn routing_registry_snapshot_with_credential_failure(
3856 &self,
3857 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
3858 let parslee = car_auth::resolve_credential(car_auth::CredentialReadMode::Use).await;
3864 let (parslee_api_base, parslee_signed_out, credential_failure) = match &parslee {
3865 Ok(Some(credential)) => (Some(credential.api_base.as_str()), false, None),
3866 Ok(None) => (None, true, Some(parslee_signed_out_route_failure())),
3867 Err(error) => {
3868 let source_error = error.to_string();
3869 (
3870 None,
3871 false,
3872 Some(RouteCredentialFailure {
3873 summary: format!(
3874 "Parslee {AUTH_STORE_UNREADABLE_MARKER} — unlock or grant access to the credential store, then retry"
3875 ),
3876 source_error,
3877 }),
3878 )
3879 }
3880 };
3881 let mut registry = (*self.unified_registry()).clone();
3882 registry.prune_missing_on_disk_models();
3883 registry.refresh_routing_availability(parslee_api_base, parslee_signed_out);
3884 self.filter_disabled_local_models(&mut registry);
3885 (registry, credential_failure)
3886 }
3887
3888 async fn work_context_routing_snapshot(
3889 &self,
3890 context: &car_auth::context::CredentialContext,
3891 requested_model: Option<&str>,
3892 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
3893 let credential = if requested_model.is_some() {
3896 None
3897 } else {
3898 Some(car_auth::context::resolve_context_credential(context).await)
3899 };
3900 let available = context.organization_id.is_some()
3901 && credential.as_ref().is_none_or(|result| result.is_ok());
3902 let failure =
3903 credential
3904 .and_then(|result| result.err())
3905 .map(|error| RouteCredentialFailure {
3906 summary: format!("Work-context credentials unavailable: {error}"),
3907 source_error: error.to_string(),
3908 });
3909 let mut registry = (*self.unified_registry()).clone();
3910 registry.prune_missing_on_disk_models();
3911 registry.refresh_work_context_availability(&context.api_base, available);
3912 self.filter_disabled_local_models(&mut registry);
3913 (registry, failure)
3914 }
3915
3916 async fn routing_registry_snapshot(&self) -> UnifiedRegistry {
3917 self.routing_registry_snapshot_with_credential_failure()
3918 .await
3919 .0
3920 }
3921
3922 async fn request_routing_registry_snapshot(
3927 &self,
3928 requested_model: Option<&str>,
3929 ) -> UnifiedRegistry {
3930 if requested_model.is_some() {
3931 self.catalog_registry_snapshot()
3932 } else {
3933 self.routing_registry_snapshot().await
3934 }
3935 }
3936
3937 pub async fn route_adaptive(&self, prompt: &str) -> AdaptiveRoutingDecision {
3939 self.route_adaptive_with_intent(prompt, None).await
3940 }
3941
3942 pub async fn route_adaptive_with_intent(
3952 &self,
3953 prompt: &str,
3954 intent: Option<crate::intent::IntentHint>,
3955 ) -> AdaptiveRoutingDecision {
3956 if let Some(model) = self.preferred_model_for_capability(ModelCapability::Generate) {
3957 let exclude_set = self
3958 .adaptive_router
3959 .build_exclude_set(intent.as_ref(), &self.unified_registry());
3960 let overridden_is_excluded =
3961 Self::model_is_excluded(&exclude_set, &self.unified_registry(), model);
3962 if !overridden_is_excluded {
3963 let ctx_len = self
3969 .resolve_schema(model)
3970 .map(|s| s.context_length)
3971 .unwrap_or(0);
3972 return AdaptiveRoutingDecision {
3973 model_id: model.to_string(),
3974 model_name: model.to_string(),
3975 task: InferenceTask::Generate,
3976 complexity: TaskComplexity::assess(prompt),
3977 reason: "preferred generation model override".into(),
3978 strategy: RoutingStrategy::Explicit,
3979 predicted_quality: 0.5,
3980 fallbacks: vec![],
3981 context_length: ctx_len,
3982 needs_compaction: false,
3983 candidates: vec![],
3984 };
3985 }
3986 }
3987 let routing_registry = self.routing_registry_snapshot().await;
3988 let tracker = self.outcome_tracker.read().await;
3989 match intent {
3990 Some(hint) => self
3991 .adaptive_router
3992 .route_with(crate::adaptive_router::RouteRequest {
3993 intent: Some(&hint),
3994 ..crate::adaptive_router::RouteRequest::new(prompt, &routing_registry, &tracker)
3995 }),
3996 None => self
3997 .adaptive_router
3998 .route(prompt, &routing_registry, &tracker),
3999 }
4000 }
4001
4002 pub fn route(&self, prompt: &str) -> RoutingDecision {
4004 self.router.route_generate(prompt, &self.registry)
4005 }
4006
4007 pub fn estimated_tokens(
4018 &self,
4019 req: &GenerateRequest,
4020 model_id: Option<&str>,
4021 ) -> (usize, usize, bool) {
4022 let prompt_tokens = remote::estimate_tokens(&req.prompt);
4023 let context_tokens = req
4024 .context
4025 .as_ref()
4026 .map(|c| remote::estimate_tokens(c))
4027 .unwrap_or(0);
4028 let tools_tokens = req
4029 .tools
4030 .as_ref()
4031 .map(|t| remote::estimate_tokens(&serde_json::to_string(t).unwrap_or_default()))
4032 .unwrap_or(0);
4033 let media_tokens = media_tokens::request_media_and_history_tokens(
4034 req.images.as_deref(),
4035 req.messages.as_deref(),
4036 );
4037 let total_input = prompt_tokens + context_tokens + tools_tokens + media_tokens;
4038
4039 let context_window = match model_id {
4048 Some(id) => {
4049 let routing_registry = self.catalog_registry_snapshot();
4050 routing_registry
4051 .get(id)
4052 .or_else(|| routing_registry.find_by_name(id))
4053 .map(|s| s.context_length)
4054 .unwrap_or(0)
4055 }
4056 None => 0,
4057 };
4058
4059 let fits = context_window == 0 || (total_input + req.params.max_tokens) <= context_window;
4060 (total_input, context_window, fits)
4061 }
4062
4063 fn routing_cache_estimates(req: &GenerateRequest, estimated_input: usize) -> (usize, usize) {
4069 let read = req
4070 .params
4071 .estimated_cache_read_input_tokens
4072 .min(estimated_input);
4073 let write = req
4074 .params
4075 .estimated_cache_write_input_tokens
4076 .min(estimated_input.saturating_sub(read));
4077 (read, write)
4078 }
4079
4080 pub fn model_context_window(&self, model_id: &str) -> usize {
4086 let routing_registry = self.catalog_registry_snapshot();
4087 routing_registry
4088 .get(model_id)
4089 .or_else(|| routing_registry.find_by_name(model_id))
4090 .map(|s| s.context_length)
4091 .unwrap_or(0)
4092 }
4093
4094 pub async fn generate_tracked(
4109 &self,
4110 req: GenerateRequest,
4111 ) -> Result<InferenceResult, InferenceError> {
4112 let mut ignore_retry = |_: InferenceRetryProgress| {};
4113 self.generate_tracked_with_retry_observer(req, &mut ignore_retry)
4114 .await
4115 }
4116
4117 pub async fn generate_tracked_with_retry_observer(
4121 &self,
4122 req: GenerateRequest,
4123 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4124 ) -> Result<InferenceResult, InferenceError> {
4125 let mut ignore_exclusion = |_: RouterExclusion| {};
4126 self.generate_tracked_with_observers(req, retry_observer, &mut ignore_exclusion)
4127 .await
4128 }
4129
4130 pub async fn generate_tracked_with_observers(
4134 &self,
4135 req: GenerateRequest,
4136 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4137 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4138 ) -> Result<InferenceResult, InferenceError> {
4139 self.with_pinned_registry(self.generate_tracked_with_observers_unpinned(
4141 req,
4142 retry_observer,
4143 exclusion_observer,
4144 ))
4145 .await
4146 }
4147
4148 async fn generate_tracked_with_observers_unpinned(
4149 &self,
4150 req: GenerateRequest,
4151 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4152 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4153 ) -> Result<InferenceResult, InferenceError> {
4154 crate::offload::ensure_not_controlled_terminated()?;
4155 let catalog_snapshot = self
4156 .catalog_snapshot()
4157 .map_err(InferenceError::InferenceFailed)?;
4158 let recover = matches!(req.params.thinking, ThinkingMode::Auto);
4159 let mut result = self
4160 .generate_tracked_inner(
4161 req.clone(),
4162 &catalog_snapshot,
4163 retry_observer,
4164 exclusion_observer,
4165 )
4166 .await?;
4167
4168 let action = classify_empty_pass(
4169 recover,
4170 result.stop_reason.as_deref(),
4171 &result.text,
4172 result.tool_calls.is_empty(),
4173 );
4174 let hit_decode_ceiling = action == EmptyPassAction::FailDecodeCeiling;
4175
4176 if action == EmptyPassAction::RetryWithoutThinking {
4177 result.stop_reason = Some("thinking_truncated".to_string());
4178 let mut retry = req;
4179 retry.params.thinking = ThinkingMode::Off;
4180 crate::offload::ensure_not_controlled_terminated()?;
4181 match self
4182 .generate_tracked_inner(
4183 retry,
4184 &catalog_snapshot,
4185 retry_observer,
4186 exclusion_observer,
4187 )
4188 .await
4189 {
4190 Ok(mut recovered) => {
4191 if !recovered.text.trim().is_empty() || !recovered.tool_calls.is_empty() {
4192 recovered.stop_reason = Some("thinking_recovered".to_string());
4193 return Ok(recovered);
4194 }
4195 }
4196 Err(error @ InferenceError::ControlledTermination) => return Err(error),
4197 Err(_) => {}
4198 }
4199 }
4200
4201 if hit_decode_ceiling && result.text.trim().is_empty() && result.tool_calls.is_empty() {
4207 return Err(InferenceError::InferenceFailed(format!(
4208 "local generation hit its {}s wall-clock ceiling before producing any output. \
4209 The `local prefill starting` / `local decode in progress` log lines show where \
4210 the time went — a large prompt can spend most of it on prefill. Try a smaller \
4211 model, shorten the prompt, check for another process contending for the GPU, \
4212 or raise the ceiling with CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables it).",
4213 local_decode_timeout().map_or(0, |t| t.as_secs())
4214 )));
4215 }
4216
4217 Ok(result)
4218 }
4219
4220 #[instrument(
4235 name = "inference.generate",
4236 skip_all,
4237 fields(
4238 model = tracing::field::Empty,
4239 max_tokens = req.params.max_tokens,
4240 prompt_tokens = tracing::field::Empty,
4241 completion_tokens = tracing::field::Empty,
4242 latency_ms = tracing::field::Empty,
4243 ttft_ms = tracing::field::Empty,
4247 gen_ai.operation.name = "chat",
4248 gen_ai.request.model = tracing::field::Empty,
4249 gen_ai.response.model = tracing::field::Empty,
4250 gen_ai.request.max_tokens = req.params.max_tokens,
4251 gen_ai.request.temperature = req.params.temperature,
4252 gen_ai.usage.input_tokens = tracing::field::Empty,
4253 gen_ai.usage.output_tokens = tracing::field::Empty,
4254 )
4255 )]
4256 async fn generate_tracked_inner(
4257 &self,
4258 mut req: GenerateRequest,
4259 catalog_snapshot: &CatalogSnapshot,
4260 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4261 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4262 ) -> Result<InferenceResult, InferenceError> {
4263 self.refuse_system_one_generate(&req)?;
4264 crate::offload::ensure_not_controlled_terminated()?;
4265 validate_expected_catalog_revision(&req, catalog_snapshot)?;
4266 if let Some(requested) = req.model.as_deref() {
4270 tracing::Span::current().record("gen_ai.request.model", requested);
4271 }
4272 let requested_model_id = exact_pinned_model_id(&req).map(str::to_string);
4273 if let Some(model_id) = requested_model_id.as_ref() {
4274 req.model = Some(model_id.clone());
4275 req.params.strict_model = true;
4276 }
4277 let start = Instant::now();
4278 let has_requested_route = req.model.is_some();
4279 let (routing_registry, initial_route_credential_failure) =
4280 if let Some(context) = &req.work_context {
4281 self.work_context_routing_snapshot(context, req.model.as_deref())
4282 .await
4283 } else if has_requested_route {
4284 (self.catalog_registry_snapshot(), None)
4285 } else {
4286 self.routing_registry_snapshot_with_credential_failure()
4287 .await
4288 };
4289 if let Some(requested) = requested_model_id.as_deref() {
4290 if routing_registry.get(requested).is_none() {
4291 return Err(InferenceError::ModelNotFound(requested.to_string()));
4292 }
4293 } else if let Some(requested) = req.model.as_deref() {
4294 if routing_registry
4295 .get(requested)
4296 .or_else(|| routing_registry.find_by_name(requested))
4297 .is_none()
4298 {
4299 return Err(InferenceError::ModelNotFound(requested.to_string()));
4300 }
4301 }
4302
4303 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
4305 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
4313 let (estimated_cache_read, estimated_cache_write) =
4314 Self::routing_cache_estimates(&req, estimated_input);
4315 let tracker_read = self.outcome_tracker.read().await;
4316 let has_tools = Self::request_has_tools(&req);
4317 let has_vision = Self::request_needs_vision(&req);
4318 let preferred_model = self
4319 .preferred_model_for_capability(ModelCapability::Generate)
4320 .map(str::to_string);
4321 let exclude_set = self
4322 .adaptive_router
4323 .build_exclude_set(req.intent.as_ref(), &routing_registry);
4324 let unpinned_override = self
4325 .lane_pin_for(&req, &routing_registry)
4326 .or(preferred_model)
4327 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model));
4328 let decision = match req.model.clone().or(unpinned_override) {
4329 Some(m) => {
4330 let ctx_len = routing_registry
4331 .get(&m)
4332 .or_else(|| routing_registry.find_by_name(&m))
4333 .map(|s| s.context_length)
4334 .unwrap_or(0);
4335 AdaptiveRoutingDecision {
4336 model_id: m.clone(),
4337 model_name: m.clone(),
4338 task: InferenceTask::Generate,
4339 complexity: TaskComplexity::assess(&req.prompt),
4340 reason: "explicit model".into(),
4341 strategy: RoutingStrategy::Explicit,
4342 predicted_quality: 0.5,
4343 fallbacks: vec![],
4344 context_length: ctx_len,
4345 needs_compaction: ctx_len > 0 && estimated_footprint > ctx_len,
4346 candidates: vec![],
4347 }
4348 }
4349 None => self.adaptive_router.route_with_exclusion_observer(
4350 crate::adaptive_router::RouteRequest {
4351 estimated_total_tokens: estimated_footprint,
4352 estimated_input_tokens: estimated_input,
4353 estimated_output_tokens: req.params.max_tokens,
4354 estimated_cache_read_tokens: estimated_cache_read,
4355 estimated_cache_write_tokens: estimated_cache_write,
4356 has_tools,
4357 has_vision,
4358 workload: req.params.workload,
4359 intent: req.intent.as_ref(),
4360 ..crate::adaptive_router::RouteRequest::new(
4361 &req.prompt,
4362 &routing_registry,
4363 &tracker_read,
4364 )
4365 },
4366 exclusion_observer,
4367 ),
4368 };
4369 drop(tracker_read);
4370
4371 if decision.model_id.is_empty() {
4372 let excluded_models = req
4373 .intent
4374 .as_ref()
4375 .map(|hint| hint.exclude_models.join(", "))
4376 .unwrap_or_default();
4377 return Err(InferenceError::NoEligibleModel { excluded_models });
4378 }
4379
4380 if decision.needs_compaction {
4381 tracing::info!(
4382 model = %decision.model_name,
4383 prompt_tokens = estimated_input,
4384 context_window = decision.context_length,
4385 "prompt exceeds model context window — compaction or truncation needed"
4386 );
4387 }
4388
4389 debug!(
4394 model = %decision.model_name,
4395 strategy = ?decision.strategy,
4396 reason = %decision.reason,
4397 "adaptive-routed generate request"
4398 );
4399
4400 let mut req = req;
4403
4404 if let Some(schema) = routing_registry
4410 .get(&decision.model_id)
4411 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4412 {
4413 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
4414 }
4415
4416 let is_code_intent = is_explicit_code_intent(req.intent.as_ref());
4425 let is_complex = matches!(decision.complexity, TaskComplexity::Complex);
4426 if req.params.budget_tokens == 0 && (is_code_intent || is_complex) {
4427 let supports_thinking = routing_registry
4431 .get(&decision.model_id)
4432 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4433 .map(|s| {
4434 s.supported_params
4435 .contains(&schema::GenerateParam::ExtendedThinking)
4436 })
4437 .unwrap_or(false);
4438 if let Some(budget) =
4439 auto_thinking_budget(is_code_intent, is_complex, supports_thinking)
4440 {
4441 req.params.budget_tokens = budget;
4442 tracing::info!(
4443 model = %decision.model_name,
4444 budget,
4445 code_intent = is_code_intent,
4446 "auto-enabled extended thinking"
4447 );
4448 }
4449 }
4450
4451 let mut models_to_try = vec![decision.model_id.clone()];
4453 models_to_try.extend(decision.fallbacks.iter().cloned());
4454
4455 let chain_has_local = models_to_try.iter().any(|m| {
4470 routing_registry
4471 .get(m)
4472 .or_else(|| routing_registry.find_by_name(m))
4473 .map(|s| s.is_local())
4474 .unwrap_or(false)
4475 });
4476 let mut local_last_resort_id = None;
4477 if should_append_local_last_resort(chain_has_local, req.params.strict_model) {
4478 if let Some(local) = self.first_installed_local_model(has_tools) {
4482 tracing::info!(
4483 local_model = %local,
4484 needs_tools = has_tools,
4485 "appended on-device model as last-resort fallback (chain was remote-only)"
4486 );
4487 models_to_try.push(local.clone());
4488 local_last_resort_id = Some(local);
4489 }
4490 } else if !chain_has_local && req.params.strict_model {
4491 tracing::info!(
4492 model = %decision.model_id,
4493 "strict_model set — not degrading to on-device; a remote failure will surface as an error"
4494 );
4495 }
4496
4497 let mut last_error = None;
4498 let mut auth_dead_lane: Option<String> = None;
4504 let mut route_credential_failure = if chain_includes_parslee_route(
4514 |m| {
4515 routing_registry
4516 .get(m)
4517 .or_else(|| routing_registry.find_by_name(m))
4518 },
4519 &models_to_try,
4520 ) {
4521 initial_route_credential_failure
4522 } else {
4523 None
4524 };
4525 let mut fallback_hops: Vec<FallbackFrom> = Vec::new();
4529
4530 let mut candidate_queue: std::collections::VecDeque<String> =
4537 models_to_try.iter().cloned().collect();
4538 let mut is_primary_attempt = true;
4539 while let Some(candidate_owned) = candidate_queue.pop_front() {
4540 crate::offload::ensure_not_controlled_terminated()?;
4541 let was_primary = is_primary_attempt;
4542 is_primary_attempt = false;
4543 let candidate_id = &candidate_owned;
4544 #[allow(unused_mut)]
4547 let mut schema = routing_registry
4548 .get(candidate_id)
4549 .or_else(|| routing_registry.find_by_name(candidate_id))
4550 .cloned();
4551
4552 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4560 if requested_model_id.is_none() {
4561 if let Some(ref s) = schema {
4562 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
4563 tracing::info!(
4564 from = %s.id, to = %mlx_equiv.id,
4565 "redirecting GGUF model to MLX equivalent on Apple Silicon"
4566 );
4567 schema = Some(mlx_equiv.clone());
4568 }
4569 }
4570 }
4571
4572 let candidate_name = schema
4577 .as_ref()
4578 .map(|s| s.name.clone())
4579 .unwrap_or_else(|| candidate_id.clone());
4580
4581 if has_tools
4592 && schema
4593 .as_ref()
4594 .map(|s| !s.has_capability(ModelCapability::ToolUse))
4595 .unwrap_or(false)
4596 {
4597 let backend = schema
4598 .as_ref()
4599 .map(|s| if s.is_local() { "local" } else { "remote" })
4600 .unwrap_or("unknown");
4601 tracing::warn!(
4602 model = %candidate_id,
4603 backend,
4604 "tools requested but resolved model lacks ToolUse capability — skipping candidate"
4605 );
4606 let unsupported = InferenceError::UnsupportedMode {
4607 mode: "tool_use",
4608 backend,
4609 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
4610 };
4611 record_fallback_from(&mut fallback_hops, &candidate_name, &unsupported);
4617 last_error = Some(unsupported);
4618 continue;
4619 }
4620
4621 let resolved_id = schema
4632 .as_ref()
4633 .map(|s| s.id.clone())
4634 .unwrap_or_else(|| candidate_id.clone());
4635 let reported_model_used = if requested_model_id.is_some() {
4636 resolved_id.clone()
4637 } else {
4638 candidate_name.clone()
4639 };
4640 validate_expected_catalog_row(&req, catalog_snapshot, &resolved_id)?;
4641
4642 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4648 if requested_model_id.is_some()
4649 && schema
4650 .as_ref()
4651 .is_some_and(|schema| matches!(schema.source, ModelSource::Local { .. }))
4652 {
4653 let pinned = InferenceError::InferenceFailed(format!(
4654 "exact model_id `{resolved_id}` names a GGUF/Candle row, but Candle is disabled on Apple Silicon; MLX-equivalent substitution is disabled for exact pins"
4655 ));
4656 record_fallback_from(&mut fallback_hops, &candidate_name, &pinned);
4657 last_error = Some(pinned);
4658 continue;
4659 }
4660
4661 let is_remote = schema
4662 .as_ref()
4663 .map(|s| s.is_remote() || s.is_vllm_mlx())
4664 .unwrap_or(false);
4665 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
4666 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
4667
4668 let attempt_trace = {
4673 let mut tracker = self.outcome_tracker.write().await;
4674 tracker.record_start(&resolved_id, decision.task, &decision.reason)
4675 };
4676
4677 if is_delegated {
4683 let runner = match runner::current_inference_runner() {
4684 Some(r) => r,
4685 None => {
4686 let msg = "model declares ModelSource::Delegated but no inference runner is registered";
4687 self.outcome_tracker.write().await.record_not_model_fault(
4690 &attempt_trace,
4691 msg,
4692 outcome::UnattributedReason::Capability,
4693 );
4694 let unregistered = InferenceError::InferenceFailed(msg.into());
4695 record_fallback_from(&mut fallback_hops, &candidate_name, &unregistered);
4696 last_error = Some(unregistered);
4697 continue;
4698 }
4699 };
4700 let (tx, mut rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
4701 let emitter = runner::EventEmitter::new(tx);
4702 let runner_req = req.clone();
4703 let runner_handle = AbortOnDropTask(Some(tokio::spawn(async move {
4704 runner.run(runner_req, emitter).await
4705 })));
4706 let mut accumulator = stream::StreamAccumulator::default();
4707 while let Some(evt) = rx.recv().await {
4708 accumulator.push(&evt);
4709 }
4710 let (acc_text, acc_tool_calls, acc_usage, acc_stop_reason) =
4739 accumulator.finish_with_usage();
4740 match runner_handle.join().await {
4741 Ok(Ok(runner_result)) => {
4742 let elapsed = start.elapsed().as_millis() as u64;
4743 let acc_text = if acc_text.trim().is_empty() {
4744 runner_result.text
4745 } else {
4746 acc_text
4747 };
4748 let acc_tool_calls = if acc_tool_calls.is_empty() {
4749 runner_result.tool_calls
4750 } else {
4751 acc_tool_calls
4752 };
4753 let est_out = acc_text.split_whitespace().count();
4760 {
4761 let mut tracker = self.outcome_tracker.write().await;
4762 tracker.record_complete(
4763 &attempt_trace,
4764 elapsed,
4765 estimated_input,
4766 est_out,
4767 );
4768 }
4769 let local_last_resort = report_local_last_resort_served(
4770 local_last_resort_id.as_deref(),
4771 candidate_id,
4772 &resolved_id,
4773 );
4774 record_generation_span(candidate_name.as_str(), elapsed, None, None);
4781 return Ok(InferenceResult {
4782 text: acc_text,
4783 tool_calls: acc_tool_calls,
4784 bounding_boxes: vec![],
4785 trace_id: attempt_trace,
4786 model_used: reported_model_used,
4787 model_identity: bound_model_identity(
4788 catalog_snapshot,
4789 requested_model_id.as_deref(),
4790 &resolved_id,
4791 )?,
4792 latency_ms: elapsed,
4793 time_to_first_token_ms: None,
4794 usage: acc_usage,
4797 provider_output_items: vec![],
4798 thinking: vec![],
4801 stop_reason: acc_stop_reason,
4802 auth_fallback_from: auth_dead_lane.clone(),
4803 local_last_resort,
4804 fallback_from: fallback_hops.clone(),
4805 });
4806 }
4807 Ok(Err(e)) => {
4808 {
4809 let mut tracker = self.outcome_tracker.write().await;
4810 match &e {
4811 runner::RunnerError::Declined(_) => tracker.record_not_model_fault(
4814 &attempt_trace,
4815 &e.to_string(),
4816 outcome::UnattributedReason::Capability,
4817 ),
4818 runner::RunnerError::Failed(_) => {
4819 tracker.record_failure(&attempt_trace, &e.to_string())
4820 }
4821 }
4822 }
4823 let failed = InferenceError::InferenceFailed(e.to_string());
4830 record_auth_dead_lane(
4831 &mut auth_dead_lane,
4832 &candidate_name,
4833 &failed.to_string(),
4834 );
4835 record_route_credential_failure(
4836 &mut route_credential_failure,
4837 &candidate_name,
4838 &failed,
4839 was_primary && has_requested_route,
4840 );
4841 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
4842 last_error = Some(failed);
4843 continue;
4844 }
4845 Err(join_err) => {
4846 let msg = format!("runner task panicked: {join_err}");
4847 self.outcome_tracker
4848 .write()
4849 .await
4850 .record_failure(&attempt_trace, &msg);
4851 let failed = InferenceError::InferenceFailed(msg);
4857 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
4858 last_error = Some(failed);
4859 continue;
4860 }
4861 }
4862 }
4863
4864 let has_tools = Self::request_has_tools(&req);
4865
4866 let context = if has_tools
4868 && req.tools.as_ref().is_some_and(|t| {
4869 t.iter().any(|tool| {
4870 tool.get("function")
4871 .and_then(|f| f.get("name"))
4872 .and_then(|n| n.as_str())
4873 == Some("done")
4874 })
4875 }) {
4876 let base = req.context.as_deref().unwrap_or("");
4877 Some(format!(
4878 "{base}\n\nIMPORTANT: When calling the `done` tool, the `result` field MUST contain a DETAILED summary of everything you found and did. This is the ONLY output the user sees. Do NOT just say 'completed' — include specific findings, data, and conclusions."
4879 ))
4880 } else {
4881 req.context.clone()
4882 };
4883
4884 let mut captured_thinking: Vec<crate::tasks::generate::ThinkingBlock> = Vec::new();
4888 let mut captured_provider_output_items: Vec<serde_json::Value> = Vec::new();
4889 let mut local_reservation = if !is_delegated {
4895 match schema
4896 .as_ref()
4897 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
4898 {
4899 Some(local_schema) => {
4900 match self
4901 .reserve_local_request_with_worker_retry(
4902 local_schema,
4903 estimated_footprint,
4904 )
4905 .await
4906 {
4907 Ok(reservation) => Some(reservation),
4908 Err(error) if req.model.is_some() || req.params.strict_model => {
4909 self.outcome_tracker
4910 .write()
4911 .await
4912 .record_capability_rejection(
4913 &attempt_trace,
4914 &error.to_string(),
4915 );
4916 return Err(error);
4917 }
4918 Err(error) => {
4919 tracing::warn!(
4920 model = %local_schema.id,
4921 error = %error,
4922 "adaptive local candidate blocked by resource policy; trying next route"
4923 );
4924 self.outcome_tracker
4925 .write()
4926 .await
4927 .record_capability_rejection(
4928 &attempt_trace,
4929 &error.to_string(),
4930 );
4931 record_fallback_from(&mut fallback_hops, &candidate_name, &error);
4936 last_error = Some(error);
4937 continue;
4938 }
4939 }
4940 }
4941 None => None,
4942 }
4943 } else {
4944 None
4945 };
4946 let result = if is_codex_cli {
4947 let schema_ref = schema
4948 .as_ref()
4949 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
4950 if Self::request_needs_vision(&req)
4951 || Self::request_has_video(&req)
4952 || Self::request_has_audio(&req)
4953 {
4954 Err(InferenceError::UnsupportedMode {
4955 mode: "multimodal-content",
4956 backend: "codex-cli",
4957 reason: "the subscription-backed Codex source is text-only",
4958 })
4959 } else if req.response_format.is_some() {
4960 Err(InferenceError::UnsupportedMode {
4961 mode: "response-format",
4962 backend: "codex-cli",
4963 reason: "the subscription-backed Codex source returns plain text and does not expose provider-enforced schemas",
4964 })
4965 } else {
4966 let model = match &schema_ref.source {
4967 ModelSource::CodexCli { model } => model,
4968 _ => unreachable!("is_codex_cli matched a different source"),
4969 };
4970 crate::backend::codex_cli::generate_request(
4971 model,
4972 &req,
4973 context.as_deref(),
4974 schema_ref.context_length,
4975 )
4976 .await
4977 .map(|(output, calls)| (output.text, calls, Some(output.usage), None, None))
4978 }
4979 } else if is_remote {
4980 let (schema_val, _remote_request_reservation) = match self
4984 .vllm_live_schema(
4985 schema.unwrap(),
4986 local_reservation.take(),
4987 estimated_footprint,
4988 )
4989 .await
4990 {
4991 Ok(pair) => pair,
4992 Err(e) => {
4993 record_dispatch_failure(
4997 &mut *self.outcome_tracker.write().await,
4998 &attempt_trace,
4999 &e,
5000 );
5001 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5002 last_error = Some(e);
5003 continue;
5004 }
5005 };
5006 let _ctx_len = schema_val.context_length;
5007 let temperature = if !schema_val.supported_params.is_empty()
5010 && !schema_val
5011 .supported_params
5012 .contains(&crate::schema::GenerateParam::Temperature)
5013 {
5014 -1.0
5015 } else {
5016 req.params.temperature
5017 };
5018
5019 self.remote_backend
5025 .with_work_context(req.work_context.clone())
5026 .generate_with_tools_multi_observed(
5027 &schema_val,
5028 &req.prompt,
5029 context.as_deref(),
5030 temperature,
5031 req.params.max_tokens,
5032 req.tools.as_deref(),
5033 req.images.as_deref(),
5034 req.messages.as_deref(),
5035 req.params.tool_choice.as_deref(),
5036 req.params.parallel_tool_calls,
5037 req.params.budget_tokens,
5038 req.cache_control,
5039 req.params.cache_ttl,
5040 req.context_stable_prefix.as_deref(),
5041 req.response_format.as_ref(),
5042 retry_observer,
5043 )
5044 .await
5045 .map(|(t, c, thinking, provider_items, u, stop)| {
5051 captured_thinking = thinking;
5052 captured_provider_output_items = provider_items;
5053 (t, c, u, None::<u64>, stop)
5054 })
5055 } else if let Some(offload) = schema
5056 .as_ref()
5057 .filter(|schema| Self::supports_worker_offload(schema))
5058 .and_then(|_| crate::offload::current_local_offload())
5059 {
5060 let mut offload_req = req.clone();
5071 offload_req.model = Some(resolved_id.clone());
5072 let schema_ref = schema
5073 .as_ref()
5074 .ok_or_else(|| InferenceError::ModelNotFound(resolved_id.clone()))?;
5075 let reservation = local_reservation.as_mut().ok_or_else(|| {
5076 InferenceError::InferenceFailed(
5077 "local worker dispatch missing admission reservation".into(),
5078 )
5079 })?;
5080 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_id) {
5081 reservation.bind_allocation_id(&allocation_id);
5087 }
5088 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
5089 match offload.generate_admitted(offload_req, admission).await {
5090 Ok(outcome) => {
5091 Self::reconcile_worker_residency(
5092 offload.as_ref(),
5093 &resolved_id,
5094 &outcome.residency,
5095 outcome.retention,
5096 reservation,
5097 )
5098 .await?;
5099 let ir = outcome.result;
5100 captured_thinking = ir.thinking;
5107 captured_provider_output_items = ir.provider_output_items;
5108 Ok((
5109 ir.text,
5110 ir.tool_calls,
5111 ir.usage,
5112 ir.time_to_first_token_ms,
5113 ir.stop_reason,
5114 ))
5115 }
5116 Err(e) => Err(e),
5117 }
5118 } else {
5119 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5120 {
5121 let schema_ref = schema
5125 .as_ref()
5126 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5127
5128 if schema_ref.is_foundation_models() {
5136 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5137 if Self::request_has_video(&req) || Self::request_has_audio(&req) {
5138 Err(InferenceError::UnsupportedMode {
5139 mode: "multimodal-content",
5140 backend: "foundation-models",
5141 reason: "the FoundationModels bridge exposes text and image \
5142 input — route audio/video to a remote VL model",
5143 })
5144 } else if has_images {
5145 let images: Vec<String> = req
5151 .images
5152 .as_ref()
5153 .map(|blocks| {
5154 blocks
5155 .iter()
5156 .filter_map(|b| match b {
5157 crate::ContentBlock::ImageBase64 { data, .. } => {
5158 Some(data.clone())
5159 }
5160 _ => None,
5164 })
5165 .collect()
5166 })
5167 .unwrap_or_default();
5168 if images.is_empty() {
5169 return Err(InferenceError::UnsupportedMode {
5170 mode: "multimodal-content",
5171 backend: "foundation-models",
5172 reason: "FoundationModels takes inline image bytes; \
5173 image URLs must be fetched by the caller first",
5174 });
5175 }
5176 let prompt = req.prompt.clone();
5177 let instructions = context.clone();
5178 let fm_prompt = prompt.clone();
5179 let fm_instructions = instructions.clone();
5180 let fm_ctx_window = schema_ref.context_length as u64;
5181 let max_tokens = req.params.max_tokens as u32;
5182 let temperature = req.params.temperature;
5183 tokio::task::spawn_blocking(move || {
5184 crate::backend::foundation_models::generate_with_images(
5185 &prompt,
5186 instructions.as_deref(),
5187 &images,
5188 max_tokens,
5189 temperature as f32,
5190 )
5191 })
5192 .await
5193 .map_err(|e| {
5194 InferenceError::InferenceFailed(format!(
5195 "FoundationModels task panicked: {e}"
5196 ))
5197 })
5198 .and_then(|r| r)
5199 .map(|text| {
5200 let usage = foundation_models_usage(
5201 fm_instructions.as_deref(),
5202 &fm_prompt,
5203 &text,
5204 fm_ctx_window,
5205 );
5206 (text, vec![], usage, None, None)
5207 })
5208 } else if has_tools {
5209 if req.response_format.is_some() {
5215 tracing::warn!(
5216 "FoundationModels: response_format is ignored when tools \
5217 are present — one turn is either tool-enabled or \
5218 schema-constrained"
5219 );
5220 }
5221 let prompt = req.prompt.clone();
5222 let instructions = context.clone();
5223 let fm_prompt = prompt.clone();
5226 let fm_instructions = instructions.clone();
5227 let fm_ctx_window = schema_ref.context_length as u64;
5228 let tools_defs = req.tools.clone().unwrap_or_default();
5229 let fm_tool_choice = req.params.tool_choice.clone();
5230 let max_tokens = req.params.max_tokens as u32;
5231 let temperature = req.params.temperature;
5232 tokio::task::spawn_blocking(move || {
5233 crate::backend::foundation_models::generate_with_tools(
5234 &prompt,
5235 instructions.as_deref(),
5236 &tools_defs,
5237 fm_tool_choice.as_deref(),
5238 max_tokens,
5239 temperature as f32,
5240 )
5241 })
5242 .await
5243 .map_err(|e| {
5244 InferenceError::InferenceFailed(format!(
5245 "FoundationModels task panicked: {e}"
5246 ))
5247 })
5248 .and_then(|r| r)
5249 .map(|(text, calls)| {
5250 let usage = foundation_models_usage(
5251 fm_instructions.as_deref(),
5252 &fm_prompt,
5253 &text,
5254 fm_ctx_window,
5255 );
5256 (text, calls, usage, None, None)
5257 })
5258 } else if let Some(crate::tasks::generate::ResponseFormat::JsonSchema {
5259 schema,
5260 ..
5261 }) = &req.response_format
5262 {
5263 let prompt = req.prompt.clone();
5267 let instructions = context.clone();
5268 let fm_prompt = prompt.clone();
5271 let fm_instructions = instructions.clone();
5272 let fm_ctx_window = schema_ref.context_length as u64;
5273 let schema_val = schema.clone();
5274 let max_tokens = req.params.max_tokens as u32;
5275 let temperature = req.params.temperature;
5276 tokio::task::spawn_blocking(move || {
5277 crate::backend::foundation_models::generate_structured(
5278 &prompt,
5279 instructions.as_deref(),
5280 &schema_val,
5281 max_tokens,
5282 temperature as f32,
5283 )
5284 })
5285 .await
5286 .map_err(|e| {
5287 InferenceError::InferenceFailed(format!(
5288 "FoundationModels task panicked: {e}"
5289 ))
5290 })
5291 .and_then(|r| r)
5292 .map(|text| {
5293 let usage = foundation_models_usage(
5294 fm_instructions.as_deref(),
5295 &fm_prompt,
5296 &text,
5297 fm_ctx_window,
5298 );
5299 (text, vec![], usage, None, None)
5300 })
5301 } else {
5302 let instructions = if matches!(
5306 req.response_format,
5307 Some(crate::tasks::generate::ResponseFormat::JsonObject)
5308 ) {
5309 tracing::warn!(
5310 "FoundationModels: JsonObject response_format has no native \
5311 constrained mode — enforcing via instruction injection"
5312 );
5313 let base = context.clone().unwrap_or_default();
5314 Some(format!(
5315 "{base}\n\nRespond with a single valid JSON object and \
5316 nothing else."
5317 ))
5318 } else {
5319 context.clone()
5320 };
5321 let prompt = req.prompt.clone();
5322 let fm_prompt = prompt.clone();
5325 let fm_instructions = instructions.clone();
5326 let fm_ctx_window = schema_ref.context_length as u64;
5327 let max_tokens = req.params.max_tokens as u32;
5328 let temperature = req.params.temperature;
5329 tokio::task::spawn_blocking(move || {
5330 crate::backend::foundation_models::generate(
5331 &prompt,
5332 instructions.as_deref(),
5333 max_tokens,
5334 temperature as f32,
5335 )
5336 })
5337 .await
5338 .map_err(|e| {
5339 InferenceError::InferenceFailed(format!(
5340 "FoundationModels task panicked: {e}"
5341 ))
5342 })
5343 .and_then(|r| r)
5344 .map(|text| {
5345 let usage = foundation_models_usage(
5346 fm_instructions.as_deref(),
5347 &fm_prompt,
5348 &text,
5349 fm_ctx_window,
5350 );
5351 (text, vec![], usage, None, None)
5352 })
5353 }
5354 } else if !schema_ref.is_mlx() {
5355 Err(InferenceError::InferenceFailed(format!(
5356 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
5357 schema_ref.id
5358 )))
5359 } else if schema_ref.tags.iter().any(|t| t == "mlx-vlm-cli") {
5360 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5372 if !has_images {
5373 return Err(InferenceError::UnsupportedMode {
5374 mode: "text-only-on-mlx-vlm-id",
5375 backend: "mlx-vlm-cli",
5376 reason: "the `mlx-vlm/...` model IDs route exclusively \
5377 through the mlx-vlm CLI for image inference. \
5378 For text-only generation, route to a Qwen3 \
5379 text model (`mlx/qwen3-4b:4bit` etc.) — the \
5380 CLI shell-out has higher latency than the \
5381 in-process MLX text tower.",
5382 });
5383 }
5384 let vlm_status = crate::backend::mlx_vlm_cli::runtime_status();
5385 if !vlm_status.is_available() {
5386 return Err(InferenceError::InferenceFailed(vlm_status.user_message()));
5387 }
5388 let model_dir =
5389 self.unified_registry().ensure_local(&schema_ref.id).await?;
5390 let reservation = local_reservation
5391 .as_mut()
5392 .expect("local VLM branch has admission reservation");
5393 Self::reconcile_transient_local_allocation(
5394 reservation,
5395 backend_cache::estimate_model_size(&model_dir),
5396 )?;
5397 let detached_lease = reservation.detached_lease();
5398 let repo = match &schema_ref.source {
5399 crate::schema::ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
5400 _ => {
5401 return Err(InferenceError::InferenceFailed(format!(
5402 "model '{}' is tagged mlx-vlm-cli but its \
5403 source isn't ModelSource::Mlx — registry bug",
5404 schema_ref.id
5405 )));
5406 }
5407 };
5408 let imgs = req.images.clone().unwrap_or_default();
5409 let temp = req.params.temperature;
5410 let max_t = req.params.max_tokens;
5411 let prompt = req.prompt.clone();
5412 let (text, cli_usage) = run_admitted_blocking(detached_lease, move || {
5413 crate::backend::mlx_vlm_cli::generate(
5414 &repo, &prompt, &imgs, temp, max_t,
5415 )
5416 })
5417 .await
5418 .map_err(|e| {
5419 InferenceError::InferenceFailed(format!(
5420 "mlx_vlm CLI task panicked: {e}"
5421 ))
5422 })??;
5423 let bounding_boxes = parse_boxes(&text);
5424 let latency_ms = start.elapsed().as_millis() as u64;
5425 let usage = cli_usage.map(|u| TokenUsage {
5435 prompt_tokens: u.prompt_tokens,
5436 completion_tokens: u.completion_tokens,
5437 total_tokens: u.prompt_tokens + u.completion_tokens,
5438 context_window: schema_ref.context_length as u64,
5439 ..Default::default()
5441 });
5442 {
5443 let (in_tokens, out_tokens) = match &usage {
5449 Some(u) => (u.prompt_tokens as usize, u.completion_tokens as usize),
5450 None => (estimated_input, text.split_whitespace().count()),
5451 };
5452 let mut tracker = self.outcome_tracker.write().await;
5453 tracker.record_complete(
5454 &attempt_trace,
5455 latency_ms,
5456 in_tokens,
5457 out_tokens,
5458 );
5459 }
5460 let local_last_resort = report_local_last_resort_served(
5461 local_last_resort_id.as_deref(),
5462 candidate_id,
5463 &resolved_id,
5464 );
5465 record_generation_span(
5470 candidate_name.as_str(),
5471 latency_ms,
5472 None,
5473 usage.as_ref(),
5474 );
5475 return Ok(InferenceResult {
5476 text,
5477 tool_calls: vec![],
5478 bounding_boxes,
5479 trace_id: attempt_trace,
5480 model_used: schema_ref.id.clone(),
5481 model_identity: bound_model_identity(
5482 catalog_snapshot,
5483 requested_model_id.as_deref(),
5484 &resolved_id,
5485 )?,
5486 latency_ms,
5487 time_to_first_token_ms: None,
5488 usage,
5489 provider_output_items: Vec::new(),
5490 thinking: Vec::new(), stop_reason: None,
5492 auth_fallback_from: auth_dead_lane.clone(),
5493 local_last_resort,
5494 fallback_from: fallback_hops.clone(),
5495 });
5496 } else if !Self::supports_native_mlx(schema_ref) {
5497 if req.images.as_ref().is_some_and(|i| !i.is_empty())
5507 || Self::request_has_video(&req)
5508 || Self::request_has_audio(&req)
5509 {
5510 return Err(InferenceError::UnsupportedMode {
5511 mode: "multimodal-content-block",
5512 backend: "native-mlx-local",
5513 reason: "this in-process MLX backend is text-only; route \
5514 image/video/audio understanding to a vLLM-MLX or remote \
5515 multimodal model",
5516 });
5517 }
5518 self.generate_local(
5529 req.clone(),
5530 &schema_ref.id,
5531 local_reservation
5532 .as_mut()
5533 .expect("local branch has admission reservation"),
5534 )
5535 .await
5536 } else {
5537 if Self::request_has_video(&req) {
5543 return Err(InferenceError::UnsupportedMode {
5544 mode: "video-content-block",
5545 backend: "native-mlx-qwen25vl",
5546 reason: "Qwen2.5-VL video understanding is on the request surface \
5547 but the video-tokenization path (frame sampling + merger) \
5548 is not yet wired; route to a remote VL provider for now",
5549 });
5550 }
5551 if Self::request_has_audio(&req) {
5552 return Err(InferenceError::UnsupportedMode {
5553 mode: "audio-content-block",
5554 backend: "native-mlx-qwen25vl",
5555 reason: "audio understanding is on the request surface (Gemma 4 \
5556 E2B/E4B and Gemini accept it) but the native MLX path \
5557 for this model does not — route to Gemini or Gemma-4",
5558 });
5559 }
5560 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5561 if has_images {
5562 let (handle, _retention) = self
5582 .ensure_text_backend(
5583 schema_ref,
5584 local_reservation
5585 .as_mut()
5586 .expect("local branch has admission reservation"),
5587 )
5588 .await?;
5589 let can_do_vision = {
5590 let guard = handle.lock().map_err(|_| {
5591 InferenceError::InferenceFailed(
5592 "MLX backend mutex poisoned".into(),
5593 )
5594 })?;
5595 guard.supports_capability(crate::schema::ModelCapability::Vision)
5596 };
5597 if !can_do_vision {
5598 return Err(InferenceError::UnsupportedMode {
5608 mode: "image-content-block",
5609 backend: "native-mlx-text",
5610 reason: "this MLX backend is a plain Qwen3 text tower. \
5611 For local image inference, route to \
5612 `mlx-vlm/qwen3-vl-2b:bf16` or another `mlx-vlm/...` \
5613 catalog ID so CAR shells out to `mlx_vlm.generate`. \
5614 Alternatives: a local vLLM-MLX VLM server, or a \
5615 remote VL model. (#115)",
5616 });
5617 }
5618 }
5619 self.generate_mlx(
5620 render_for_local_backend(req.clone()),
5621 &schema_ref.id,
5622 local_reservation
5623 .as_mut()
5624 .expect("local branch has admission reservation"),
5625 )
5626 .await
5627 .map(|(text, usage, ttft, stop)| (text, vec![], usage, ttft, stop))
5628 }
5629 }
5630
5631 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
5632 {
5633 let schema_ref = schema
5634 .as_ref()
5635 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5636 match self
5637 .ensure_backend(
5638 schema_ref,
5639 local_reservation
5640 .as_mut()
5641 .expect("local branch has admission reservation"),
5642 )
5643 .await
5644 {
5645 Ok(()) => {
5646 let mut write = self.backend.write().await;
5647 let backend = write.get_mut(&schema_ref.id).ok_or_else(|| {
5648 InferenceError::InferenceFailed(format!(
5649 "candle backend missing after ensure_backend for {}",
5650 schema_ref.id
5651 ))
5652 })?;
5653 let ctx_window = backend.context_length().unwrap_or(0) as u64;
5659 tasks::generate::generate(
5660 backend,
5661 render_for_local_backend(req.clone()),
5662 )
5663 .await
5664 .map(
5665 |(text, ttft, prompt_tokens, completion_tokens)| {
5666 let usage = TokenUsage {
5667 prompt_tokens: prompt_tokens as u64,
5668 completion_tokens: completion_tokens as u64,
5669 total_tokens: (prompt_tokens + completion_tokens) as u64,
5670 context_window: ctx_window,
5671 ..Default::default()
5674 };
5675 (text, vec![], Some(usage), ttft, None)
5676 },
5677 )
5678 }
5679 Err(e) => Err(e),
5680 }
5681 }
5682 };
5683
5684 match result {
5685 Ok((text, mut tool_calls, usage, time_to_first_token_ms, stop_reason)) => {
5686 let text = if !is_remote
5693 && !is_codex_cli
5694 && !is_delegated
5695 && req.tools.is_some()
5696 && tool_calls.is_empty()
5697 {
5698 let (clean, parsed) = tasks::generate::parse_tool_calls(&text);
5699 tool_calls = parsed;
5700 clean
5701 } else {
5702 text
5703 };
5704 let latency_ms = start.elapsed().as_millis() as u64;
5705 let estimated_tokens = usage
5706 .as_ref()
5707 .map(|u| u.completion_tokens as usize)
5708 .unwrap_or_else(|| text.split_whitespace().count());
5709 let input_tokens = usage
5714 .as_ref()
5715 .map(|u| u.prompt_tokens as usize)
5716 .unwrap_or(estimated_input);
5717 let (cache_read, cache_creation) = usage
5722 .as_ref()
5723 .map(|u| {
5724 (
5725 u.cache_read_input_tokens as usize,
5726 u.cache_creation_input_tokens as usize,
5727 )
5728 })
5729 .unwrap_or((0, 0));
5730 {
5731 let mut tracker = self.outcome_tracker.write().await;
5732 tracker.record_complete_cached(
5733 &attempt_trace,
5734 latency_ms,
5735 input_tokens,
5736 estimated_tokens,
5737 cache_read,
5738 cache_creation,
5739 );
5740 }
5741 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
5747 cb.record_success(&resolved_id);
5748 }
5749 self.auto_save_outcomes().await;
5751
5752 record_generation_span(
5754 candidate_name.as_str(),
5755 latency_ms,
5756 time_to_first_token_ms,
5757 usage.as_ref(),
5758 );
5759
5760 let bounding_boxes = tasks::grounding::parse_boxes(&text);
5763 let local_last_resort = report_local_last_resort_served(
5764 local_last_resort_id.as_deref(),
5765 candidate_id,
5766 &resolved_id,
5767 );
5768 return Ok(InferenceResult {
5769 text,
5770 tool_calls,
5771 bounding_boxes,
5772 trace_id: attempt_trace,
5773 model_used: reported_model_used,
5774 model_identity: bound_model_identity(
5775 catalog_snapshot,
5776 requested_model_id.as_deref(),
5777 &resolved_id,
5778 )?,
5779 latency_ms,
5780 time_to_first_token_ms,
5781 usage,
5782 provider_output_items: captured_provider_output_items,
5783 thinking: captured_thinking,
5784 stop_reason,
5785 auth_fallback_from: auth_dead_lane.clone(),
5786 local_last_resort,
5787 fallback_from: fallback_hops.clone(),
5788 });
5789 }
5790 Err(e) => {
5791 if matches!(e, InferenceError::ControlledTermination) {
5792 self.outcome_tracker.write().await.record_not_model_fault(
5793 &attempt_trace,
5794 &e.to_string(),
5795 outcome::UnattributedReason::Cancelled,
5796 );
5797 return Err(e);
5798 }
5799 tracing::warn!(
5800 model = %candidate_name,
5801 error = %e,
5802 remaining = candidate_queue.len(),
5803 "model failed, trying next fallback immediately"
5804 );
5805 record_auth_dead_lane(&mut auth_dead_lane, &candidate_name, &e.to_string());
5812 record_route_credential_failure(
5813 &mut route_credential_failure,
5814 &candidate_name,
5815 &e,
5816 was_primary && has_requested_route,
5817 );
5818 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5819 {
5824 let mut tracker = self.outcome_tracker.write().await;
5825 record_dispatch_failure(&mut tracker, &attempt_trace, &e);
5826 }
5827 if error_counts_against_circuit_breaker(&e) {
5837 let err_str = e.to_string();
5838 let is_client_error =
5839 err_str.contains("API returned 4") && !err_str.contains("429");
5840 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
5841 cb.record_failure(&resolved_id);
5843 if is_client_error {
5846 cb.record_failure(&resolved_id);
5847 }
5848 }
5849 }
5850 #[cfg(not(all(
5852 target_os = "macos",
5853 target_arch = "aarch64",
5854 not(car_skip_mlx)
5855 )))]
5856 {
5857 let mut write = self.backend.write().await;
5858 if write.remove(&resolved_id).is_some() {
5859 self.local_admission.mark_evicted(&resolved_id);
5860 }
5861 }
5862 if let InferenceError::GatewayUnconfigured { namespace, .. } = &e {
5890 let before = candidate_queue.len();
5891 candidate_queue.retain(|id| !id.starts_with(namespace.as_str()));
5892 let dropped = before - candidate_queue.len();
5893 if dropped > 0 {
5894 tracing::info!(
5895 %namespace,
5896 dropped,
5897 remaining = candidate_queue.len(),
5898 "gateway has no upstream for this namespace; dropping its \
5899 remaining candidates instead of replaying the same rejection"
5900 );
5901 }
5902 }
5903 if error_ends_fallback_chain(&e) {
5904 let dropped = candidate_queue.len();
5905 candidate_queue.clear();
5906 if dropped > 0 {
5907 tracing::info!(
5908 dropped,
5909 "content refused for this request; ending the fallback chain \
5910 rather than answering with a different model"
5911 );
5912 }
5913 }
5914 if let InferenceError::ProviderAccount { provider, .. } = &e {
5915 let mut rest: Vec<String> = candidate_queue.iter().cloned().collect();
5919 let demoted = routing_ext::demote_provider(provider, &mut rest, |id| {
5920 routing_registry
5921 .get(id)
5922 .or_else(|| routing_registry.find_by_name(id))
5923 .map(|s| s.provider.clone())
5924 });
5925 candidate_queue = rest.into();
5926 if demoted > 0 {
5927 tracing::info!(
5928 %provider,
5929 demoted,
5930 remaining = candidate_queue.len(),
5931 "account-level rejection; deferring this provider's \
5932 remaining candidates to the end of the chain"
5933 );
5934 }
5935 }
5936 if was_primary && !candidate_queue.is_empty() {
5937 let err_str = e.to_string();
5938 let status = parse_api_returned_status(&err_str);
5945 if routing_ext::is_provider_transient(status, &err_str) {
5946 let provider_of = |id: &str| {
5951 routing_registry
5952 .get(id)
5953 .or_else(|| routing_registry.find_by_name(id))
5954 .map(|s| s.provider.clone())
5955 };
5956 let primary = provider_of(candidate_id).unwrap_or_default();
5957 let queue_vec: Vec<String> = candidate_queue.iter().cloned().collect();
5958 if let Some(cross) =
5959 routing_ext::first_cross_provider(&primary, &queue_vec, provider_of)
5960 {
5961 let cross = cross.to_string();
5962 if let Some(pos) = candidate_queue.iter().position(|m| *m == cross)
5963 {
5964 if pos > 0 {
5965 if let Some(m) = candidate_queue.remove(pos) {
5966 tracing::info!(
5967 promoted = %m,
5968 "transient provider error on primary; promoting cross-provider fallback"
5969 );
5970 candidate_queue.push_front(m);
5971 }
5972 }
5973 }
5974 }
5975 }
5976 }
5977 last_error = Some(e);
5978 }
5979 }
5980 }
5981
5982 let underlying = last_error.unwrap_or(InferenceError::InferenceFailed(
5984 "no models available".into(),
5985 ));
5986
5987 let e = apply_route_failure_context(underlying, route_credential_failure.as_ref());
6001 self.auto_save_outcomes().await;
6005 Err(e)
6006 }
6007
6008 async fn generate_stream_raw(
6013 &self,
6014 req: GenerateRequest,
6015 ) -> Result<(String, tokio::sync::mpsc::Receiver<stream::StreamEvent>), InferenceError> {
6016 let routing_registry = if let Some(context) = &req.work_context {
6017 self.work_context_routing_snapshot(context, req.model.as_deref())
6018 .await
6019 .0
6020 } else {
6021 self.request_routing_registry_snapshot(req.model.as_deref())
6022 .await
6023 };
6024 if let Some(requested) = req.model.as_deref() {
6025 if routing_registry
6026 .get(requested)
6027 .or_else(|| routing_registry.find_by_name(requested))
6028 .is_none()
6029 {
6030 return Err(InferenceError::ModelNotFound(requested.to_string()));
6031 }
6032 }
6033 let has_tools = Self::request_has_tools(&req);
6034 let has_vision = Self::request_needs_vision(&req);
6035 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6036 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
6037 let (estimated_cache_read, estimated_cache_write) =
6038 Self::routing_cache_estimates(&req, estimated_input);
6039 let preferred_model = self
6040 .preferred_model_for_capability(ModelCapability::Generate)
6041 .map(str::to_string);
6042 let exclude_set = self
6043 .adaptive_router
6044 .build_exclude_set(req.intent.as_ref(), &routing_registry);
6045 let unpinned_override = self
6046 .lane_pin_for(&req, &routing_registry)
6047 .or(preferred_model)
6048 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model));
6049 let decision = match req.model.clone().or(unpinned_override) {
6050 Some(m) => {
6051 let ctx_len = routing_registry
6052 .get(&m)
6053 .or_else(|| routing_registry.find_by_name(&m))
6054 .map(|s| s.context_length)
6055 .unwrap_or(0);
6056 AdaptiveRoutingDecision {
6057 model_id: m.clone(),
6058 model_name: m,
6059 task: InferenceTask::Generate,
6060 complexity: TaskComplexity::assess(&req.prompt),
6061 reason: "explicit model".into(),
6062 strategy: RoutingStrategy::Explicit,
6063 predicted_quality: 0.5,
6064 fallbacks: vec![],
6065 context_length: ctx_len,
6066 needs_compaction: false,
6067 candidates: vec![],
6068 }
6069 }
6070 None => {
6071 let tracker_read = self.outcome_tracker.read().await;
6072 self.adaptive_router
6073 .route_with(crate::adaptive_router::RouteRequest {
6074 estimated_total_tokens: estimated_footprint,
6075 estimated_input_tokens: estimated_input,
6076 estimated_output_tokens: req.params.max_tokens,
6077 estimated_cache_read_tokens: estimated_cache_read,
6078 estimated_cache_write_tokens: estimated_cache_write,
6079 has_tools,
6080 has_vision,
6081 workload: req.params.workload,
6082 intent: req.intent.as_ref(),
6083 ..crate::adaptive_router::RouteRequest::new(
6084 &req.prompt,
6085 &routing_registry,
6086 &tracker_read,
6087 )
6088 })
6089 }
6090 };
6091
6092 if decision.model_id.is_empty() {
6093 let excluded_models = req
6094 .intent
6095 .as_ref()
6096 .map(|hint| hint.exclude_models.join(", "))
6097 .unwrap_or_default();
6098 return Err(InferenceError::NoEligibleModel { excluded_models });
6099 }
6100
6101 #[allow(unused_mut)]
6104 let mut schema = routing_registry
6105 .get(&decision.model_id)
6106 .or_else(|| routing_registry.find_by_name(&decision.model_id))
6107 .cloned();
6108
6109 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6111 if let Some(ref s) = schema {
6112 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
6113 tracing::info!(
6114 from = %s.id, to = %mlx_equiv.id,
6115 "redirecting GGUF model to MLX equivalent on Apple Silicon (stream)"
6116 );
6117 schema = Some(mlx_equiv.clone());
6118 }
6119 }
6120
6121 if has_tools
6127 && schema
6128 .as_ref()
6129 .map(|s| !s.has_capability(ModelCapability::ToolUse))
6130 .unwrap_or(false)
6131 {
6132 let backend = schema
6133 .as_ref()
6134 .map(|s| if s.is_local() { "local" } else { "remote" })
6135 .unwrap_or("unknown");
6136 return Err(InferenceError::UnsupportedMode {
6137 mode: "tool_use",
6138 backend,
6139 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
6140 });
6141 }
6142
6143 let resolved_model_id = schema
6145 .as_ref()
6146 .map(|s| s.id.clone())
6147 .unwrap_or_else(|| decision.model_id.clone());
6148
6149 let mut req = req;
6156 if let Some(schema) = schema.as_ref() {
6157 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
6158 }
6159
6160 let is_remote = schema
6161 .as_ref()
6162 .map(|s| s.is_remote() || s.is_vllm_mlx())
6163 .unwrap_or(false);
6164
6165 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
6166 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
6167
6168 if is_codex_cli {
6169 return Err(InferenceError::UnsupportedMode {
6170 mode: "streaming",
6171 backend: "codex-cli",
6172 reason: "codex exec reports final answer items rather than token deltas; use non-streaming infer/generate_tracked",
6173 });
6174 }
6175
6176 if is_delegated {
6177 let runner = runner::current_inference_runner().ok_or_else(|| {
6181 InferenceError::InferenceFailed(
6182 "model declares ModelSource::Delegated but no inference runner is registered \
6183 (call set_inference_runner / registerInferenceRunner / register_inference_runner)"
6184 .into(),
6185 )
6186 })?;
6187 let (tx, rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6188 let emitter = runner::EventEmitter::new(tx);
6189 let request = req.clone();
6190 tokio::spawn(async move {
6191 if let Err(e) = runner.run(request, emitter).await {
6192 tracing::warn!(error = %e, "delegated inference runner failed");
6193 }
6194 });
6195 return Ok((resolved_model_id, rx));
6196 }
6197
6198 let mut local_reservation = if !is_remote
6200 || schema.as_ref().is_some_and(ModelSchema::is_vllm_mlx)
6201 {
6202 if let Some(local_schema) = schema
6203 .as_ref()
6204 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
6205 {
6206 Some(
6207 self.reserve_local_request_with_worker_retry(local_schema, estimated_footprint)
6208 .await?,
6209 )
6210 } else {
6211 None
6212 }
6213 } else {
6214 None
6215 };
6216
6217 if !is_remote && schema.as_ref().is_some_and(Self::supports_worker_offload) {
6224 if let Some(offload) = crate::offload::current_local_offload() {
6225 let mut offload_req = req.clone();
6226 offload_req.model = Some(resolved_model_id.clone());
6227 let schema_ref = schema
6228 .as_ref()
6229 .ok_or_else(|| InferenceError::ModelNotFound(resolved_model_id.clone()))?;
6230 let reservation = local_reservation.as_mut().ok_or_else(|| {
6231 InferenceError::InferenceFailed(
6232 "local worker stream missing admission reservation".into(),
6233 )
6234 })?;
6235 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_model_id) {
6236 reservation.bind_allocation_id(&allocation_id);
6237 }
6238 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
6239 let offload_stream = offload.stream_admitted(offload_req, admission).await?;
6240 Self::reconcile_worker_residency(
6241 offload.as_ref(),
6242 &resolved_model_id,
6243 &offload_stream.residency,
6244 offload_stream.retention,
6245 reservation,
6246 )
6247 .await?;
6248 let rx = offload_stream.events;
6249 let rx = match local_reservation {
6250 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6251 None => rx,
6252 };
6253 return Ok((resolved_model_id, rx));
6254 }
6255 }
6256
6257 if is_remote {
6258 let mut candidates = vec![schema.unwrap()];
6259 let mut local_fallback_ids = Vec::new();
6260 for fallback_id in &decision.fallbacks {
6261 if let Some(fallback) = routing_registry
6262 .get(fallback_id)
6263 .or_else(|| routing_registry.find_by_name(fallback_id))
6264 {
6265 if (fallback.is_remote() || fallback.is_vllm_mlx())
6266 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6267 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6268 && !candidates
6269 .iter()
6270 .any(|candidate| candidate.id == fallback.id)
6271 {
6272 candidates.push(fallback.clone());
6273 } else if fallback.is_local()
6274 && !fallback.is_vllm_mlx()
6275 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6276 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6277 {
6278 local_fallback_ids.push(fallback.id.clone());
6279 }
6280 }
6281 }
6282 let mut last_error = None;
6283 for candidate in candidates {
6284 let (candidate, mut candidate_reservation) = match self
6287 .vllm_live_schema(candidate, local_reservation.take(), estimated_footprint)
6288 .await
6289 {
6290 Ok(candidate) => candidate,
6291 Err(error) => {
6292 last_error = Some(error);
6293 continue;
6294 }
6295 };
6296 self.remote_backend.register_model_keys(&candidate).await;
6297
6298 let spend_guard = self
6299 .spend_limits
6300 .read()
6301 .unwrap()
6302 .as_ref()
6303 .and_then(|limits| limits.per_request_usd)
6304 .map(|budget| {
6305 let mut prompt_tokens =
6306 routing_ext::MidStreamSpendGuard::estimate_tokens(&req.prompt)
6307 + req
6308 .context
6309 .as_deref()
6310 .map(routing_ext::MidStreamSpendGuard::estimate_tokens)
6311 .unwrap_or(0);
6312 prompt_tokens += media_tokens::request_media_and_history_tokens(
6313 req.images.as_deref(),
6314 req.messages.as_deref(),
6315 ) as u64;
6316 if let Some(tools) = &req.tools {
6317 prompt_tokens += routing_ext::MidStreamSpendGuard::estimate_tokens(
6318 &serde_json::to_string(tools).unwrap_or_default(),
6319 );
6320 }
6321 let prices = candidate.cost.prices_for(prompt_tokens as usize);
6322 let input_price = prices
6323 .input_per_mtok
6324 .map(|c| c / 1_000_000.0)
6325 .unwrap_or(0.0);
6326 let output_price = prices
6327 .output_per_mtok
6328 .map(|c| c / 1_000_000.0)
6329 .unwrap_or(0.0);
6330 routing_ext::MidStreamSpendGuard::new(
6331 Some(budget),
6332 prompt_tokens as f64 * input_price,
6333 input_price,
6334 output_price,
6335 )
6336 });
6337
6338 match self
6339 .remote_backend
6340 .with_work_context(req.work_context.clone())
6341 .generate_stream(
6342 &candidate,
6343 &req.prompt,
6344 req.messages.as_deref(),
6345 req.context.as_deref(),
6346 req.params.temperature,
6347 req.params.max_tokens,
6348 req.tools.as_deref(),
6349 req.images.as_deref(),
6350 req.params.tool_choice.as_deref(),
6351 req.params.parallel_tool_calls,
6352 req.response_format.as_ref(),
6353 spend_guard,
6354 )
6355 .await
6356 {
6357 Ok(receiver) => {
6358 let receiver = Self::hold_optional_reservation_for_stream(
6359 receiver,
6360 candidate_reservation.take(),
6361 );
6362 return Ok((candidate.id, receiver));
6363 }
6364 Err(error) => {
6365 tracing::warn!(
6366 model = %candidate.id,
6367 %error,
6368 "remote stream setup failed; trying routed fallback"
6369 );
6370 last_error = Some(error);
6371 }
6372 }
6373 }
6374
6375 if !req.params.strict_model {
6380 if let Some(local) = self.first_installed_local_model(has_tools) {
6381 let schema = routing_registry
6382 .get(&local)
6383 .or_else(|| routing_registry.find_by_name(&local));
6384 let supports_request = schema.is_some_and(|schema| {
6385 !has_vision || schema.has_capability(ModelCapability::Vision)
6386 });
6387 if supports_request
6388 && !local_fallback_ids.iter().any(|candidate| {
6389 routing_registry
6390 .get(candidate)
6391 .or_else(|| routing_registry.find_by_name(candidate))
6392 .is_some_and(|schema| schema.id == local || schema.name == local)
6393 })
6394 {
6395 local_fallback_ids.push(local);
6396 }
6397 }
6398 for local_id in local_fallback_ids {
6399 let mut fallback_req = req.clone();
6400 fallback_req.model = Some(local_id);
6401 fallback_req.params.strict_model = true;
6402 match Box::pin(self.generate_stream_raw(fallback_req)).await {
6403 Ok(stream) => return Ok(stream),
6404 Err(error) => {
6405 tracing::warn!(%error, "local streaming fallback failed");
6406 last_error = Some(error);
6407 }
6408 }
6409 }
6410 }
6411
6412 Err(last_error.unwrap_or_else(|| {
6413 InferenceError::InferenceFailed(
6414 "no compatible remote streaming model available".to_string(),
6415 )
6416 }))
6417 } else {
6418 let schema =
6419 schema.ok_or_else(|| InferenceError::ModelNotFound(decision.model_id.clone()))?;
6420 let (tx, rx) = tokio::sync::mpsc::channel(64);
6421
6422 #[cfg(any(
6428 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
6429 all(target_os = "ios", target_arch = "aarch64")
6430 ))]
6431 {
6432 if schema.is_foundation_models() {
6433 if Self::request_has_video(&req)
6437 || Self::request_has_audio(&req)
6438 || req.images.as_ref().is_some_and(|imgs| !imgs.is_empty())
6439 {
6440 return Err(InferenceError::UnsupportedMode {
6441 mode: "multimodal-content",
6442 backend: "foundation-models",
6443 reason: "the FoundationModels bridge currently exposes text-only \
6444 generation — route image/audio/video to a remote VL model",
6445 });
6446 }
6447 if let Some(tools_defs) = req.tools.clone().filter(|t| !t.is_empty()) {
6453 let prompt = req.prompt.clone();
6454 let instructions = req.context.clone();
6455 let fm_tool_choice = req.params.tool_choice.clone();
6456 let max_tokens = req.params.max_tokens as u32;
6457 let temperature = req.params.temperature;
6458 tokio::task::spawn_blocking(move || {
6459 match crate::backend::foundation_models::generate_with_tools(
6460 &prompt,
6461 instructions.as_deref(),
6462 &tools_defs,
6463 fm_tool_choice.as_deref(),
6464 max_tokens,
6465 temperature as f32,
6466 ) {
6467 Ok((text, tool_calls)) => {
6468 if !text.is_empty() {
6469 let _ = tx.blocking_send(stream::StreamEvent::TextDelta(
6470 text.clone(),
6471 ));
6472 }
6473 let _ = tx.blocking_send(stream::StreamEvent::Done {
6474 text,
6475 tool_calls,
6476 });
6477 }
6478 Err(e) => {
6479 tracing::warn!(
6480 error = %e,
6481 "FoundationModels tool-enabled stream failed"
6482 );
6483 send_stream_failure_blocking(&tx, &e);
6489 }
6490 }
6491 });
6492 return Ok((resolved_model_id, rx));
6493 }
6494 let prompt = req.prompt.clone();
6495 let instructions = req.context.clone();
6496 let max_tokens = req.params.max_tokens as u32;
6497 let temperature = req.params.temperature;
6498 let tx_clone = tx.clone();
6499 tokio::task::spawn_blocking(move || {
6500 let accum = std::sync::Arc::new(std::sync::Mutex::new(String::new()));
6507 let accum_cb = accum.clone();
6508 let cb = crate::backend::foundation_models::StreamCallback::new(
6509 move |delta: &str| {
6510 if let Ok(mut g) = accum_cb.lock() {
6511 g.push_str(delta);
6512 }
6513 tx_clone
6514 .blocking_send(stream::StreamEvent::TextDelta(
6515 delta.to_string(),
6516 ))
6517 .is_ok()
6518 },
6519 );
6520 let result = crate::backend::foundation_models::stream(
6521 &prompt,
6522 instructions.as_deref(),
6523 max_tokens,
6524 temperature as f32,
6525 cb,
6526 );
6527 let final_text = accum.lock().map(|g| g.clone()).unwrap_or_default();
6528 match &result {
6532 Ok(()) => {
6533 let _ = tx.blocking_send(stream::StreamEvent::Done {
6534 text: final_text,
6535 tool_calls: vec![],
6536 });
6537 }
6538 Err(e) => send_stream_failure_blocking(&tx, e),
6539 }
6540 result
6541 });
6542 return Ok((resolved_model_id, rx));
6543 }
6544 }
6545
6546 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6548 {
6549 if !schema.is_mlx() {
6552 return Err(InferenceError::InferenceFailed(format!(
6553 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
6554 schema.id
6555 )));
6556 }
6557 let (backend, _retention) = self
6558 .ensure_text_backend(
6559 &schema,
6560 local_reservation
6561 .as_mut()
6562 .expect("local stream has admission reservation"),
6563 )
6564 .await?;
6565 let model_id = schema.id.clone();
6566 let cache = Arc::clone(&self.text_backends);
6567 let device_guard = Self::mlx_device_lock().lock_owned().await;
6574 tokio::task::spawn_blocking(move || {
6580 let _device_guard = device_guard;
6581 let _ = Self::stream_local_mlx(backend, cache, model_id, req, tx);
6582 });
6583 let rx = match local_reservation {
6584 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6585 None => rx,
6586 };
6587 Ok((resolved_model_id, rx))
6588 }
6589
6590 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6591 {
6592 self.ensure_backend(
6593 &schema,
6594 local_reservation
6595 .as_mut()
6596 .expect("local stream has admission reservation"),
6597 )
6598 .await?;
6599 let backend = self.backend.clone();
6600 let model_id = schema.id.clone();
6601 tokio::spawn(async move {
6602 let _ = Self::stream_local_candle(backend, model_id, req, tx).await;
6603 });
6604 let rx = match local_reservation {
6605 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6606 None => rx,
6607 };
6608 Ok((resolved_model_id, rx))
6609 }
6610 }
6611 }
6612
6613 pub async fn generate_tracked_stream(
6658 &self,
6659 req: GenerateRequest,
6660 ) -> Result<TrackedStream, InferenceError> {
6661 self.refuse_system_one_generate(&req)?;
6662 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6666 let start = Instant::now();
6667
6668 let (model_id, mut producer_rx) = self.generate_stream_raw(req).await?;
6672
6673 let trace = {
6674 let mut t = self.outcome_tracker.write().await;
6675 t.record_start(&model_id, InferenceTask::Generate, "stream")
6676 };
6677 let trace_for_return = trace.clone();
6681
6682 let (out_tx, out_rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6683 let tracker = Arc::clone(&self.outcome_tracker);
6684 tokio::spawn(async move {
6685 let mut acc = stream::StreamAccumulator::default();
6686 let mut stream_error: Option<String> = None;
6687 let mut attribution: Option<outcome::UnattributedReason> = None;
6688 let mut saw_done = false;
6689 let mut receiver_abandoned = false;
6690 while let Some(evt) = producer_rx.recv().await {
6691 if let stream::StreamEvent::Attribution(reason) = &evt {
6692 attribution = Some(*reason);
6694 continue;
6695 }
6696 if let stream::StreamEvent::Error(message) = &evt {
6697 stream_error = Some(message.clone());
6698 }
6699 if matches!(evt, stream::StreamEvent::Done { .. }) {
6700 saw_done = true;
6701 }
6702 acc.push(&evt);
6703 if out_tx.send(evt).await.is_err() {
6704 receiver_abandoned = true;
6705 break;
6706 }
6707 }
6708 if stream_error.is_none() && !saw_done && !receiver_abandoned {
6709 let error = "stream ended without positive provider completion".to_string();
6710 stream_error = Some(error.clone());
6711 let _ = out_tx.send(stream::StreamEvent::Error(error)).await;
6712 } else if stream_error.is_none() && receiver_abandoned {
6713 stream_error = Some("stream receiver was abandoned before completion".to_string());
6717 attribution = Some(outcome::UnattributedReason::Cancelled);
6718 }
6719 let (text, tool_calls, usage, _stop) = acc.finish_with_usage();
6720 let latency_ms = start.elapsed().as_millis() as u64;
6721 let input_tokens = usage
6722 .as_ref()
6723 .map(|u| u.prompt_tokens as usize)
6724 .unwrap_or(estimated_input);
6725 let mut output_tokens = usage
6730 .as_ref()
6731 .map(|u| u.completion_tokens as usize)
6732 .unwrap_or_else(|| text.split_whitespace().count());
6733 if output_tokens == 0 && !tool_calls.is_empty() {
6734 output_tokens = 1;
6735 }
6736 let (cache_read, cache_creation) = usage
6741 .as_ref()
6742 .map(|u| {
6743 (
6744 u.cache_read_input_tokens as usize,
6745 u.cache_creation_input_tokens as usize,
6746 )
6747 })
6748 .unwrap_or((0, 0));
6749 let mut t = tracker.write().await;
6750 if let Some(error) = stream_error {
6751 let reason = attribution.or_else(|| {
6759 stream::content_refusal_tags(&error)
6760 .map(|_| outcome::UnattributedReason::ContentRefused)
6761 });
6762 t.note_elapsed(&trace, latency_ms);
6763 match reason {
6764 Some(reason) => t.record_not_model_fault(&trace, &error, reason),
6765 None => t.record_failure(&trace, &error),
6766 }
6767 } else {
6768 t.record_complete_cached(
6769 &trace,
6770 latency_ms,
6771 input_tokens,
6772 output_tokens,
6773 cache_read,
6774 cache_creation,
6775 );
6776 }
6777 });
6778
6779 Ok(TrackedStream {
6780 model_used: model_id,
6781 trace_id: trace_for_return,
6782 events: out_rx,
6783 })
6784 }
6785
6786 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6787 async fn stream_local_candle(
6788 backend_lock: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
6789 model_id: String,
6790 req: GenerateRequest,
6791 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
6792 ) -> Result<(), InferenceError> {
6793 let mut write = backend_lock.write().await;
6794 let backend = write.get_mut(&model_id).ok_or_else(|| {
6795 InferenceError::InferenceFailed(format!("backend not initialized for {model_id}"))
6796 })?;
6797 backend.clear_kv_cache();
6798
6799 let formatted = tasks::generate::render_chat_prompt(&req);
6800 let tokens = backend.encode(&formatted)?;
6801 let eos = backend.eos_token_id();
6802 let eos_alt = backend.token_id("<|im_end|>");
6803 let params = &req.params;
6804
6805 if tokens.is_empty() {
6806 let _ = tx
6807 .send(stream::StreamEvent::Done {
6808 text: String::new(),
6809 tool_calls: vec![],
6810 })
6811 .await;
6812 return Ok(());
6813 }
6814
6815 let max_ctx = backend.context_length().unwrap_or(32768);
6816 let headroom = params.max_tokens.min(max_ctx / 4);
6817 let max_prompt = max_ctx.saturating_sub(headroom);
6818 let tokens = if tokens.len() > max_prompt {
6819 tokens[tokens.len() - max_prompt..].to_vec()
6820 } else {
6821 tokens
6822 };
6823
6824 let mut generated = Vec::new();
6825 let logits = backend.forward(&tokens, 0)?;
6826 let mut next_token = tasks::generate::sample_token(&logits, params)?;
6827
6828 for _ in 0..params.max_tokens {
6829 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
6830 break;
6831 }
6832
6833 generated.push(next_token);
6834 let delta = backend.decode(&[next_token])?;
6835 if !delta.is_empty()
6836 && tx
6837 .send(stream::StreamEvent::TextDelta(delta))
6838 .await
6839 .is_err()
6840 {
6841 return Ok(());
6842 }
6843
6844 if !params.stop.is_empty() {
6845 let text_so_far = backend.decode(&generated)?;
6846 if params.stop.iter().any(|s| text_so_far.contains(s)) {
6847 break;
6848 }
6849 }
6850
6851 let pos = tokens.len() + generated.len() - 1;
6852 let logits = backend.forward(&[next_token], pos)?;
6853 next_token = tasks::generate::sample_token(&logits, params)?;
6854 }
6855
6856 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
6857 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
6858 let _ = tx
6862 .send(stream::StreamEvent::Usage {
6863 input_tokens: tokens.len() as u64,
6864 output_tokens: generated.len() as u64,
6865 cache_read_input_tokens: 0,
6866 cache_creation_input_tokens: 0,
6867 })
6868 .await;
6869 let _ = tx
6870 .send(stream::StreamEvent::Done {
6871 text,
6872 tool_calls: vec![],
6873 })
6874 .await;
6875 Ok(())
6876 }
6877
6878 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6882 fn stream_local_mlx(
6883 handle: backend_cache::CachedBackend<backend::SwiftLmBackend>,
6884 cache: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
6885 model_id: String,
6886 req: GenerateRequest,
6887 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
6888 ) -> Result<(), InferenceError> {
6889 let mut guard = handle.lock().map_err(|_| {
6890 InferenceError::InferenceFailed(format!("MLX backend mutex poisoned for {model_id}"))
6891 })?;
6892 let backend: &mut backend::SwiftLmBackend = &mut guard;
6893 backend.clear_kv_cache();
6894
6895 let formatted = tasks::generate::render_chat_prompt(&req);
6896 let tokens = backend.encode(&formatted)?;
6897 let eos = backend.eos_token_id();
6898 let eos_alt = backend.token_id("<|im_end|>");
6899 let params = &req.params;
6900
6901 if tokens.is_empty() {
6902 let _ = tx.blocking_send(stream::StreamEvent::Done {
6903 text: String::new(),
6904 tool_calls: vec![],
6905 });
6906 return Ok(());
6907 }
6908
6909 let max_ctx = backend.context_length();
6910 let headroom = params.max_tokens.min(max_ctx / 4);
6911 let max_prompt = max_ctx.saturating_sub(headroom);
6912 let tokens = if tokens.len() > max_prompt {
6913 tokens[tokens.len() - max_prompt..].to_vec()
6914 } else {
6915 tokens
6916 };
6917
6918 let mut generated = Vec::new();
6919
6920 let started = std::time::Instant::now();
6926 let timeout = local_decode_timeout();
6927 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
6928 let mut last_heartbeat = std::time::Duration::ZERO;
6929 tracing::info!(
6930 prompt_tokens = tokens.len(),
6931 max_tokens = params.max_tokens,
6932 timeout_secs = timeout.map(|t| t.as_secs()),
6933 "local stream prefill starting"
6934 );
6935
6936 let logits = match Self::catch_mlx("stream prefill", || backend.forward(&tokens, 0)) {
6941 Ok(v) => v,
6942 Err(e) => {
6943 cache.invalidate(&model_id);
6944 return Err(e);
6945 }
6946 };
6947 let mut next_token = Self::sample_from_logits(&logits, params)?;
6948
6949 for _ in 0..params.max_tokens {
6950 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
6951 break;
6952 }
6953
6954 generated.push(next_token);
6955 let delta = backend.decode(&[next_token])?;
6956 if !delta.is_empty()
6957 && tx
6958 .blocking_send(stream::StreamEvent::TextDelta(delta))
6959 .is_err()
6960 {
6961 return Ok(());
6962 }
6963
6964 if !params.stop.is_empty() {
6965 let text_so_far = backend.decode(&generated)?;
6966 if params.stop.iter().any(|s| text_so_far.contains(s)) {
6967 break;
6968 }
6969 }
6970
6971 let elapsed = started.elapsed();
6976 if deadline_exceeded(elapsed, timeout) {
6977 tracing::warn!(
6978 elapsed_secs = elapsed.as_secs_f64(),
6979 timeout_secs = timeout.map(|t| t.as_secs()),
6980 completion_tokens = generated.len(),
6981 max_tokens = params.max_tokens,
6982 "local stream hit its wall-clock ceiling and was cut short; \
6983 returning what was streamed so far. Raise or disable it with \
6984 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
6985 );
6986 let _ = tx.blocking_send(stream::StreamEvent::StopReason(
6991 LOCAL_DECODE_TIMEOUT_STOP_REASON.to_string(),
6992 ));
6993 break;
6994 }
6995 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
6996 last_heartbeat = elapsed;
6997 tracing::info!(
6998 completion_tokens = generated.len(),
6999 max_tokens = params.max_tokens,
7000 elapsed_secs = elapsed.as_secs_f64(),
7001 tokens_per_sec =
7002 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7003 "local stream in progress"
7004 );
7005 }
7006
7007 let pos = tokens.len() + generated.len() - 1;
7008 let logits =
7009 match Self::catch_mlx("stream forward", || backend.forward(&[next_token], pos)) {
7010 Ok(v) => v,
7011 Err(e) => {
7012 cache.invalidate(&model_id);
7013 return Err(e);
7014 }
7015 };
7016 next_token = Self::sample_from_logits(&logits, params)?;
7017 }
7018
7019 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
7020 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
7021 let _ = tx.blocking_send(stream::StreamEvent::Usage {
7029 input_tokens: tokens.len() as u64,
7030 output_tokens: generated.len() as u64,
7031 cache_read_input_tokens: 0,
7033 cache_creation_input_tokens: 0,
7034 });
7035 let _ = tx.blocking_send(stream::StreamEvent::Done {
7036 text,
7037 tool_calls: vec![],
7038 });
7039 Ok(())
7040 }
7041
7042 pub async fn route_context_snapshot(
7044 &self,
7045 prompt: &str,
7046 workload: RoutingWorkload,
7047 has_tools: bool,
7048 has_vision: bool,
7049 ) -> AdaptiveRoutingDecision {
7050 let routing_registry = self.catalog_registry_snapshot();
7051 let tracker = self.outcome_tracker.read().await;
7052 self.adaptive_router.route_context_aware(
7053 prompt,
7054 0,
7055 &routing_registry,
7056 &tracker,
7057 has_tools,
7058 has_vision,
7059 workload,
7060 )
7061 }
7062
7063 pub async fn generate(&self, req: GenerateRequest) -> Result<String, InferenceError> {
7066 Ok(self.generate_tracked(req).await?.text)
7067 }
7068
7069 pub async fn tokenize(&self, model: &str, text: &str) -> Result<Vec<u32>, InferenceError> {
7081 self.assert_local_for_tokenize(model)?;
7082 let admission_schema = self
7083 .resolve_schema(model)
7084 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7085 .clone();
7086 let mut reservation =
7088 self.reserve_local_request(&admission_schema, text.len().div_ceil(4))?;
7089
7090 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7091 {
7092 let schema = admission_schema;
7093 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7094 let guard = handle.lock().map_err(|_| {
7095 InferenceError::InferenceFailed(format!(
7096 "MLX backend mutex poisoned for {}",
7097 schema.id
7098 ))
7099 })?;
7100 let tokens = guard.tokenize_raw(text)?;
7101 Ok(tokens)
7102 }
7103
7104 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7105 {
7106 self.ensure_backend(&admission_schema, &mut reservation)
7107 .await?;
7108 let read = self.backend.read().await;
7109 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7110 InferenceError::InferenceFailed(
7111 "candle backend missing after ensure_backend".to_string(),
7112 )
7113 })?;
7114 let tokens = backend.tokenize_raw(text)?;
7115 Ok(tokens)
7116 }
7117 }
7118
7119 pub async fn detokenize(&self, model: &str, tokens: &[u32]) -> Result<String, InferenceError> {
7121 self.assert_local_for_tokenize(model)?;
7122 let admission_schema = self
7123 .resolve_schema(model)
7124 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7125 .clone();
7126 let mut reservation = self.reserve_local_request(&admission_schema, tokens.len())?;
7128
7129 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7130 {
7131 let schema = admission_schema;
7132 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7133 let guard = handle.lock().map_err(|_| {
7134 InferenceError::InferenceFailed(format!(
7135 "MLX backend mutex poisoned for {}",
7136 schema.id
7137 ))
7138 })?;
7139 let text = guard.detokenize_raw(tokens)?;
7140 Ok(text)
7141 }
7142
7143 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7144 {
7145 self.ensure_backend(&admission_schema, &mut reservation)
7146 .await?;
7147 let read = self.backend.read().await;
7148 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7149 InferenceError::InferenceFailed(
7150 "candle backend missing after ensure_backend".to_string(),
7151 )
7152 })?;
7153 let text = backend.detokenize_raw(tokens)?;
7154 Ok(text)
7155 }
7156 }
7157
7158 fn assert_local_for_tokenize(&self, model: &str) -> Result<(), InferenceError> {
7162 if let Some(schema) = self.resolve_schema(model) {
7163 if !schema.is_local() {
7164 return Err(InferenceError::UnsupportedMode {
7165 mode: "tokenize/detokenize",
7166 backend: "remote",
7167 reason: "remote provider tokenizer is not exposed by the runtime; \
7168 use a local model (Qwen3 GGUF / MLX) for tokenizer-correctness checks",
7169 });
7170 }
7171 }
7172 Ok(())
7174 }
7175
7176 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7182 fn catch_mlx<F, T>(context: &str, f: F) -> Result<T, InferenceError>
7183 where
7184 F: FnOnce() -> Result<T, InferenceError>,
7185 {
7186 std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|e| {
7187 InferenceError::InferenceFailed(format!("MLX panicked during {context}: {e:?}"))
7188 })?
7189 }
7190
7191 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7201 fn drive_generation(
7202 backend: &mut dyn backend::local::TextDecoder,
7203 prompt: &str,
7204 params: &GenerateParams,
7205 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7206 Self::drive_generation_with_timeout(backend, prompt, params, local_decode_timeout())
7207 }
7208
7209 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7213 fn drive_generation_with_timeout(
7214 backend: &mut dyn backend::local::TextDecoder,
7215 prompt: &str,
7216 params: &GenerateParams,
7217 timeout: Option<std::time::Duration>,
7218 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7219 use backend::local::{DriveError, LocalGeneration};
7220
7221 let start = std::time::Instant::now();
7222
7223 let tokens = backend.encode(prompt).map_err(DriveError::Recoverable)?;
7224 let eos_ids = backend.eos_ids();
7225
7226 if tokens.is_empty() {
7227 backend.clear_kv_cache();
7228 return Ok(LocalGeneration {
7229 text: String::new(),
7230 ttft_ms: None,
7231 stop_reason: None,
7232 prompt_tokens: 0,
7233 completion_tokens: 0,
7234 });
7235 }
7236
7237 let max_ctx = backend.context_length();
7239 let headroom = params.max_tokens.min(max_ctx / 4);
7240 let max_prompt = max_ctx.saturating_sub(headroom);
7241 let tokens = if tokens.len() > max_prompt {
7242 tokens[tokens.len() - max_prompt..].to_vec()
7243 } else {
7244 tokens
7245 };
7246
7247 let mut generated = Vec::new();
7248
7249 tracing::info!(
7256 prompt_tokens = tokens.len(),
7257 max_tokens = params.max_tokens,
7258 timeout_secs = timeout.map(|t| t.as_secs()),
7259 "local prefill starting"
7260 );
7261
7262 let offset = backend.begin_prompt(&tokens);
7267 let logits = Self::catch_mlx("prefill", || backend.forward(&tokens[offset..], offset))
7268 .map_err(DriveError::BackendCorrupted)?;
7269 let mut next_token =
7270 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7271 let ttft_ms = Some(start.elapsed().as_millis() as u64);
7272
7273 tracing::info!(
7278 prompt_tokens = tokens.len(),
7279 max_tokens = params.max_tokens,
7280 prefill_ms = ttft_ms,
7281 timeout_secs = timeout.map(|t| t.as_secs()),
7282 "local decode starting"
7283 );
7284
7285 let mut natural_stop = false;
7288 let mut timed_out = false;
7289 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
7290 let mut last_heartbeat = std::time::Duration::ZERO;
7291 for _ in 0..params.max_tokens {
7292 if eos_ids.contains(&next_token) {
7293 natural_stop = true;
7294 break;
7295 }
7296
7297 generated.push(next_token);
7298
7299 if !params.stop.is_empty() {
7300 let text_so_far = backend
7301 .decode(&generated)
7302 .map_err(DriveError::Recoverable)?;
7303 if params.stop.iter().any(|s| text_so_far.contains(s)) {
7304 natural_stop = true;
7305 break;
7306 }
7307 }
7308
7309 let elapsed = start.elapsed();
7310 if deadline_exceeded(elapsed, timeout) {
7311 timed_out = true;
7312 break;
7313 }
7314 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7315 last_heartbeat = elapsed;
7316 tracing::info!(
7317 completion_tokens = generated.len(),
7318 max_tokens = params.max_tokens,
7319 elapsed_secs = elapsed.as_secs_f64(),
7320 tokens_per_sec =
7321 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7322 "local decode in progress"
7323 );
7324 }
7325
7326 let pos = tokens.len() + generated.len() - 1;
7327 let logits = Self::catch_mlx("forward", || backend.forward(&[next_token], pos))
7328 .map_err(DriveError::BackendCorrupted)?;
7329 next_token =
7330 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7331 }
7332
7333 let elapsed = start.elapsed();
7334 let tokens_per_sec = generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON);
7335 if timed_out {
7336 tracing::warn!(
7337 elapsed_secs = elapsed.as_secs_f64(),
7338 timeout_secs = timeout.map(|t| t.as_secs()),
7339 completion_tokens = generated.len(),
7340 max_tokens = params.max_tokens,
7341 tokens_per_sec,
7342 "local decode hit its wall-clock ceiling and was cut short; \
7343 returning the partial response. Raise or disable it with \
7344 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
7345 );
7346 } else {
7347 tracing::info!(
7348 elapsed_secs = elapsed.as_secs_f64(),
7349 completion_tokens = generated.len(),
7350 max_tokens = params.max_tokens,
7351 tokens_per_sec,
7352 natural_stop,
7353 "local decode finished"
7354 );
7355 }
7356
7357 let decoded = backend
7358 .decode(&generated)
7359 .map_err(DriveError::Recoverable)?;
7360 let text = tasks::generate::truncate_at_stop(&decoded, ¶ms.stop);
7361 let stop_reason = Some(
7362 if timed_out {
7363 LOCAL_DECODE_TIMEOUT_STOP_REASON
7364 } else if natural_stop {
7365 "stop"
7366 } else {
7367 "length"
7368 }
7369 .to_string(),
7370 );
7371 Ok(LocalGeneration {
7372 text: tasks::generate::strip_thinking(&text, params.thinking),
7373 ttft_ms,
7374 stop_reason,
7375 prompt_tokens: tokens.len(),
7376 completion_tokens: generated.len(),
7377 })
7378 }
7379
7380 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7383 async fn generate_mlx(
7393 &self,
7394 req: GenerateRequest,
7395 model_id: &str,
7396 reservation: &mut resource_policy::LocalLoadReservation,
7397 ) -> Result<(String, Option<TokenUsage>, Option<u64>, Option<String>), InferenceError> {
7398 let schema = self
7399 .unified_registry()
7400 .get(model_id)
7401 .cloned()
7402 .ok_or_else(|| {
7403 InferenceError::InferenceFailed(format!(
7404 "generate_mlx: unknown schema id {model_id}"
7405 ))
7406 })?;
7407 let (handle, _retention) = self.ensure_text_backend(&schema, reservation).await?;
7408 let formatted = tasks::generate::render_chat_prompt(&req);
7415 let params = req.params.clone();
7416 let model_id = model_id.to_string();
7417 let cache = self.text_backends.clone();
7418 let device_guard = Self::mlx_device_lock().lock_owned().await;
7419 let detached_lease = reservation.detached_lease();
7420
7421 run_admitted_blocking(detached_lease, move || {
7427 let _device_guard = device_guard;
7428 let mut guard = handle.lock().map_err(|_| {
7429 InferenceError::InferenceFailed(format!(
7430 "MLX backend mutex poisoned for {model_id}"
7431 ))
7432 })?;
7433 let backend: &mut backend::SwiftLmBackend = &mut guard;
7434 let ctx_window = backend.context_length() as u64;
7435 match Self::drive_generation(backend, &formatted, ¶ms) {
7436 Ok(gen) => {
7437 let usage = TokenUsage {
7438 prompt_tokens: gen.prompt_tokens as u64,
7439 completion_tokens: gen.completion_tokens as u64,
7440 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7441 context_window: ctx_window,
7442 ..Default::default()
7443 };
7444 Ok((gen.text, Some(usage), gen.ttft_ms, gen.stop_reason))
7445 }
7446 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
7447 Err(backend::local::DriveError::BackendCorrupted(error)) => {
7448 drop(guard);
7449 cache.invalidate(&model_id);
7450 Err(error)
7451 }
7452 }
7453 })
7454 .await
7455 .map_err(|error| {
7456 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
7457 })?
7458 }
7459
7460 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7465 async fn generate_local(
7466 &self,
7467 req: GenerateRequest,
7468 model_id: &str,
7469 reservation: &mut resource_policy::LocalLoadReservation,
7470 ) -> Result<
7471 (
7472 String,
7473 Vec<tasks::generate::ToolCall>,
7474 Option<TokenUsage>,
7475 Option<u64>,
7476 Option<String>,
7477 ),
7478 InferenceError,
7479 > {
7480 let schema = self
7481 .unified_registry()
7482 .get(model_id)
7483 .cloned()
7484 .ok_or_else(|| {
7485 InferenceError::InferenceFailed(format!(
7486 "generate_local: unknown schema id {model_id}"
7487 ))
7488 })?;
7489 let (handle, _retention) = self.ensure_local_backend(&schema, reservation).await?;
7490 let params = req.params.clone();
7491 let model_id = model_id.to_string();
7492 let cache = self.local_backends.clone();
7493 let device_guard = Self::mlx_device_lock().lock_owned().await;
7494 let detached_lease = reservation.detached_lease();
7495
7496 run_admitted_blocking(detached_lease, move || {
7497 let _device_guard = device_guard;
7498 let mut guard = handle.lock().map_err(|_| {
7499 InferenceError::InferenceFailed(format!(
7500 "local backend mutex poisoned for {model_id}"
7501 ))
7502 })?;
7503 let formatted = guard.render_prompt(&req)?;
7504 let backend: &mut dyn backend::local::TextDecoder = &mut **guard;
7505 let outcome = Self::drive_generation(backend, &formatted, ¶ms);
7506 match outcome {
7507 Ok(gen) => {
7508 let (clean, tool_calls) = guard.parse_tool_calls(&gen.text);
7509 let usage = TokenUsage {
7510 prompt_tokens: gen.prompt_tokens as u64,
7511 completion_tokens: gen.completion_tokens as u64,
7512 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7513 context_window: guard.context_length() as u64,
7514 ..Default::default()
7515 };
7516 Ok((clean, tool_calls, Some(usage), gen.ttft_ms, gen.stop_reason))
7517 }
7518 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
7519 Err(backend::local::DriveError::BackendCorrupted(error)) => {
7520 drop(guard);
7521 cache.invalidate(&model_id);
7522 Err(error)
7523 }
7524 }
7525 })
7526 .await
7527 .map_err(|error| {
7528 InferenceError::InferenceFailed(format!("native local task panicked: {error}"))
7529 })?
7530 }
7531
7532 #[allow(dead_code)] fn apply_top_k_top_p(probs: &mut [f32], top_k: usize, top_p: f64) {
7540 if top_k > 0 && top_k < probs.len() {
7542 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
7543 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
7544 let allowed: std::collections::HashSet<usize> =
7545 indexed[..top_k].iter().map(|(i, _)| *i).collect();
7546 for (i, p) in probs.iter_mut().enumerate() {
7547 if !allowed.contains(&i) {
7548 *p = 0.0;
7549 }
7550 }
7551 let sum: f32 = probs.iter().sum();
7552 if sum > 0.0 {
7553 for p in probs.iter_mut() {
7554 *p /= sum;
7555 }
7556 }
7557 }
7558
7559 if top_p < 1.0 {
7561 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
7562 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
7563 let mut cumsum = 0.0f32;
7564 let mut cutoff_idx = indexed.len();
7565 for (i, &(_, p)) in indexed.iter().enumerate() {
7566 cumsum += p;
7567 if cumsum > top_p as f32 {
7568 cutoff_idx = i + 1;
7569 break;
7570 }
7571 }
7572 let allowed: std::collections::HashSet<usize> =
7573 indexed[..cutoff_idx].iter().map(|(i, _)| *i).collect();
7574 for (i, p) in probs.iter_mut().enumerate() {
7575 if !allowed.contains(&i) {
7576 *p = 0.0;
7577 }
7578 }
7579 let sum: f32 = probs.iter().sum();
7580 if sum > 0.0 {
7581 for p in probs.iter_mut() {
7582 *p /= sum;
7583 }
7584 }
7585 }
7586 }
7587
7588 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7590 fn sample_from_logits(logits: &[f32], params: &GenerateParams) -> Result<u32, InferenceError> {
7591 if params.temperature <= 0.0 {
7592 let (idx, _) = logits
7594 .iter()
7595 .enumerate()
7596 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
7597 .ok_or_else(|| InferenceError::InferenceFailed("empty logits".into()))?;
7598 return Ok(idx as u32);
7599 }
7600
7601 let temp = params.temperature as f32;
7603 let max_logit = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
7604 let mut probs: Vec<f32> = logits
7605 .iter()
7606 .map(|&l| ((l - max_logit) / temp).exp())
7607 .collect();
7608 let sum: f32 = probs.iter().sum();
7609 for p in &mut probs {
7610 *p /= sum;
7611 }
7612
7613 Self::apply_top_k_top_p(&mut probs, params.top_k, params.top_p);
7619
7620 use rand::Rng;
7622 let mut rng = rand::rng();
7623 let r: f32 = rng.random();
7624 let mut cumsum = 0.0;
7625 for (i, &p) in probs.iter().enumerate() {
7626 cumsum += p;
7627 if cumsum >= r {
7628 return Ok(i as u32);
7629 }
7630 }
7631 Ok((probs.len() - 1) as u32)
7632 }
7633
7634 pub async fn embed(&self, req: EmbedRequest) -> Result<Vec<Vec<f32>>, InferenceError> {
7637 let instruction = req
7638 .instruction
7639 .as_deref()
7640 .unwrap_or("Retrieve relevant memory facts");
7641 let embedding_model = self
7642 .preferred_model_for_capability(ModelCapability::Embed)
7643 .unwrap_or(&self.config.embedding_model);
7644 let admission_schema = self
7645 .resolve_schema(embedding_model)
7646 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model.to_string()))?
7647 .clone();
7648 let estimated_tokens = req.texts.iter().map(|text| text.len().div_ceil(4)).sum();
7649 let mut reservation = self.reserve_local_request(&admission_schema, estimated_tokens)?;
7651
7652 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7653 {
7654 let model_id = self.ensure_mlx_embedding_backend().await?;
7655 let schema = self
7656 .unified_registry()
7657 .get(&model_id)
7658 .cloned()
7659 .ok_or_else(|| {
7660 InferenceError::InferenceFailed(format!("embed: unknown schema id {model_id}"))
7661 })?;
7662 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7663 let _device_guard = Self::mlx_device_lock().lock_owned().await;
7667 let mut guard = handle.lock().map_err(|_| {
7668 InferenceError::InferenceFailed(format!(
7669 "MLX embedding backend mutex poisoned for {model_id}"
7670 ))
7671 })?;
7672 let backend: &mut backend::SwiftLmBackend = &mut guard;
7673
7674 let mut results = Vec::with_capacity(req.texts.len());
7675 for text in &req.texts {
7676 let embedding = if req.is_query {
7677 backend.embed_query(text, instruction)?
7678 } else {
7679 backend.embed_one(text)?
7680 };
7681 results.push(embedding);
7682 }
7683 Ok(results)
7684 }
7685
7686 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7687 {
7688 self.ensure_embedding_backend(&mut reservation).await?;
7689 let mut write = self.embedding_backend.write().await;
7690 let backend = write.as_mut().unwrap();
7691
7692 let mut results = Vec::with_capacity(req.texts.len());
7693 for text in &req.texts {
7694 let embedding = if req.is_query {
7695 backend.embed_query(text, instruction)?
7696 } else {
7697 backend.embed_one(text)?
7698 };
7699 results.push(embedding);
7700 }
7701 Ok(results)
7702 }
7703 }
7704
7705 pub async fn rerank(&self, req: RerankRequest) -> Result<RerankResult, InferenceError> {
7739 if req.documents.is_empty() {
7740 return Ok(RerankResult {
7741 ranked: Vec::new(),
7742 model_used: None,
7743 });
7744 }
7745
7746 let model_name = match req.model.clone() {
7747 Some(m) => m,
7748 None => self
7749 .preferred_model_for_capability(ModelCapability::Rerank)
7750 .map(str::to_string)
7751 .ok_or_else(|| {
7752 InferenceError::InferenceFailed(
7753 "no reranker model available — pull a Qwen3-Reranker model first".into(),
7754 )
7755 })?,
7756 };
7757
7758 let schema = self.resolve_schema_name_first(&model_name).ok_or_else(|| {
7759 InferenceError::InferenceFailed(format!("rerank: unknown reranker model {model_name}"))
7760 })?;
7761 if !schema.has_capability(ModelCapability::Rerank) {
7762 return Err(InferenceError::InferenceFailed(format!(
7763 "model {} does not declare the Rerank capability",
7764 schema.name
7765 )));
7766 }
7767
7768 let instruction = req.instruction.as_deref().unwrap_or(
7769 "Given a web search query, retrieve relevant passages that answer the query",
7770 );
7771
7772 let mut scored: Vec<RerankedDocument> = Vec::with_capacity(req.documents.len());
7773 for (idx, doc) in req.documents.iter().enumerate() {
7774 let prompt = rerank_prompt(instruction, &req.query, doc);
7775 let gen_req = GenerateRequest {
7776 prompt,
7777 model: Some(schema.id.clone()),
7778 params: tasks::generate::GenerateParams {
7779 temperature: 0.0,
7780 max_tokens: 3,
7784 thinking: tasks::generate::ThinkingMode::Off,
7785 ..Default::default()
7786 },
7787 context: None,
7788 context_stable_prefix: None,
7789 tools: None,
7790 images: None,
7791 messages: None,
7792 cache_control: false,
7793 response_format: None,
7794 intent: None,
7795 client_ref: None,
7796 expected_row_digest: None,
7797 expected_catalog_revision: None,
7798 caller: None,
7799 work_context: None,
7800 };
7801 let out = self.generate(gen_req).await?;
7802 let score = score_from_rerank_output(&out, &schema.name);
7803 scored.push(RerankedDocument {
7804 index: idx,
7805 score,
7806 document: doc.clone(),
7807 });
7808 }
7809
7810 scored.sort_by(|a, b| {
7813 b.score
7814 .partial_cmp(&a.score)
7815 .unwrap_or(std::cmp::Ordering::Equal)
7816 .then_with(|| a.index.cmp(&b.index))
7817 });
7818 if let Some(n) = req.top_n {
7819 scored.truncate(n);
7820 }
7821
7822 Ok(RerankResult {
7823 ranked: scored,
7824 model_used: Some(schema.name),
7825 })
7826 }
7827
7828 pub async fn ground(&self, req: GroundRequest) -> Result<GroundResult, InferenceError> {
7838 let model_name = match req.model.clone() {
7839 Some(m) => m,
7840 None => self
7841 .preferred_model_for_capability(ModelCapability::Grounding)
7842 .map(str::to_string)
7843 .ok_or_else(|| {
7844 InferenceError::InferenceFailed(
7845 "no grounding-capable model available — pull a Qwen2.5-VL model first"
7846 .into(),
7847 )
7848 })?,
7849 };
7850
7851 let gen_req = GenerateRequest {
7852 prompt: req.prompt.clone(),
7853 model: Some(model_name),
7854 params: GenerateParams::default(),
7855 context: None,
7856 context_stable_prefix: None,
7857 tools: None,
7858 images: Some(vec![req.image.clone()]),
7859 messages: None,
7860 cache_control: false,
7861 response_format: None,
7862 intent: None,
7863 client_ref: None,
7864 expected_row_digest: None,
7865 expected_catalog_revision: None,
7866 caller: None,
7867 work_context: None,
7868 };
7869 let result = self.generate_tracked(gen_req).await?;
7870 Ok(GroundResult {
7871 boxes: result.bounding_boxes,
7872 raw_text: result.text,
7873 model_used: Some(result.model_used),
7874 })
7875 }
7876
7877 pub async fn classify(
7880 &self,
7881 req: ClassifyRequest,
7882 ) -> Result<Vec<ClassifyResult>, InferenceError> {
7883 let registry = if let Some(context) = &req.work_context {
7884 Arc::new(
7885 self.work_context_routing_snapshot(context, req.model.as_deref())
7886 .await
7887 .0,
7888 )
7889 } else {
7890 self.unified_registry()
7891 };
7892 let model = match req.model.clone().or_else(|| {
7893 self.preferred_model_for_capability(ModelCapability::Classify)
7894 .map(str::to_string)
7895 }) {
7896 Some(m) => m,
7897 None => match default_system_one_classifier(registry.all()) {
7901 Some(m) => {
7902 debug!(model = %m, "classify defaulted to an available System One model");
7903 m
7904 }
7905 None => {
7906 let m = self.router.route_small(&self.registry);
7907 debug!(model = %m, "auto-routed classify request");
7908 m
7909 }
7910 },
7911 };
7912 let schema = self
7913 .resolve_schema(&model)
7914 .ok_or_else(|| InferenceError::ModelNotFound(model.clone()))?
7915 .clone();
7916
7917 if let ModelSource::Proprietary { protocol, .. } = &schema.source {
7918 if protocol.wire == ProprietaryWire::SystemOne {
7919 return self.classify_via_system_one(req, &schema).await;
7920 }
7921 }
7922 if !schema.is_local() {
7923 return self.classify_via_generate(req, &schema.id).await;
7924 }
7925
7926 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7929 {
7930 return self.classify_via_generate(req, &model).await;
7931 }
7932
7933 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7934 {
7935 let mut reservation =
7937 self.reserve_local_request(&schema, req.text.len().div_ceil(4))?;
7938 self.ensure_backend(&schema, &mut reservation).await?;
7939 let mut write = self.backend.write().await;
7940 let backend = write.get_mut(&schema.id).ok_or_else(|| {
7941 InferenceError::InferenceFailed(format!(
7942 "candle backend missing after ensure_backend for {}",
7943 schema.id
7944 ))
7945 })?;
7946 let result = tasks::classify::classify(backend, req).await?;
7947 Ok(result)
7948 }
7949 }
7950
7951 fn refuse_system_one_generate(&self, req: &GenerateRequest) -> Result<(), InferenceError> {
7960 let Some(model) = req.model.as_deref() else {
7961 return Ok(());
7962 };
7963 let model = model.strip_prefix(EXACT_MODEL_ID_PREFIX).unwrap_or(model);
7964 let schema = self.resolve_schema(model);
7965 match schema.as_ref().map(|s| &s.source) {
7966 Some(ModelSource::Proprietary { protocol, .. })
7967 if protocol.wire == ProprietaryWire::SystemOne =>
7968 {
7969 Err(InferenceError::UnsupportedMode {
7970 mode: "generate",
7971 backend: "parslee-system-one",
7972 reason: "a System One model returns typed decisions, not text; use classify",
7973 })
7974 }
7975 _ => Ok(()),
7976 }
7977 }
7978
7979 pub async fn option_probabilities(
7998 &self,
7999 model: &str,
8000 prompt: &str,
8001 options: &[String],
8002 ) -> Result<OptionProbabilities, InferenceError> {
8003 if options.is_empty() {
8004 return Err(InferenceError::InvalidClassifyLabels(
8005 "no options given".into(),
8006 ));
8007 }
8008 let schema = self
8009 .resolve_schema(model)
8010 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
8011 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8012 {
8013 let _ = (schema, prompt);
8014 Err(InferenceError::UnsupportedMode {
8015 mode: "option-logits",
8016 backend: "local",
8017 reason: "option probabilities read logits from the native MLX text backend, \
8018 which is Apple Silicon only",
8019 })
8020 }
8021 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8022 {
8023 if !schema.is_local() || !schema.family.to_lowercase().starts_with("qwen") {
8024 return Err(InferenceError::UnsupportedMode {
8025 mode: "option-logits",
8026 backend: "local",
8027 reason: "option probabilities need a local Qwen-family model: the prompt is \
8028 rendered with Qwen's chat template",
8029 });
8030 }
8031 let req = GenerateRequest {
8032 prompt: prompt.to_string(),
8033 model: Some(schema.id.clone()),
8034 params: tasks::generate::GenerateParams {
8035 thinking: tasks::generate::ThinkingMode::Off,
8036 ..Default::default()
8037 },
8038 ..Default::default()
8039 };
8040 let formatted = tasks::generate::render_chat_prompt(&req);
8041 let mut reservation =
8042 self.reserve_local_request(&schema, formatted.len().div_ceil(4))?;
8043 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
8044 let options = options.to_vec();
8045 let model_id = schema.id.clone();
8046 let device_guard = Self::mlx_device_lock().lock_owned().await;
8047 let detached_lease = reservation.detached_lease();
8048 run_admitted_blocking(detached_lease, move || {
8049 let _device_guard = device_guard;
8050 let mut guard = handle.lock().map_err(|_| {
8051 InferenceError::InferenceFailed(format!(
8052 "MLX backend mutex poisoned for {model_id}"
8053 ))
8054 })?;
8055 let backend: &mut backend::SwiftLmBackend = &mut guard;
8056 let result = tasks::classify::score_options(backend, &formatted, &options);
8057 backend.clear_kv_cache();
8058 result
8059 })
8060 .await
8061 .map_err(|error| {
8062 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
8063 })?
8064 }
8065 }
8066
8067 async fn classify_via_system_one(
8072 &self,
8073 req: ClassifyRequest,
8074 schema: &ModelSchema,
8075 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8076 let body = tasks::classify::system_one_request_body(&schema.name, &req.text, &req.labels)?;
8077 let trace = {
8080 let mut tracker = self.outcome_tracker.write().await;
8081 tracker.record_start(
8082 &schema.id,
8083 InferenceTask::Classify,
8084 "explicit system-one model",
8085 )
8086 };
8087 let start = Instant::now();
8088 let result = match self
8089 .remote_backend
8090 .with_work_context(req.work_context.clone())
8091 .parslee_system_one(schema, &body)
8092 .await
8093 {
8094 Ok(response) => tasks::classify::system_one_results(&response, &req.labels)
8095 .map(|results| (results, response)),
8096 Err(error) => Err(error),
8097 };
8098 let mut tracker = self.outcome_tracker.write().await;
8099 match result {
8100 Ok((results, response)) => {
8101 let usage = |key: &str| {
8102 response
8103 .pointer(&format!("/usage/{key}"))
8104 .and_then(|t| t.as_u64())
8105 .unwrap_or(0) as usize
8106 };
8107 tracker.record_complete(
8110 &trace,
8111 start.elapsed().as_millis() as u64,
8112 usage("input_tokens"),
8113 usage("output_tokens").max(1),
8114 );
8115 Ok(results)
8116 }
8117 Err(error) => {
8118 record_dispatch_failure(&mut tracker, &trace, &error);
8119 Err(error)
8120 }
8121 }
8122 }
8123
8124 async fn classify_via_generate(
8125 &self,
8126 req: ClassifyRequest,
8127 model: &str,
8128 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8129 let labels_str = req
8130 .labels
8131 .iter()
8132 .enumerate()
8133 .map(|(i, l)| format!("{}. {}", i + 1, l))
8134 .collect::<Vec<_>>()
8135 .join("\n");
8136
8137 let prompt = format!(
8138 "Classify the following text into one of these categories:\n\
8139 {labels_str}\n\n\
8140 Text: {}\n\n\
8141 Respond with ONLY the category name, nothing else.",
8142 req.text
8143 );
8144
8145 let gen_req = GenerateRequest {
8146 prompt,
8147 model: Some(model.to_string()),
8148 params: tasks::generate::GenerateParams {
8149 temperature: 0.0,
8150 max_tokens: 32,
8151 thinking: tasks::generate::ThinkingMode::Off,
8154 ..Default::default()
8155 },
8156 context: None,
8157 context_stable_prefix: None,
8158 tools: None,
8159 images: None,
8160 messages: None,
8161 cache_control: false,
8162 response_format: None,
8163 intent: None,
8164 client_ref: None,
8165 expected_row_digest: None,
8166 expected_catalog_revision: None,
8167 caller: None,
8168 work_context: req.work_context.clone(),
8169 };
8170
8171 let response = self.generate(gen_req).await?;
8172 tasks::classify::score_reply(&response, &req.labels)
8173 }
8174
8175 pub async fn transcribe(
8177 &self,
8178 req: TranscribeRequest,
8179 ) -> Result<TranscribeResult, InferenceError> {
8180 let candidates =
8181 self.speech_candidates(ModelCapability::SpeechToText, req.model.as_deref())?;
8182 let mut last_error = None;
8183 let mut local_resource_block = None;
8184
8185 for schema in candidates {
8186 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8188 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8189 SpeechCandidateAdmission::SkipBlocked(error) => {
8190 local_resource_block = Some(error.to_string());
8191 last_error = Some(error);
8192 continue;
8193 }
8194 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8195 };
8196 let result = match &schema.source {
8197 ModelSource::Mlx { .. } => {
8198 self.transcribe_local_mlx(&schema, &req, reservation.as_mut())
8199 .await
8200 }
8201 ModelSource::WhisperCpp { model } => {
8202 self.transcribe_whisper(&schema, model, &req, reservation.as_mut())
8203 .await
8204 }
8205 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8206 self.transcribe_elevenlabs(&schema, &req).await
8207 }
8208 _ => Err(InferenceError::InferenceFailed(format!(
8209 "speech-to-text not implemented for model source: {}",
8210 schema.id
8211 ))),
8212 };
8213
8214 match result {
8215 Ok(mut result) => {
8216 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8217 if let Some(reason) = local_resource_block.take() {
8218 result.routing_explanation = Some(format!(
8219 "Local speech recognition was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8220 ));
8221 }
8222 }
8223 return Ok(result);
8224 }
8225 Err(err) if req.model.is_some() => return Err(err),
8226 Err(err) => last_error = Some(err),
8227 }
8228 }
8229
8230 Err(last_error.unwrap_or_else(|| {
8231 InferenceError::InferenceFailed("no speech-to-text models available".into())
8232 }))
8233 }
8234
8235 pub async fn synthesize(
8237 &self,
8238 req: SynthesizeRequest,
8239 ) -> Result<SynthesizeResult, InferenceError> {
8240 let candidates =
8241 self.speech_candidates(ModelCapability::TextToSpeech, req.model.as_deref())?;
8242 let mut last_error = None;
8243 let mut local_resource_block = None;
8244
8245 for schema in candidates {
8246 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8248 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8249 SpeechCandidateAdmission::SkipBlocked(error) => {
8250 local_resource_block = Some(error.to_string());
8251 last_error = Some(error);
8252 continue;
8253 }
8254 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8255 };
8256 let result = match &schema.source {
8257 ModelSource::Mlx { .. } => {
8258 self.synthesize_local_mlx(&schema, &req, reservation.as_mut())
8259 .await
8260 }
8261 ModelSource::WindowsSpeech {} => {
8262 self.synthesize_windows_speech(&schema, &req).await
8263 }
8264 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8265 self.synthesize_elevenlabs(&schema, &req).await
8266 }
8267 _ => Err(InferenceError::InferenceFailed(format!(
8268 "text-to-speech not implemented for model source: {}",
8269 schema.id
8270 ))),
8271 };
8272
8273 match result {
8274 Ok(mut result) => {
8275 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8276 if let Some(reason) = local_resource_block.take() {
8277 result.routing_explanation = Some(format!(
8278 "Local speech synthesis was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8279 ));
8280 }
8281 }
8282 return Ok(result);
8283 }
8284 Err(err) if req.model.is_some() => return Err(err),
8285 Err(err) => last_error = Some(err),
8286 }
8287 }
8288
8289 Err(last_error.unwrap_or_else(|| {
8290 InferenceError::InferenceFailed("no text-to-speech models available".into())
8291 }))
8292 }
8293
8294 async fn synthesize_windows_speech(
8300 &self,
8301 schema: &ModelSchema,
8302 req: &SynthesizeRequest,
8303 ) -> Result<SynthesizeResult, InferenceError> {
8304 #[cfg(target_os = "windows")]
8305 {
8306 let text = req.text.clone();
8307 let voice = req.voice.clone().unwrap_or_default();
8308 let rate = req.speed.unwrap_or(1.0) as f64;
8309 let bytes =
8310 tokio::task::spawn_blocking(move || winrt_synthesize_wav(&text, &voice, rate))
8311 .await
8312 .map_err(|e| {
8313 InferenceError::InferenceFailed(format!("winrt tts join: {e}"))
8314 })??;
8315 let dest = requested_or_temp_output(req.output_path.as_deref(), "wav")?;
8316 ensure_parent_dir(&dest)?;
8317 std::fs::write(&dest, &bytes)?;
8318 Ok(SynthesizeResult {
8319 audio_path: dest.to_string_lossy().to_string(),
8320 media_type: "audio/wav".to_string(),
8321 model_used: Some(schema.name.clone()),
8322 voice_used: req.voice.clone(),
8323 routing_explanation: None,
8324 })
8325 }
8326 #[cfg(not(target_os = "windows"))]
8327 {
8328 let _ = (schema, req);
8329 Err(InferenceError::InferenceFailed(
8330 "Windows OS TTS is only available on Windows".into(),
8331 ))
8332 }
8333 }
8334
8335 pub async fn generate_image(
8337 &self,
8338 req: GenerateImageRequest,
8339 ) -> Result<GenerateImageResult, InferenceError> {
8340 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8357 {
8358 use crate::backend::external_flux;
8359 tracing::info!(
8360 model = req.model.as_deref().unwrap_or("<default>"),
8361 "routing image generation to mflux"
8362 );
8363 let schema = self
8364 .media_generation_candidates(
8365 ModelCapability::ImageGeneration,
8366 req.model.as_deref(),
8367 )?
8368 .into_iter()
8369 .next()
8370 .ok_or_else(|| {
8371 InferenceError::InferenceFailed("no image generation models available".into())
8372 })?;
8373 let mut reservation = self.reserve_local_request(&schema, 0)?;
8375 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8376 reservation
8377 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8378 .map_err(InferenceError::from)?;
8379 let mut req = req;
8380 req.model = self
8381 .resolve_external_hf_repo(req.model.as_deref(), ModelCapability::ImageGeneration);
8382 external_flux::generate_image(&req)
8383 }
8384
8385 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8390 {
8391 let _ = &req;
8392 Err(InferenceError::InferenceFailed(
8393 "local image generation requires Apple Silicon (mflux)".into(),
8394 ))
8395 }
8396 }
8397
8398 pub async fn generate_image_batch(
8414 &self,
8415 req: GenerateImageRequest,
8416 ) -> Result<Vec<GenerateImageResult>, InferenceError> {
8417 let count = req.variant_count.unwrap_or(1).max(1);
8418 if count == 1 {
8419 return self.generate_image(req).await.map(|r| vec![r]);
8420 }
8421 let base_seed = req.seed.unwrap_or(0);
8422 let mut results = Vec::with_capacity(count as usize);
8423 for i in 0..count {
8424 let mut variant_req = req.clone();
8429 variant_req.seed = Some(base_seed.wrapping_add(i as u64));
8430 variant_req.variant_count = Some(1);
8434 results.push(self.generate_image(variant_req).await?);
8435 }
8436 Ok(results)
8437 }
8438
8439 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8451 fn mlx_device_lock() -> Arc<tokio::sync::Mutex<()>> {
8452 static MLX_DEVICE_LOCK: std::sync::OnceLock<Arc<tokio::sync::Mutex<()>>> =
8453 std::sync::OnceLock::new();
8454 MLX_DEVICE_LOCK
8455 .get_or_init(|| Arc::new(tokio::sync::Mutex::new(())))
8456 .clone()
8457 }
8458
8459 pub async fn generate_video(
8461 &self,
8462 req: GenerateVideoRequest,
8463 ) -> Result<GenerateVideoResult, InferenceError> {
8464 if let Err(msg) = req.validate() {
8467 return Err(InferenceError::InferenceFailed(format!(
8468 "invalid GenerateVideoRequest: {}",
8469 msg
8470 )));
8471 }
8472 #[allow(unused_variables)]
8474 let requires_audio_conditioning = req.requires_audio_passthrough_opt_in();
8475 let candidates = self
8478 .media_generation_candidates(ModelCapability::VideoGeneration, req.model.as_deref())?;
8479 let mut last_error = None;
8480
8481 for schema in candidates {
8482 let mut reservation = self.reserve_local_request(&schema, 0)?;
8484 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8485 reservation
8486 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8487 .map_err(InferenceError::from)?;
8488 let result = match &schema.source {
8489 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8490 ModelSource::Mlx { hf_repo, .. } => {
8491 if !crate::backend::external_ltx::is_available() {
8505 return Err(InferenceError::InferenceFailed(
8506 "video generation requires the external `ltx-2-mlx` CLI on PATH"
8507 .to_string(),
8508 ));
8509 }
8510 let mut req = req.clone();
8511 req.model = Some(hf_repo.clone());
8512 crate::backend::external_ltx::generate_video(&req)
8513 }
8514 _ => Err(InferenceError::InferenceFailed(format!(
8515 "video generation not implemented for model source: {}",
8516 schema.id
8517 ))),
8518 };
8519
8520 match result {
8521 Ok(result) => return Ok(result),
8522 Err(err) if req.model.is_some() => return Err(err),
8523 Err(err) => last_error = Some(err),
8524 }
8525 }
8526
8527 Err(last_error.unwrap_or_else(|| {
8528 InferenceError::InferenceFailed("no video generation models available".into())
8529 }))
8530 }
8531
8532 pub fn response_format_rejection_reason(
8540 &self,
8541 model: &str,
8542 rf: &crate::tasks::generate::ResponseFormat,
8543 ) -> Option<String> {
8544 let registry = (*self.unified_registry()).clone();
8545 let schema = registry
8546 .list()
8547 .into_iter()
8548 .find(|s| s.id == model || s.name == model)?;
8549 match &schema.source {
8550 ModelSource::Proprietary { .. } => {
8551 Some("the Parslee gateway does not accept response_format".to_string())
8552 }
8553 ModelSource::RemoteApi { protocol, .. } => {
8554 let handler = crate::protocol::handler_for(*protocol);
8555 (!handler.supports_response_format(rf)).then(|| {
8556 format!(
8557 "the {} protocol rejects response_format",
8558 handler.protocol_name()
8559 )
8560 })
8561 }
8562 _ => None,
8563 }
8564 }
8565
8566 pub fn list_models_unified(&self) -> Vec<ModelInfo> {
8571 let hardware = HardwareInfo::detect();
8572 let policy = self.active_local_resource_policy();
8573 self.list_models_unified_for(&hardware, &policy.policy)
8574 }
8575
8576 pub fn model_fit(&self, schema: &ModelSchema) -> recommend::ModelFit {
8580 recommend::model_fit(
8581 schema,
8582 &HardwareInfo::detect(),
8583 Some(&self.active_local_resource_policy().policy),
8584 )
8585 }
8586
8587 pub fn list_models_unified_for(
8591 &self,
8592 hardware: &HardwareInfo,
8593 policy: &resource_policy::ResourcePolicy,
8594 ) -> Vec<ModelInfo> {
8595 self.list_models_unified_for_inner(hardware, policy, false)
8596 }
8597
8598 pub fn list_models_unified_for_explicit_status(&self) -> Vec<ModelInfo> {
8619 let hardware = HardwareInfo::detect();
8620 let policy = self.active_local_resource_policy();
8621 self.list_models_unified_for_inner(&hardware, &policy.policy, true)
8622 }
8623
8624 fn list_models_unified_for_inner(
8625 &self,
8626 hardware: &HardwareInfo,
8627 policy: &resource_policy::ResourcePolicy,
8628 authoritative_credentials: bool,
8629 ) -> Vec<ModelInfo> {
8630 let mut registry = (*self.unified_registry()).clone();
8631 registry.prune_missing_on_disk_models();
8632 if authoritative_credentials {
8633 registry.refresh_availability_for_explicit_status();
8634 } else {
8635 registry.refresh_availability();
8636 }
8637 registry
8638 .list()
8639 .iter()
8640 .map(|schema| {
8641 let mut info = ModelInfo::from(*schema).with_fit(recommend::model_fit(
8642 schema,
8643 hardware,
8644 Some(policy),
8645 ));
8646 if !authoritative_credentials {
8647 info.credential_required = None;
8664 } else if let Some(env) = info.credential_required.as_deref() {
8665 if registry.credential_state_is_unknown(env) {
8676 info.credential_required = None;
8677 }
8678 }
8679 if schema.downloads_weights() {
8680 let enabled = self
8681 .model_management
8682 .car_enabled(&schema.id)
8683 .unwrap_or(false);
8684 if !enabled {
8685 info.available = false;
8689 }
8690 if enabled
8691 && self
8692 .model_management
8693 .load_receipt(&schema.id)
8694 .is_ok_and(|receipt| receipt.is_none())
8695 {
8696 if let Ok(_mutation) = self.model_management.begin_mutation(&schema.id) {
8703 if let Some(path) = registry.existing_local_artifact(&schema.id) {
8704 if std::fs::symlink_metadata(&path)
8705 .is_ok_and(|metadata| metadata.file_type().is_symlink())
8706 {
8707 let generation = self
8708 .resource_policy_generation
8709 .load(std::sync::atomic::Ordering::Acquire);
8710 let _ = self.model_management.record_managed_artifact(
8711 &schema.id,
8712 model_source_identity(schema),
8713 None,
8714 generation,
8715 true,
8716 path,
8717 );
8718 }
8719 }
8720 }
8721 }
8722 let receipt = self.model_management.load_receipt(&schema.id);
8723 let directory_cleanup_unsupported =
8724 !model_management::directory_removal_supported()
8725 && receipt.as_ref().is_ok_and(|receipt| {
8726 receipt.as_ref().is_some_and(|receipt| {
8727 receipt.artifact_kind
8728 == model_management::ManagedArtifactKind::Directory
8729 })
8730 });
8731 info.car_enabled = enabled;
8732 info.can_remove = enabled
8733 && self
8734 .model_management
8735 .can_remove(&schema.id)
8736 .unwrap_or(false);
8737 info.in_use = self.local_model_in_use(&schema.id);
8738 info.management_evidence = if !enabled {
8739 Some("disabled_tombstone".into())
8740 } else if directory_cleanup_unsupported {
8741 Some("install_receipt_directory_cleanup_unsupported".into())
8742 } else if receipt.is_ok_and(|receipt| receipt.is_some()) {
8743 Some("install_receipt".into())
8744 } else if info.weights_ready {
8745 Some("shared_or_hand_installed".into())
8746 } else {
8747 None
8748 };
8749 }
8750 info
8751 })
8752 .collect()
8753 }
8754
8755 fn local_model_in_use(&self, model_id: &str) -> bool {
8759 self.local_admission.active_request_count(model_id) > 0
8760 || self.local_admission.is_resident(model_id)
8761 || self.local_admission.teardown_pending(model_id)
8762 || self.model_management.model_in_use(model_id).unwrap_or(true)
8763 }
8764
8765 pub fn retire_plan(&self, model_id: &str) -> Result<retire::RetirePlan, InferenceError> {
8769 self.retire_plan_in(model_id, &hf_cache::hub_dir())
8770 }
8771
8772 pub fn retire_plan_in(
8774 &self,
8775 model_id: &str,
8776 hub: &Path,
8777 ) -> Result<retire::RetirePlan, InferenceError> {
8778 self.retire_plan_with(model_id, hub, None)
8779 .map(|(plan, _)| plan)
8780 }
8781
8782 fn retire_plan_with(
8787 &self,
8788 model_id: &str,
8789 hub: &Path,
8790 held: Option<&std::collections::BTreeSet<String>>,
8791 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
8792 if self.unified_registry().get(model_id).is_none() {
8793 return Err(InferenceError::ModelNotFound(model_id.to_string()));
8794 }
8795 let (refreshed, owned) = self.retire_snapshot(hub);
8796 self.retire_plan_on(&refreshed, &owned, model_id, hub, held)
8797 }
8798
8799 fn retire_snapshot(&self, hub: &Path) -> (UnifiedRegistry, retire::Ownership) {
8804 let mut refreshed = (*self.unified_registry()).clone();
8805 refreshed.refresh_availability();
8806 let owned = retire::ownership(
8807 retire::catalog_ownership(),
8808 retire::recorded_files(&self.config.state_root, hub),
8809 );
8810 (refreshed, owned)
8811 }
8812
8813 fn retire_plan_on(
8815 &self,
8816 refreshed: &UnifiedRegistry,
8817 owned: &retire::Ownership,
8818 model_id: &str,
8819 hub: &Path,
8820 held: Option<&std::collections::BTreeSet<String>>,
8821 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
8822 let schema = refreshed
8823 .get(model_id)
8824 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
8825 let registry: Vec<&ModelSchema> = refreshed.all().collect();
8826 let installed = |m: &ModelSchema| {
8827 m.downloads_weights()
8828 && m.weights_ready
8829 && self.model_management.car_enabled(&m.id).unwrap_or(true)
8830 };
8831 let primary = retire::primary_whole_repo(schema);
8837 let expands = |repo: &str| {
8841 hf_cache::checked_repo_dir_in(hub, repo).is_some()
8842 && owned.get(repo) == Some(&retire::Owned::Whole)
8843 };
8844 let mut targets: Vec<&ModelSchema> = vec![schema];
8845 targets.extend(registry.iter().copied().filter(|m| {
8846 m.id != schema.id
8847 && primary.is_some_and(|repo| {
8848 retire::primary_whole_repo(m) == Some(repo) && expands(repo)
8849 })
8850 }));
8851 let mut plan = retire::plan_hub_retirement(&retire::RetireInputs {
8852 targets: &targets,
8853 registry: ®istry,
8854 installed: &installed,
8855 owned,
8856 hub,
8857 });
8858 if plan.refusals.iter().any(|r| {
8859 matches!(
8860 r,
8861 retire::RetireRefusal::NothingToRetire | retire::RetireRefusal::Unsupported { .. }
8862 )
8863 }) {
8864 return Ok((plan, std::collections::BTreeSet::new()));
8865 }
8866
8867 let models_dir = self
8871 .config
8872 .models_dir
8873 .canonicalize()
8874 .unwrap_or_else(|_| self.config.models_dir.clone());
8875 for target in &targets {
8876 let mut projections = Vec::new();
8877 if retire::single_component(&target.name) {
8878 projections.push(models_dir.join(&target.name));
8879 } else {
8880 plan.refusals.push(retire::RetireRefusal::UnsafePath {
8881 path: self.config.models_dir.join(&target.name),
8882 why: "the model's name is not a single path component".into(),
8883 });
8884 }
8885 if let Ok(adopted) = self.model_management.adopted_projection_path(&target.id) {
8886 projections.push(adopted);
8887 }
8888 let receipt_path = self
8893 .model_management
8894 .load_receipt(&target.id)
8895 .ok()
8896 .flatten()
8897 .map(|r| r.managed_path);
8898 for path in projections {
8899 if std::fs::symlink_metadata(&path).is_err()
8900 || plan.projections.iter().any(|p| p.path == path)
8901 || plan.user_content.contains(&path)
8902 {
8903 continue;
8904 }
8905 let receipted = receipt_path
8910 .as_ref()
8911 .is_some_and(|managed| retire::same_entry(managed, &path));
8912 if receipted || retire::links_only(&path) {
8913 plan.freed_bytes += retire::tree_bytes(&path);
8914 plan.projections.push(retire::Projection {
8915 path,
8916 receipt_for: receipted.then(|| target.id.clone()),
8917 });
8918 } else {
8919 plan.user_content.push(path);
8920 }
8921 }
8922 }
8923
8924 if targets.iter().any(|t| self.local_model_in_use(&t.id)) {
8925 plan.refusals.push(retire::RetireRefusal::InUse);
8926 }
8927 let repos: std::collections::HashSet<String> = targets
8931 .iter()
8932 .flat_map(|t| retire::hf_references(t))
8933 .map(|r| r.repo)
8934 .collect();
8935 let lock_keys: std::collections::BTreeSet<String> = registry
8936 .iter()
8937 .filter(|m| {
8938 targets.iter().any(|t| t.id == m.id)
8939 || retire::hf_references(m)
8940 .iter()
8941 .any(|r| repos.contains(&r.repo))
8942 })
8943 .flat_map(|m| [m.id.clone(), m.name.clone()])
8944 .collect();
8945 match held {
8946 None => {
8947 if lock_keys
8948 .iter()
8949 .any(|key| crate::download::model_lock_held(key))
8950 {
8951 plan.refusals.push(retire::RetireRefusal::Downloading);
8952 }
8953 }
8954 Some(held) => {
8955 if !lock_keys.is_subset(held) {
8956 plan.refusals.push(retire::RetireRefusal::StateChanged);
8957 }
8958 }
8959 }
8960 for default in self.lane_defaults().defaults {
8961 if targets.iter().any(|t| t.id == default.model_id) {
8962 let use_case = serde_json::to_value(default.use_case)
8963 .ok()
8964 .and_then(|v| v.as_str().map(str::to_string))
8965 .unwrap_or_default();
8966 plan.refusals
8967 .push(retire::RetireRefusal::LaneDefault { use_case });
8968 }
8969 }
8970 let protected = self.protected_models(refreshed, false);
8974 for target in &targets {
8975 if let Some(why) = protected.get(&target.id) {
8976 plan.refusals
8977 .push(retire::RetireRefusal::Protected { why: why.clone() });
8978 }
8979 }
8980 if plan.deletions.is_empty() && plan.projections.is_empty() && plan.refusals.is_empty() {
8982 plan.refusals.push(retire::RetireRefusal::NothingToRetire);
8983 }
8984 plan.seal();
8985 Ok((plan, lock_keys))
8986 }
8987
8988 pub async fn retire_model(
8994 &self,
8995 model_id: &str,
8996 dry_run: bool,
8997 expect: Option<&str>,
8998 ) -> Result<retire::RetireOutcome, InferenceError> {
8999 self.retire_model_in(model_id, &hf_cache::hub_dir(), dry_run, expect)
9000 .await
9001 }
9002
9003 pub async fn retire_model_in(
9019 &self,
9020 model_id: &str,
9021 hub: &Path,
9022 dry_run: bool,
9023 expect: Option<&str>,
9024 ) -> Result<retire::RetireOutcome, InferenceError> {
9025 let not_executed = |plan| retire::RetireOutcome {
9026 plan,
9027 executed: false,
9028 removed: Vec::new(),
9029 errors: Vec::new(),
9030 };
9031 let (first, keys) = self.retire_plan_with(model_id, hub, None)?;
9032 if dry_run || first.is_refused() {
9033 return Ok(not_executed(first));
9034 }
9035 let _concierge = self.concierge_action_lock.lock().await;
9039 let mut locks = Vec::with_capacity(keys.len());
9040 for key in &keys {
9041 match crate::download::try_acquire_model_lock(key) {
9042 Some(guard) => locks.push(guard),
9043 None => {
9044 let mut refused = first;
9045 refused.refusals.push(retire::RetireRefusal::Downloading);
9046 return Ok(not_executed(refused));
9047 }
9048 }
9049 }
9050 let (mut plan, _) = self.retire_plan_with(model_id, hub, Some(&keys))?;
9051 if expect.is_some_and(|digest| digest != plan.digest) {
9052 plan.refusals.push(retire::RetireRefusal::StateChanged);
9053 }
9054 if plan.is_refused() {
9055 return Ok(not_executed(plan));
9056 }
9057 let targets: Vec<String> = std::iter::once(plan.model_id.clone())
9058 .chain(plan.also_retires.iter().cloned())
9059 .collect();
9060 let mut maintenance = Vec::with_capacity(targets.len());
9061 for id in &targets {
9062 match self.prepare_local_model_removal(id).await {
9063 Ok(guard) => maintenance.push(guard),
9064 Err(error) => {
9065 tracing::info!(model = %id, %error, "retirement refused: the model could not be released");
9066 plan.refusals.push(retire::RetireRefusal::InUse);
9067 return Ok(not_executed(plan));
9068 }
9069 }
9070 }
9071
9072 let started_at = std::time::SystemTime::now()
9075 .duration_since(std::time::UNIX_EPOCH)
9076 .map(|d| d.as_secs())
9077 .unwrap_or(0);
9078 for id in &targets {
9079 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9080 seq: 0,
9081 kind: action_ledger::ConciergeActionKind::RetireStarted,
9082 model_id: id.clone(),
9083 use_case: None,
9084 project: None,
9085 prior_model_id: None,
9086 detail: format!(
9087 "retiring with {}; plan {}; {} bytes planned",
9088 targets.join(", "),
9089 plan.digest,
9090 plan.freed_bytes
9091 ),
9092 timestamp: started_at,
9093 });
9094 }
9095
9096 let mut removed = Vec::new();
9097 let mut errors = Vec::new();
9098
9099 for projection in &plan.projections {
9102 let result = match &projection.receipt_for {
9103 Some(id) => {
9104 let result = async {
9107 self.model_management
9108 .note_retiring(id)
9109 .map_err(|e| e.to_string())?;
9110 self.remove_model_from_car_inner(id, true)
9111 .await
9112 .map_err(|e| e.to_string())?;
9113 self.model_management
9114 .clear_tombstone(id)
9115 .map_err(|e| e.to_string())?;
9116 self.model_management.retired(id);
9117 Ok::<_, String>(vec![projection.path.clone()])
9118 }
9119 .await;
9120 result
9121 }
9122 None => retire::remove_links_only_tree(&projection.path, &self.config.models_dir),
9123 };
9124 match result {
9125 Ok(paths) => removed.extend(paths),
9126 Err(why) => {
9127 errors.push(format!("{}: {why}", projection.path.display()));
9128 break;
9129 }
9130 }
9131 }
9132
9133 let hub_canonical = hub.canonicalize().unwrap_or_else(|_| hub.to_path_buf());
9134 if errors.is_empty() {
9135 'hub: for deletion in &plan.deletions {
9136 match deletion {
9137 retire::HubDeletion::Repo { path, .. } => {
9138 let quarantine = path.with_file_name(format!(
9139 "{}{:016x}",
9140 model_management::RETIRE_QUARANTINE_PREFIX,
9141 rand::random::<u64>()
9142 ));
9143 let verify = || retire::verify_repo_dir(path, hub);
9144 match self.model_management.detach_and_remove_tree(
9145 path,
9146 &quarantine,
9147 model_id,
9148 verify,
9149 ) {
9150 Ok(()) => removed.push(path.clone()),
9151 Err(failure) => {
9152 if failure.detached {
9153 removed.push(path.clone());
9156 }
9157 errors.push(format!("{}: {}", path.display(), failure.error));
9158 break 'hub;
9159 }
9160 }
9161 }
9162 retire::HubDeletion::Files {
9163 repo,
9164 entries,
9165 blobs,
9166 ..
9167 } => {
9168 let Some(dir) = hf_cache::checked_repo_dir_in(&hub_canonical, repo) else {
9169 errors.push(format!("{repo}: not a Hub repo id"));
9170 break 'hub;
9171 };
9172 for blob in blobs {
9176 let checked =
9177 retire::verify_blob(blob, &dir.join("blobs")).and_then(|()| {
9178 model_management::unlink_checked(
9179 blob,
9180 model_management::UnlinkKind::File,
9181 )
9182 .map_err(|e| e.to_string())
9183 });
9184 match checked {
9185 Ok(()) => removed.push(blob.clone()),
9186 Err(why) => {
9187 errors.push(format!("{}: {why}", blob.display()));
9188 break 'hub;
9189 }
9190 }
9191 }
9192 let snapshots = dir.join("snapshots");
9193 for entry in entries {
9194 let checked = retire::verify_entry(entry, &snapshots).and_then(|()| {
9195 model_management::unlink_checked(
9196 entry,
9197 model_management::UnlinkKind::FileOrLink,
9198 )
9199 .map_err(|e| e.to_string())
9200 });
9201 match checked {
9202 Ok(()) => removed.push(entry.clone()),
9203 Err(why) => {
9204 errors.push(format!("{}: {why}", entry.display()));
9205 break 'hub;
9206 }
9207 }
9208 }
9209 for entry in entries {
9212 let mut at = entry.parent();
9213 while let Some(parent) = at {
9214 if !parent.starts_with(&snapshots) || parent == snapshots {
9215 break;
9216 }
9217 if std::fs::remove_dir(parent).is_err() {
9218 break;
9219 }
9220 at = parent.parent();
9221 }
9222 }
9223 retire::prune_empty_repo(&dir);
9224 }
9225 }
9226 }
9227 }
9228 drop(maintenance);
9229 drop(locks);
9230
9231 if !removed.is_empty() {
9232 let now = std::time::SystemTime::now()
9233 .duration_since(std::time::UNIX_EPOCH)
9234 .map(|d| d.as_secs())
9235 .unwrap_or(0);
9236 for id in &targets {
9237 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9238 seq: 0,
9239 kind: action_ledger::ConciergeActionKind::Retire,
9240 model_id: id.clone(),
9241 use_case: None,
9242 project: None,
9243 prior_model_id: None,
9244 detail: format!(
9245 "retired with {}; plan {}; {} bytes planned; {} paths removed{}",
9246 targets.join(", "),
9247 plan.digest,
9248 plan.freed_bytes,
9249 removed.len(),
9250 if errors.is_empty() {
9251 String::new()
9252 } else {
9253 format!("; stopped: {}", errors.join("; "))
9254 }
9255 ),
9256 timestamp: now,
9257 });
9258 }
9259 }
9260 Ok(retire::RetireOutcome {
9261 executed: !removed.is_empty(),
9262 plan,
9263 removed,
9264 errors,
9265 })
9266 }
9267
9268 pub fn set_external_pins(&self, pins: std::collections::BTreeMap<String, Vec<String>>) {
9276 *self
9277 .external_pins
9278 .write()
9279 .unwrap_or_else(std::sync::PoisonError::into_inner) = pins;
9280 }
9281
9282 pub fn start_usage_tracking(&self) {
9286 self.model_management.start_usage_tracking();
9287 self.model_management.beat();
9288 }
9289
9290 fn recent_concierge_actions(&self, now: u64) -> Vec<portfolio::RecentAction> {
9294 use action_ledger::ConciergeActionKind as Kind;
9295 let since = now.saturating_sub(30 * 24 * 60 * 60);
9296 let actions = crate::action_ledger::read_actions(&self.action_ledger_path(), 0);
9297 let mut out: Vec<portfolio::RecentAction> = Vec::new();
9298 for (i, a) in actions.iter().enumerate() {
9299 if a.timestamp < since {
9300 continue;
9301 }
9302 let (kind, undo, interrupted) = match a.kind {
9303 Kind::Retire => (
9304 "retire",
9305 (!a.model_id.starts_with("repo:"))
9306 .then(|| format!("car models pull {}", a.model_id)),
9307 false,
9308 ),
9309 Kind::RetireStarted => {
9310 let finished = actions[i + 1..]
9311 .iter()
9312 .any(|later| later.kind == Kind::Retire && later.model_id == a.model_id);
9313 if finished {
9314 continue;
9315 }
9316 ("retire", None, true)
9317 }
9318 Kind::DiscardPartials => ("discard_partials", None, false),
9319 Kind::ResourcePolicy => (
9320 "resource_policy",
9321 a.prior_model_id
9322 .as_ref()
9323 .map(|prior| format!("car models resource-policy set {prior}")),
9324 false,
9325 ),
9326 Kind::AutoUpgrade => (
9327 "upgrade",
9328 match (a.use_case, &a.prior_model_id) {
9329 (Some(use_case), _) => serde_json::to_value(use_case)
9330 .ok()
9331 .and_then(|v| v.as_str().map(str::to_owned))
9332 .map(|lane| format!("car concierge rollback {lane}")),
9333 (None, Some(prior)) => {
9334 Some(format!("keep using {prior}; nothing was switched"))
9335 }
9336 (None, None) => None,
9337 },
9338 false,
9339 ),
9340 Kind::RevokedLaneCleared => ("lane_cleared", None, false),
9343 Kind::Install | Kind::SetDefault | Kind::ClearDefault | Kind::Rollback => continue,
9344 };
9345 out.push(portfolio::RecentAction {
9346 at: a.timestamp,
9347 kind: kind.to_string(),
9348 subject: a.model_id.clone(),
9349 detail: a.detail.clone(),
9350 undo,
9351 use_case: a.use_case.and_then(|use_case| {
9352 serde_json::to_value(use_case)
9353 .ok()
9354 .and_then(|v| v.as_str().map(str::to_owned))
9355 }),
9356 prior: a.prior_model_id.clone(),
9357 interrupted,
9358 });
9359 }
9360 out.reverse();
9361 out
9362 }
9363
9364 pub async fn set_resource_policy_by_user(
9369 &self,
9370 policy: resource_policy::ResourcePolicy,
9371 ) -> Result<(), resource_policy::ResourcePolicyError> {
9372 let _concierge = self.concierge_action_lock.lock().await;
9373 let repository =
9374 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9375 repository.clear_concierge_marker()?;
9376 resource_policy::ResourcePolicyRepository::save(&repository, &policy)?;
9377 self.apply_local_resource_policy(policy);
9378 Ok(())
9379 }
9380
9381 async fn apply_one_curated_upgrade(
9390 &self,
9391 hub: &Path,
9392 nudge: &crate::nudge::NudgeState,
9393 now: u64,
9394 ) -> Result<Option<maintenance::Upgraded>, String> {
9395 let findings = self.detect_upgrades().await;
9396 self.apply_one_curated_upgrade_from(findings, hub, nudge, now)
9397 .await
9398 }
9399
9400 async fn apply_one_curated_upgrade_from(
9402 &self,
9403 findings: Vec<crate::upgrade::UpgradeFinding>,
9404 hub: &Path,
9405 nudge: &crate::nudge::NudgeState,
9406 now: u64,
9407 ) -> Result<Option<maintenance::Upgraded>, String> {
9408 use crate::upgrade::UpgradeSource;
9409 for finding in findings {
9410 let label = match finding.source {
9411 UpgradeSource::Benchmark => "benchmark upgrade",
9412 UpgradeSource::Curated | UpgradeSource::Upstream => "curated upgrade",
9413 };
9414 let key = format!("{}=>{}", finding.from_id, finding.to_id);
9415 let backing_off = self
9416 .failed_upgrades
9417 .lock()
9418 .unwrap_or_else(std::sync::PoisonError::into_inner)
9419 .get(&key)
9420 .is_some_and(|at| now.saturating_sub(*at) < 24 * 60 * 60);
9421 if finding.source == UpgradeSource::Upstream
9422 || finding.trust_tier != crate::schema::TrustTier::Curated
9423 || !finding.target_pullable
9424 || nudge.dismissed.contains(&key)
9425 || backing_off
9426 {
9427 continue;
9428 }
9429 let Some(target) = self.unified_registry().get(&finding.to_id).cloned() else {
9430 continue;
9431 };
9432 let fit = self.model_fit(&target);
9433 if !fit.platform_compatible || fit.fit == recommend::ModelFitStatus::TooBig {
9434 continue;
9435 }
9436 let lanes: Vec<crate::lane_defaults::LaneDefault> = self
9437 .lane_defaults()
9438 .defaults
9439 .into_iter()
9440 .filter(|d| d.model_id == finding.from_id)
9441 .collect();
9442 if lanes.is_empty() {
9443 continue;
9444 }
9445 let present = self
9447 .unified_registry()
9448 .existing_local_artifact(&finding.to_id)
9449 .is_some();
9450 if !present && crate::download::check_disk_space(hub, target.size_mb()).is_err() {
9451 continue;
9452 }
9453 let mut switched = Vec::new();
9454 for lane in &lanes {
9455 if let Err(e) = self
9456 .apply_concierge(lane.use_case, &finding.to_id, lane.project.clone())
9457 .await
9458 {
9459 self.failed_upgrades
9460 .lock()
9461 .unwrap_or_else(std::sync::PoisonError::into_inner)
9462 .insert(key, now);
9463 if let Some(first) = lanes.first().filter(|_| !switched.is_empty()) {
9465 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9466 seq: 0,
9467 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
9468 model_id: finding.to_id.clone(),
9469 use_case: Some(first.use_case),
9470 project: first.project.clone(),
9471 prior_model_id: Some(finding.from_id.clone()),
9472 detail: format!(
9473 "{label}, partial: switched {}; then {e}",
9474 switched.join(", ")
9475 ),
9476 timestamp: now,
9477 });
9478 }
9479 return Err(format!(
9480 "upgrading {} to {} (fetch, then switch the lane): {e}; retried in a day",
9481 finding.from_id, finding.to_id
9482 ));
9483 }
9484 switched.push(
9485 serde_json::to_value(lane.use_case)
9486 .ok()
9487 .and_then(|v| v.as_str().map(str::to_owned))
9488 .unwrap_or_default(),
9489 );
9490 }
9491 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9492 seq: 0,
9493 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
9494 model_id: finding.to_id.clone(),
9495 use_case: lanes.first().map(|l| l.use_case),
9496 project: lanes.first().and_then(|l| l.project.clone()),
9497 prior_model_id: Some(finding.from_id.clone()),
9498 detail: format!("{label}: {}", finding.reason),
9499 timestamp: now,
9500 });
9501 return Ok(Some(maintenance::Upgraded {
9502 from: finding.from_id,
9503 to: finding.to_id,
9504 lanes: switched,
9505 }));
9506 }
9507 Ok(None)
9508 }
9509
9510 async fn size_memory_policy(
9519 &self,
9520 hw: &HardwareInfo,
9521 observable_since: Option<u64>,
9522 apply: bool,
9523 now: u64,
9524 ) -> Option<maintenance::PolicySizing> {
9525 use resource_policy::{ResourcePolicy, ResourcePolicyLoadSource};
9526 let since = observable_since?;
9527 let _concierge = self.concierge_action_lock.lock().await;
9528 let repository =
9529 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9530 let loaded = repository.load_with_evidence().ok()?;
9531 let concierge = repository.concierge_choice();
9532 let user_chose = match loaded.source {
9533 ResourcePolicyLoadSource::MissingDefault => false,
9534 ResourcePolicyLoadSource::CorruptDefault => true,
9535 ResourcePolicyLoadSource::Loaded => {
9536 concierge.as_ref().map(|c| &c.policy) != Some(&loaded.policy)
9537 }
9538 };
9539 let everyday = ResourcePolicy::everyday();
9540 let local_focused = ResourcePolicy::local_focused();
9541 let fit = |m: &ModelSchema, policy: &ResourcePolicy| {
9542 recommend::model_fit(m, hw, Some(policy)).fit
9543 };
9544 let raised_at = concierge
9545 .as_ref()
9546 .filter(|c| !c.reverted && c.policy == local_focused)
9547 .map(|c| c.at);
9548 let mut evidence = maintenance::PolicyEvidence::default();
9549 let mut days = std::collections::BTreeSet::new();
9550 for e in crate::outcome::read_ledger(
9551 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9552 0,
9553 ) {
9554 if e.timestamp < since
9555 || e.attribution != Some(crate::outcome::UnattributedReason::ResourceBlocked)
9556 {
9557 continue;
9558 }
9559 let registry = self.unified_registry();
9560 let Some(m) = registry.get(&e.model_id).filter(|m| m.is_local()) else {
9561 continue;
9562 };
9563 if fit(m, &loaded.policy) == recommend::ModelFitStatus::TooBig
9568 && fit(m, &local_focused) == recommend::ModelFitStatus::Fits
9569 {
9570 evidence.ceiling_refusals += 1;
9571 days.insert(e.timestamp / (24 * 60 * 60));
9572 }
9573 if raised_at.is_some_and(|at| e.timestamp >= at)
9574 && fit(m, &everyday) == recommend::ModelFitStatus::Fits
9575 {
9576 evidence.live_refusals_since_raise += 1;
9577 }
9578 }
9579 if let Some(at) = raised_at {
9581 for e in crate::outcome::read_ledger(
9582 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9583 0,
9584 ) {
9585 let admitted_by_raise = self.unified_registry().get(&e.model_id).is_some_and(|m| {
9586 m.is_local() && fit(m, &everyday) == recommend::ModelFitStatus::TooBig
9587 });
9588 if e.timestamp < at || !admitted_by_raise {
9589 continue;
9590 }
9591 match (e.success, e.attribution) {
9592 (Some(true), _) => evidence.admitted_successes_since_raise += 1,
9593 (Some(false), None) => evidence.admitted_failures_since_raise += 1,
9594 _ => {}
9595 }
9596 }
9597 }
9598 evidence.ceiling_refusal_days = days.len() as u64;
9599 let decision = maintenance::decide_resource_policy(
9600 &loaded.policy,
9601 user_chose,
9602 concierge.as_ref(),
9603 hw.total_ram_mb,
9604 &evidence,
9605 )?;
9606 let (target, reason, reverted) = match decision {
9607 maintenance::PolicyMove::Raise(reason) => (local_focused, reason, false),
9608 maintenance::PolicyMove::Revert(reason) => (everyday, reason, true),
9609 };
9610 let mut sizing = maintenance::PolicySizing {
9611 from: loaded.policy.profile.clone(),
9612 to: target.profile.clone(),
9613 reason,
9614 applied: false,
9615 };
9616 if apply {
9617 match repository.save_concierge_choice(&target, &sizing.reason, now, reverted) {
9618 Ok(()) => {
9619 self.apply_local_resource_policy(target);
9620 sizing.applied = true;
9621 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9622 seq: 0,
9623 kind: action_ledger::ConciergeActionKind::ResourcePolicy,
9624 model_id: String::new(),
9625 use_case: None,
9626 project: None,
9627 prior_model_id: serde_json::to_value(&sizing.from)
9628 .ok()
9629 .and_then(|v| v.as_str().map(str::to_owned)),
9630 detail: format!("{:?} -> {:?}: {}", sizing.from, sizing.to, sizing.reason),
9631 timestamp: now,
9632 });
9633 }
9634 Err(error) => tracing::warn!(%error, "could not change the memory policy"),
9635 }
9636 }
9637 Some(sizing)
9638 }
9639
9640 async fn retire_orphan_repo(&self, repo: &str, hub: &Path) -> Result<u64, String> {
9647 let _concierge = self.concierge_action_lock.lock().await;
9648 let (refreshed, owned) = self.retire_snapshot(hub);
9649 let referenced: std::collections::HashSet<String> = refreshed
9650 .all()
9651 .flat_map(retire::hf_references)
9652 .map(|r| r.repo)
9653 .collect();
9654 let orphan = retire::unreferenced_repos(hub, &referenced, &owned)
9655 .into_iter()
9656 .find(|o| o.repo == repo && o.owner == retire::OrphanOwner::Car)
9657 .ok_or("no longer an orphan CAR owns")?;
9658 if retire::transfer_in_progress(&orphan.path)? {
9659 return Err("a transfer is writing into it".into());
9660 }
9661 let repo_dir = orphan
9666 .path
9667 .canonicalize()
9668 .map_err(|e| format!("cannot resolve the repo: {e}"))?;
9669 if let Some(user) = retire::path_resolving_into(&self.config.models_dir, &repo_dir) {
9670 return Err(format!("{} still resolves into it", user.display()));
9671 }
9672 for row in refreshed.all().filter(|m| m.is_local()) {
9673 let inside = refreshed
9674 .existing_local_artifact(&row.id)
9675 .and_then(|path| path.canonicalize().ok())
9676 .is_some_and(|path| path.starts_with(&repo_dir));
9677 if inside {
9678 return Err(format!("the model {} still loads from it", row.id));
9679 }
9680 }
9681 let label = format!("repo:{repo}");
9682 let now = std::time::SystemTime::now()
9683 .duration_since(std::time::UNIX_EPOCH)
9684 .map(|d| d.as_secs())
9685 .unwrap_or(0);
9686 let entry = |kind, detail: String| action_ledger::ConciergeActionEntry {
9687 seq: 0,
9688 kind,
9689 model_id: label.clone(),
9690 use_case: None,
9691 project: None,
9692 prior_model_id: None,
9693 detail,
9694 timestamp: now,
9695 };
9696 self.record_concierge_action(entry(
9697 action_ledger::ConciergeActionKind::RetireStarted,
9698 format!("orphaned repo; {} bytes planned", orphan.bytes),
9699 ));
9700 let quarantine = orphan.path.with_file_name(format!(
9701 "{}{:016x}",
9702 model_management::RETIRE_QUARANTINE_PREFIX,
9703 rand::random::<u64>()
9704 ));
9705 let verify = || retire::verify_repo_dir(&orphan.path, hub);
9706 self.model_management
9707 .detach_and_remove_tree(&orphan.path, &quarantine, &label, verify)
9708 .map_err(|failure| failure.error.to_string())?;
9709 self.record_concierge_action(entry(
9710 action_ledger::ConciergeActionKind::Retire,
9711 format!("orphaned repo no model uses; {} bytes", orphan.bytes),
9712 ));
9713 Ok(orphan.bytes)
9714 }
9715
9716 pub fn usage_heartbeat(&self) {
9720 self.model_management.beat();
9721 }
9722
9723 pub fn effective_update_policy(
9730 &self,
9731 prefs: &crate::update_prefs::UpdatePreferences,
9732 start_clock: bool,
9733 ) -> (crate::update_prefs::UpdatePolicy, Option<u64>) {
9734 use crate::update_prefs::UpdatePolicy;
9735 if prefs.policy != UpdatePolicy::Auto || prefs.policy_chosen {
9736 return (prefs.policy, None);
9737 }
9738 let marker = self.config.state_root.join(AUTO_GRACE_FILE);
9739 let now = std::time::SystemTime::now()
9740 .duration_since(std::time::UNIX_EPOCH)
9741 .map(|d| d.as_secs())
9742 .unwrap_or(0);
9743 let since = std::fs::read_to_string(&marker)
9744 .ok()
9745 .and_then(|text| text.trim().parse::<u64>().ok())
9746 .unwrap_or_else(|| {
9747 if start_clock {
9748 let _ = std::fs::create_dir_all(&self.config.state_root);
9749 let _ = std::fs::write(&marker, now.to_string());
9750 }
9751 now
9752 });
9753 match maintenance::auto_grace_until(since, now, AUTO_GRACE_SECS) {
9754 Some(until) => (UpdatePolicy::Notify, Some(until)),
9755 None => (UpdatePolicy::Auto, None),
9756 }
9757 }
9758
9759 pub async fn maintenance_pass(&self, dry_run: bool) -> maintenance::MaintenanceReport {
9768 self.maintenance_pass_in(&hf_cache::hub_dir(), &HardwareInfo::detect(), dry_run)
9769 .await
9770 }
9771
9772 pub async fn maintenance_pass_in(
9774 &self,
9775 hub: &Path,
9776 hw: &HardwareInfo,
9777 dry_run: bool,
9778 ) -> maintenance::MaintenanceReport {
9779 let prefs = self.update_prefs();
9780 let (policy, grace_until) = self.effective_update_policy(&prefs, !dry_run);
9781 let nudge = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
9782 let disk = crate::download::disk_space(hub).map(|(free_bytes, total_bytes)| {
9783 maintenance::DiskSpace {
9784 free_bytes,
9785 total_bytes,
9786 }
9787 });
9788 let mut report = self
9789 .maintenance_pass_with(hub, hw, dry_run, policy, &nudge, disk)
9790 .await;
9791 report.auto_grace_until = grace_until;
9792 report
9793 }
9794
9795 pub async fn maintenance_pass_with(
9798 &self,
9799 hub: &Path,
9800 hw: &HardwareInfo,
9801 dry_run: bool,
9802 policy: crate::update_prefs::UpdatePolicy,
9803 nudge: &crate::nudge::NudgeState,
9804 disk: Option<maintenance::DiskSpace>,
9805 ) -> maintenance::MaintenanceReport {
9806 if !dry_run {
9807 self.enforce_catalog_revocations().await;
9808 }
9809 let mut report = maintenance::MaintenanceReport {
9810 policy,
9811 dry_run,
9812 ..Default::default()
9813 };
9814 if policy == crate::update_prefs::UpdatePolicy::Off {
9815 return report;
9816 }
9817 if !dry_run {
9818 report.errors.extend(self.resume_retirements().await);
9819 }
9820 let portfolio = self.portfolio_in(hub, hw).await;
9821 let now = portfolio.policy.now;
9822 report.disk = disk;
9823 let was_pressured = self
9824 .disk_pressured
9825 .load(std::sync::atomic::Ordering::Relaxed);
9826 report.pressured = maintenance::disk_pressure(report.disk, was_pressured);
9827 if !dry_run {
9828 self.disk_pressured
9829 .store(report.pressured, std::sync::atomic::Ordering::Relaxed);
9830 }
9831 let keep = maintenance::kept_models(nudge, now);
9832 let churned = crate::action_ledger::read_actions(&self.action_ledger_path(), 0)
9835 .into_iter()
9836 .filter(|a| {
9837 matches!(
9838 a.kind,
9839 action_ledger::ConciergeActionKind::Retire
9840 | action_ledger::ConciergeActionKind::RetireStarted
9841 )
9842 })
9843 .map(|a| a.model_id)
9844 .collect();
9845 let ctx = maintenance::MaintenanceContext {
9846 policy,
9847 pressured: report.pressured,
9848 keep,
9849 churned,
9850 superseded_unpressured_secs:
9851 maintenance::MaintenanceContext::DEFAULT_SUPERSEDED_UNPRESSURED_SECS,
9852 };
9853 report.plan = maintenance::decide(&portfolio, &ctx);
9854 report.portfolio = Some(portfolio.clone());
9855 report.resource_policy = self
9856 .size_memory_policy(
9857 hw,
9858 portfolio.policy.tracking_since,
9859 !dry_run && policy == crate::update_prefs::UpdatePolicy::Auto,
9860 now,
9861 )
9862 .await;
9863 if dry_run || policy != crate::update_prefs::UpdatePolicy::Auto {
9864 return report;
9865 }
9866 if let Some(act) = &report.plan.act {
9867 match self
9868 .retire_model_in(&act.model_id, hub, false, Some(&act.digest))
9869 .await
9870 {
9871 Ok(outcome) => {
9872 report.retired = outcome.executed.then(|| maintenance::Retired {
9873 model_id: act.model_id.clone(),
9874 also_retired: outcome.plan.also_retires.clone(),
9875 freed_bytes: outcome.plan.freed_bytes,
9876 restore: format!("car models pull {}", act.model_id),
9877 });
9878 if !outcome.executed {
9879 report.errors.push(format!(
9882 "{}: not retired this pass ({:?})",
9883 act.model_id, outcome.plan.refusals
9884 ));
9885 }
9886 report.errors.extend(outcome.errors);
9887 }
9888 Err(error) => report.errors.push(format!("{}: {error}", act.model_id)),
9889 }
9890 }
9891 if report.retired.is_none() && report.pressured {
9894 let mut orphans: Vec<_> = portfolio
9897 .orphans
9898 .iter()
9899 .filter(|o| o.owner == retire::OrphanOwner::Car)
9900 .collect();
9901 orphans.sort_by(|a, b| b.bytes.cmp(&a.bytes));
9902 for orphan in orphans {
9903 match self.retire_orphan_repo(&orphan.repo, hub).await {
9904 Ok(freed_bytes) => {
9905 report.orphan_retired = Some(maintenance::RetiredOrphan {
9906 repo: orphan.repo.clone(),
9907 freed_bytes,
9908 });
9909 break;
9910 }
9911 Err(error) => report.errors.push(format!("{}: {error}", orphan.repo)),
9912 }
9913 }
9914 }
9915 if report.retired.is_none() && report.orphan_retired.is_none() && !report.pressured {
9917 match self.apply_one_curated_upgrade(hub, nudge, now).await {
9918 Ok(upgraded) => report.upgraded = upgraded,
9919 Err(error) => report.errors.push(error),
9920 }
9921 }
9922 let owned = retire::ownership(
9923 retire::catalog_ownership(),
9924 retire::recorded_files(&self.config.state_root, hub),
9925 );
9926 let (discarded, errors) = retire::discard_abandoned_partials(hub, &owned);
9927 report.errors.extend(errors);
9928 if !discarded.is_empty() {
9929 let bytes: u64 = discarded.iter().map(|(_, b)| b).sum();
9930 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9931 seq: 0,
9932 kind: action_ledger::ConciergeActionKind::DiscardPartials,
9933 model_id: String::new(),
9934 use_case: None,
9935 project: None,
9936 prior_model_id: None,
9937 detail: format!(
9938 "{} abandoned partial downloads, {bytes} bytes",
9939 discarded.len()
9940 ),
9941 timestamp: now,
9942 });
9943 report.partials_discarded = discarded.into_iter().map(|(path, _)| path).collect();
9944 }
9945 report
9946 }
9947
9948 pub async fn portfolio(&self) -> portfolio::Portfolio {
9951 self.portfolio_in(&hf_cache::hub_dir(), &HardwareInfo::detect())
9952 .await
9953 }
9954
9955 pub async fn portfolio_in(&self, hub: &Path, hw: &HardwareInfo) -> portfolio::Portfolio {
9957 let now = std::time::SystemTime::now()
9958 .duration_since(std::time::UNIX_EPOCH)
9959 .map(|d| d.as_secs())
9960 .unwrap_or(0);
9961 let observable_since = self
9964 .model_management
9965 .usage_tracking_since()
9966 .zip(self.model_management.usage_covered_since())
9967 .map(|(tracking, covered)| tracking.max(covered));
9968 let policy = portfolio::PortfolioPolicy::new(now, observable_since);
9969 let (refreshed, owned) = self.retire_snapshot(hub);
9970 let protected = self.protected_models(&refreshed, true);
9971 let mut ledger_last: std::collections::HashMap<String, u64> =
9972 std::collections::HashMap::new();
9973 let mut ledger_failures: std::collections::HashMap<String, Vec<u64>> =
9980 std::collections::HashMap::new();
9981 for entry in crate::outcome::read_ledger(
9982 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9983 0,
9984 ) {
9985 if entry.counts_against_model() {
9986 ledger_failures
9987 .entry(entry.model_id.clone())
9988 .or_default()
9989 .push(entry.timestamp);
9990 }
9991 let last = ledger_last.entry(entry.model_id).or_default();
9992 *last = (*last).max(entry.timestamp);
9993 }
9994 let installed_at = |m: &ModelSchema| -> Option<u64> {
9997 let repo = retire::hf_references(m).into_iter().next()?.repo;
9998 let snapshots = hf_cache::checked_repo_dir_in(hub, &repo)?.join("snapshots");
9999 std::fs::read_dir(snapshots)
10000 .ok()?
10001 .filter_map(Result::ok)
10002 .filter_map(|e| e.metadata().ok()?.modified().ok())
10003 .filter_map(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
10004 .map(|d| d.as_secs())
10005 .max()
10006 };
10007 let installed = |m: &ModelSchema| {
10008 m.is_local()
10009 && m.downloads_weights()
10010 && m.weights_ready
10011 && self.model_management.car_enabled(&m.id).unwrap_or(true)
10012 };
10013 let runnable = |m: &ModelSchema| {
10014 let fit = recommend::model_fit(m, hw, None);
10015 fit.platform_compatible && fit.fit != recommend::ModelFitStatus::TooBig
10016 };
10017 let outcomes: std::collections::HashMap<String, (u64, u64)> = {
10019 let tracker = self.outcome_tracker.read().await;
10020 refreshed
10021 .all()
10022 .filter(|m| installed(m))
10023 .filter_map(|m| {
10024 tracker
10025 .profile(&m.id)
10026 .map(|p| (m.id.clone(), (p.success_count, p.fail_count)))
10027 })
10028 .collect()
10029 };
10030 let has_run = |id: &str| outcomes.get(id).is_some_and(|(success, _)| *success > 0);
10038 let dismissed =
10040 crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path())
10041 .dismissed;
10042 let superseded: std::collections::HashMap<String, String> = refreshed
10043 .available_upgrades()
10044 .into_iter()
10045 .filter(|u| !dismissed.contains(&format!("{}=>{}", u.from_id, u.to_id)))
10046 .filter(|u| {
10047 refreshed
10048 .get(&u.to_id)
10049 .is_some_and(|to| installed(to) && runnable(to) && has_run(&to.id))
10050 })
10051 .map(|u| (u.from_id, u.to_id))
10052 .collect();
10053 let superseded: std::collections::HashMap<String, String> = superseded
10055 .iter()
10056 .filter(|(from, to)| superseded.get(*to) != Some(*from))
10057 .map(|(from, to)| (from.clone(), to.clone()))
10058 .collect();
10059 let budget = recommend::memory_budget_mb(hw);
10060 let keep = maintenance::kept_models(
10061 &crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path()),
10062 now,
10063 );
10064 let mut facts = Vec::new();
10065 for m in refreshed.all().filter(|m| installed(m)) {
10066 let fit = recommend::model_fit(m, hw, None);
10067 let success_count = outcomes.get(&m.id).map_or(0, |(success, _)| *success);
10068 let since_install = installed_at(m)
10072 .unwrap_or(0)
10073 .max(policy.tracking_since.unwrap_or(u64::MAX));
10074 let fail_count = ledger_failures.get(&m.id).map_or(0, |times| {
10075 times.iter().filter(|t| **t >= since_install).count() as u64
10076 });
10077 let retire = self
10078 .retire_plan_on(&refreshed, &owned, &m.id, hub, None)
10079 .ok()
10080 .map(|(plan, _)| portfolio::RetireSummary {
10081 freed_bytes: plan.freed_bytes,
10082 refusals: plan
10083 .refusals
10084 .iter()
10085 .map(|r| {
10086 serde_json::to_value(r)
10087 .ok()
10088 .and_then(|v| {
10089 v.get("reason").and_then(|r| r.as_str()).map(str::to_owned)
10090 })
10091 .unwrap_or_else(|| format!("{r:?}"))
10092 })
10093 .collect(),
10094 also_retires: plan.also_retires.clone(),
10095 digest: plan.digest,
10096 });
10097 facts.push(portfolio::ModelFacts {
10098 model_id: m.id.clone(),
10099 name: m.name.clone(),
10100 generates_text: m.has_capability(ModelCapability::Generate),
10101 platform_compatible: fit.platform_compatible,
10102 never_fits: fit.fit == recommend::ModelFitStatus::TooBig,
10103 estimated_peak_mb: fit.estimated_peak_mb,
10104 hardware_budget_mb: budget,
10105 protected: protected.get(&m.id).cloned(),
10106 kept: keep.contains(&m.id),
10107 last_used: self.model_management.last_used(&m.id),
10108 ledger_last_used: ledger_last.get(&m.id).copied(),
10109 success_count,
10110 fail_count,
10111 superseded_by: superseded.get(&m.id).cloned(),
10112 retire,
10113 });
10114 }
10115 let referenced: std::collections::HashSet<String> = refreshed
10116 .all()
10117 .flat_map(retire::hf_references)
10118 .map(|r| r.repo)
10119 .collect();
10120 let mut assessed = portfolio::assess_all(&facts, &policy);
10121 assessed.orphans = retire::unreferenced_repos(hub, &referenced, &owned);
10122 assessed.recent_actions = self.recent_concierge_actions(now);
10123 assessed.revoked_pins = self.revoked_pins(&refreshed);
10124 assessed.revoked_copies =
10125 retire::revoked_copies(&self.config.state_root, &self.config.models_dir);
10126 assessed
10127 }
10128
10129 fn revoked_pins(&self, registry: &UnifiedRegistry) -> Vec<portfolio::RevokedPin> {
10137 let pins = self
10138 .external_pins
10139 .read()
10140 .unwrap_or_else(std::sync::PoisonError::into_inner)
10141 .clone();
10142 if pins.is_empty() {
10143 return Vec::new();
10144 }
10145 let names =
10146 catalog::load_revoked_names(&catalog::revoked_models_path(&self.config.state_root));
10147 pins.into_iter()
10148 .filter_map(|(pin, mut agents)| {
10149 let resolved = registry
10150 .get(&pin)
10151 .or_else(|| registry.find_by_name(&pin))
10152 .map(|schema| schema.id.clone());
10153 let model_id = if registry.is_revoked(&pin) {
10154 pin.clone()
10155 } else if let Some(id) = resolved {
10156 id
10157 } else {
10158 names.get(&pin)?.clone()
10159 };
10160 if !registry.is_revoked(&model_id) {
10161 return None;
10162 }
10163 agents.sort();
10164 agents.dedup();
10165 Some(portfolio::RevokedPin {
10166 pin,
10167 model_id,
10168 agents,
10169 })
10170 })
10171 .collect()
10172 }
10173
10174 fn protected_models(
10181 &self,
10182 registry: &UnifiedRegistry,
10183 include_lanes: bool,
10184 ) -> std::collections::BTreeMap<String, String> {
10185 let mut out = std::collections::BTreeMap::new();
10186 let mut protect = |key: &str, why: String| {
10187 let Some(schema) = registry.get(key).or_else(|| registry.find_by_name(key)) else {
10188 return;
10189 };
10190 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
10193 if let Some(twin) = registry.resolve_mlx_equivalent(schema) {
10194 out.entry(twin.id.clone()).or_insert_with(|| why.clone());
10195 }
10196 out.entry(schema.id.clone()).or_insert(why);
10197 };
10198 let lanes = if include_lanes {
10199 self.lane_defaults().defaults
10200 } else {
10201 Vec::new()
10202 };
10203 for lane in lanes {
10204 let use_case = serde_json::to_value(lane.use_case)
10205 .ok()
10206 .and_then(|v| v.as_str().map(str::to_owned))
10207 .unwrap_or_else(|| format!("{:?}", lane.use_case));
10208 protect(&lane.model_id, format!("the {use_case} lane default"));
10209 }
10210 let config = &self.config;
10211 for (key, what) in [
10212 (Some(&config.generation_model), "the generation model"),
10213 (
10214 config.preferred_generation_model.as_ref(),
10215 "the preferred generation model",
10216 ),
10217 (Some(&config.embedding_model), "the embedding model"),
10218 (
10219 config.preferred_embedding_model.as_ref(),
10220 "the preferred embedding model",
10221 ),
10222 (
10223 Some(&config.classification_model),
10224 "the classification model",
10225 ),
10226 (
10227 config.preferred_classification_model.as_ref(),
10228 "the preferred classification model",
10229 ),
10230 ] {
10231 if let Some(key) = key {
10232 protect(key, format!("{what} in CAR's configuration"));
10233 }
10234 }
10235 for (capability, what) in [
10236 (
10237 ModelCapability::SpeechToText,
10238 "the local speech-to-text default",
10239 ),
10240 (
10241 ModelCapability::TextToSpeech,
10242 "the local text-to-speech default",
10243 ),
10244 ] {
10245 if let Some(name) = self.speech_health_default_name(capability, true, false) {
10246 protect(&name, what.to_string());
10247 }
10248 }
10249 protect(
10250 LOCAL_TOOL_MODEL_PULL_ID,
10251 "the local tool model CAR's setup guidance installs".to_string(),
10252 );
10253 let pins = self
10254 .external_pins
10255 .read()
10256 .unwrap_or_else(std::sync::PoisonError::into_inner)
10257 .clone();
10258 for (key, agents) in pins {
10259 protect(&key, pinned_by(&agents));
10260 }
10261 out
10262 }
10263
10264 pub async fn resume_retirements(&self) -> Vec<String> {
10269 let _concierge = self.concierge_action_lock.lock().await;
10270 let mut errors: Vec<String> = self
10271 .model_management
10272 .resume_detached_trees()
10273 .into_iter()
10274 .map(|e| e.to_string())
10275 .collect();
10276 errors.extend(
10277 self.model_management
10278 .resume_retiring()
10279 .into_iter()
10280 .map(|e| e.to_string()),
10281 );
10282 errors
10283 }
10284
10285 pub fn model_management_store(&self) -> &model_management::ModelManagementStore {
10286 &self.model_management
10287 }
10288
10289 fn ensure_model_enabled(&self, model_id: &str) -> Result<(), InferenceError> {
10290 if self.model_management.car_enabled(model_id)? {
10291 return Ok(());
10292 }
10293 Err(InferenceError::InferenceFailed(format!(
10294 "local model {model_id} was removed from CAR; reinstall it before use"
10295 )))
10296 }
10297
10298 pub fn available_model_upgrades(&self) -> Vec<ModelUpgrade> {
10300 self.unified_registry().available_upgrades()
10301 }
10302
10303 pub async fn check_upgrade_nudge(
10309 &self,
10310 inference_active: bool,
10311 ) -> (crate::nudge::NudgeDecision, crate::nudge::NudgeState) {
10312 let findings = self.detect_upgrades().await;
10313 let mut prefs = self.update_prefs();
10314 prefs.policy = self.effective_update_policy(&prefs, false).0;
10315 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10316 let now = std::time::SystemTime::now()
10317 .duration_since(std::time::UNIX_EPOCH)
10318 .map(|d| d.as_secs())
10319 .unwrap_or(0);
10320 let decision = crate::nudge::decide_nudge(
10321 &findings,
10322 &prefs,
10323 &state,
10324 now,
10325 crate::nudge::DEFAULT_THROTTLE_SECS,
10326 inference_active,
10327 );
10328 (decision, state)
10329 }
10330
10331 pub fn dismiss_upgrade_nudge(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10334 let path = crate::nudge::NudgeState::default_path();
10335 let mut state = crate::nudge::NudgeState::load_from(&path);
10336 state.dismiss(dismiss_key);
10337 state
10338 .save_to(&path)
10339 .map_err(InferenceError::InferenceFailed)
10340 }
10341
10342 pub async fn check_concierge(
10349 &self,
10350 inference_active: bool,
10351 ) -> (
10352 Vec<crate::concierge::ConciergeSuggestion>,
10353 crate::nudge::NudgeState,
10354 ) {
10355 let prefs = self.update_prefs();
10356 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10357 let hw = crate::hardware::HardwareInfo::detect();
10358 let schemas = self.list_schemas();
10359 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10360 let now = std::time::SystemTime::now()
10361 .duration_since(std::time::UNIX_EPOCH)
10362 .map(|d| d.as_secs())
10363 .unwrap_or(0);
10364 let suggestions = crate::concierge::decide_concierge(
10365 &refs,
10366 &hw,
10367 crate::concierge::DEFAULT_WATCHED_USE_CASES,
10368 crate::intent::QualityTier::Balanced,
10369 &prefs,
10370 &state,
10371 now,
10372 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
10373 inference_active,
10374 );
10375 (suggestions, state)
10376 }
10377
10378 pub fn dismiss_concierge_suggestion(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10383 self.dismiss_upgrade_nudge(dismiss_key)
10384 }
10385
10386 pub fn dismiss_concierge_labeled(
10390 &self,
10391 dismiss_key: &str,
10392 reason: crate::concierge::DismissReason,
10393 ) -> Result<(), String> {
10394 let path = crate::nudge::NudgeState::default_path();
10395 let mut state = crate::nudge::NudgeState::load_from(&path);
10396 let now = std::time::SystemTime::now()
10397 .duration_since(std::time::UNIX_EPOCH)
10398 .map(|d| d.as_secs())
10399 .unwrap_or(0);
10400 state.dismiss_labeled(dismiss_key, reason, now);
10401 state.save_to(&path).map_err(|e| e.to_string())
10402 }
10403
10404 pub async fn check_canaries(&self) -> Vec<crate::intent::UseCase> {
10412 use crate::action_ledger::ConciergeActionKind;
10413 use crate::concierge::{
10414 canary_verdict, CanaryVerdict, CANARY_MIN_SAMPLES, CANARY_REGRESSION_MARGIN,
10415 };
10416 use std::collections::BTreeMap;
10417
10418 let actions = self.concierge_actions(0);
10419 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
10420 let entries = crate::outcome::read_ledger(&ledger_path, 0);
10421
10422 let mut latest: BTreeMap<
10424 crate::intent::UseCase,
10425 &crate::action_ledger::ConciergeActionEntry,
10426 > = BTreeMap::new();
10427 for a in &actions {
10428 if a.project.is_some() {
10429 continue; }
10431 if let Some(uc) = a.use_case {
10432 latest.insert(uc, a);
10433 }
10434 }
10435
10436 let mut reverts: Vec<(crate::intent::UseCase, u64)> = Vec::new();
10441 let tracker = self.outcome_tracker.read().await;
10442 for (uc, a) in latest {
10443 if a.kind != ConciergeActionKind::SetDefault {
10446 continue;
10447 }
10448 let Some(prior) = a.prior_model_id.as_deref() else {
10449 continue; };
10451
10452 let (mut succ, mut total) = (0u64, 0u64);
10454 for e in &entries {
10455 if e.timestamp < a.timestamp
10456 || e.model_id != a.model_id
10457 || crate::usage_profile::use_case_for_task(e.task) != uc
10458 {
10459 continue;
10460 }
10461 match e.success {
10462 Some(true) => {
10463 succ += 1;
10464 total += 1;
10465 }
10466 Some(false) => total += 1,
10467 None => {}
10468 }
10469 }
10470 let new_rate = if total == 0 {
10471 None
10472 } else {
10473 Some(succ as f64 / total as f64)
10474 };
10475
10476 let (mut base_succ, mut base_total) = (0u64, 0u64);
10481 if let Some(profile) = tracker.profile(prior) {
10482 for t in [
10483 crate::outcome::InferenceTask::Generate,
10484 crate::outcome::InferenceTask::Embed,
10485 crate::outcome::InferenceTask::Classify,
10486 crate::outcome::InferenceTask::Code,
10487 crate::outcome::InferenceTask::Reasoning,
10488 ] {
10489 if crate::usage_profile::use_case_for_task(t) != uc {
10490 continue;
10491 }
10492 if let Some(ts) = profile.task_stats(t) {
10493 base_succ += ts.successes;
10494 base_total += ts.successes + ts.failures;
10495 }
10496 }
10497 }
10498 if base_total == 0 {
10501 continue;
10502 }
10503 let baseline = base_succ as f64 / base_total as f64;
10504
10505 if canary_verdict(
10506 new_rate,
10507 total,
10508 baseline,
10509 CANARY_MIN_SAMPLES,
10510 CANARY_REGRESSION_MARGIN,
10511 ) == CanaryVerdict::Revert
10512 {
10513 reverts.push((uc, a.seq));
10517 }
10518 }
10519 drop(tracker);
10520
10521 let mut reverted = Vec::new();
10524 for (uc, seq) in reverts {
10525 if self.rollback_lane_inner(uc, None, Some(seq)).await.is_ok() {
10526 tracing::info!(lane = ?uc, "concierge canary: auto-reverted a worse model switch");
10527 reverted.push(uc);
10528 }
10529 }
10530 reverted
10531 }
10532
10533 pub async fn concierge_ask(
10550 &self,
10551 question: Option<&str>,
10552 ) -> Result<concierge_answer::ConciergeAnswer, String> {
10553 let maintenance = self.maintenance_pass(true).await;
10554 let portfolio = match maintenance.portfolio.clone() {
10555 Some(portfolio) => portfolio,
10556 None => self.portfolio().await,
10557 };
10558 let memory_policy =
10559 serde_json::to_value(&self.active_local_resource_policy().policy.profile)
10560 .ok()
10561 .and_then(|v| v.as_str().map(str::to_owned))
10562 .unwrap_or_else(|| "unknown".into());
10563 let summary = concierge_answer::summarize(&portfolio, &maintenance, &memory_policy);
10564 let Some(question) = question else {
10565 return Ok(concierge_answer::ConciergeAnswer {
10566 answer: summary.clone(),
10567 summary,
10568 model_used: None,
10569 ungrounded: Vec::new(),
10570 });
10571 };
10572 use std::fmt::Write as _;
10573 let status = self.concierge_status(false).await;
10574 let hw = crate::hardware::HardwareInfo::detect();
10575 let schemas = self.list_schemas();
10576 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10577
10578 let mut evidence = String::new();
10579 evidence.push_str("OBSERVED USAGE (last 30 days):\n");
10580 if status.lanes.is_empty() {
10581 evidence.push_str(" (no usage recorded yet)\n");
10582 }
10583 for lane in &status.lanes {
10584 let rate = lane
10585 .success_rate()
10586 .map(|r| format!("{:.0}% success", r * 100.0))
10587 .unwrap_or_else(|| "no resolved signal".into());
10588 let _ = writeln!(
10589 evidence,
10590 " {:?}: {} calls, {}{}",
10591 lane.use_case,
10592 lane.calls,
10593 rate,
10594 if lane.failing_models.is_empty() {
10595 String::new()
10596 } else {
10597 format!(
10598 ", failing on {}",
10599 lane.failing_models
10600 .iter()
10601 .cloned()
10602 .collect::<Vec<_>>()
10603 .join(", ")
10604 )
10605 }
10606 );
10607 }
10608 evidence.push_str("\nMODEL HEALTH:\n");
10609 for m in &status.models {
10610 let success = match m.success_rate {
10611 Some(r) => format!("{:.0}% success", r * 100.0),
10612 None => "no resolved signal".to_string(),
10613 };
10614 let _ = writeln!(
10615 evidence,
10616 " {}: {} calls, {}, {:.0}ms avg{}",
10617 m.model_id,
10618 m.calls,
10619 success,
10620 m.avg_latency_ms,
10621 if m.excluded { " (excluded)" } else { "" }
10622 );
10623 }
10624 let mut menu_lanes: Vec<crate::intent::UseCase> =
10630 crate::concierge::DEFAULT_WATCHED_USE_CASES.to_vec();
10631 for lane in &status.lanes {
10632 if !menu_lanes.contains(&lane.use_case) {
10633 menu_lanes.push(lane.use_case);
10634 }
10635 }
10636 evidence.push_str("\nGROUNDED CANDIDATES (fit verified for this machine):\n");
10637 for uc in menu_lanes {
10638 let set = crate::recommend::recommend(
10639 &refs,
10640 &hw,
10641 uc,
10642 crate::intent::QualityTier::Balanced,
10643 crate::intent::Privacy::OnDevice,
10644 );
10645 for p in set.picks.iter().take(3) {
10646 let _ = writeln!(
10647 evidence,
10648 " [{:?}] {} — {}{}",
10649 uc,
10650 p.display_name,
10651 if p.already_installed {
10652 "installed"
10653 } else {
10654 "available"
10655 },
10656 if p.fit == crate::recommend::FitStatus::Fits {
10657 ", fits"
10658 } else {
10659 ", does NOT fit"
10660 }
10661 );
10662 }
10663 }
10664 if let Some(s) = &status.decision.suggestion {
10665 let _ = writeln!(evidence, "\nCURRENT SUGGESTION: {}", s.message);
10666 }
10667
10668 evidence.push_str("\nLOCAL MODELS ON THIS MACHINE:\n");
10669 evidence.push_str(&summary);
10670
10671 let prompt = format!(
10672 "You are CAR's model concierge. Answer the user's question ONLY from the \
10673 EVIDENCE provided as context: this machine's local models and what each is \
10674 doing, what maintenance would do, observed usage and health, and the \
10675 grounded candidate menu (the only models you may recommend acquiring). \
10676 Name models by the ids the evidence uses. NEVER invent a model and NEVER \
10677 claim a model fits or is better than the evidence states. If the evidence \
10678 doesn't answer the question, say so. Answer in a few short sentences or \
10679 bullets.\n\nQUESTION: {question}"
10680 );
10681 let req = crate::tasks::generate::GenerateRequest {
10682 prompt,
10683 context: Some(evidence.clone()),
10684 intent: Some(crate::intent::IntentHint {
10685 task: Some(crate::intent::TaskHint::Chat),
10686 prefer_local: true,
10687 ..Default::default()
10688 }),
10689 params: crate::tasks::generate::GenerateParams {
10690 temperature: 0.2,
10691 max_tokens: 600,
10692 workload: crate::tasks::generate::RoutingWorkload::Background,
10693 ..Default::default()
10694 },
10695 ..Default::default()
10696 };
10697 match self.generate_tracked(req).await {
10698 Ok(result) => {
10699 let reply = concierge_answer::trim_repetition(&result.text);
10700 let ungrounded = concierge_answer::ungrounded_ids(&reply, &evidence);
10701 let answer = if ungrounded.is_empty() {
10702 reply
10703 } else {
10704 format!(
10705 "{reply}\n\n(Not verified for this machine: {}. Only the models listed \
10706 above are checked.)",
10707 ungrounded.join(", ")
10708 )
10709 };
10710 Ok(concierge_answer::ConciergeAnswer {
10711 answer,
10712 summary,
10713 model_used: Some(result.model_used),
10714 ungrounded,
10715 })
10716 }
10717 Err(error) => {
10718 tracing::warn!(%error, "concierge.ask: no model could answer; returning the facts");
10719 Ok(concierge_answer::ConciergeAnswer {
10720 answer: format!(
10721 "{summary}\n(No model could answer the question itself: {error}.)"
10722 ),
10723 summary,
10724 model_used: None,
10725 ungrounded: Vec::new(),
10726 })
10727 }
10728 }
10729 }
10730
10731 pub async fn refresh_catalog(&self) -> Result<usize, String> {
10741 let url = catalog::catalog_url();
10742 let pubkey = catalog::catalog_public_key();
10743 let verified = self.fetch_checked_catalog(&url, &pubkey).await?;
10748 let path = crate::catalog::cache_path(&self.config.state_root);
10749 self.adopt_verified_catalog(&path, &verified, &pubkey).await
10750 }
10751
10752 async fn fetch_checked_catalog(
10757 &self,
10758 url: &str,
10759 pubkey: &str,
10760 ) -> Result<catalog::VerifiedCatalog, String> {
10761 use catalog::CheckFailure::{Older, Rejected, Unreachable};
10762 let http = crate::tls_client::catalog_refresh_client();
10763 let fetched = match crate::catalog::fetch_signed(&http, url).await {
10766 Err(error) => Err((Unreachable, error)),
10767 Ok((bytes, sig)) => {
10768 crate::catalog::verify_fetched(bytes, sig, pubkey).map_err(|e| (Rejected, e))
10769 }
10770 };
10771 let cached = crate::catalog::load_doc(
10775 &crate::catalog::cache_path(&self.config.state_root),
10776 Some(pubkey),
10777 )
10778 .map(|doc| doc.version);
10779 let outcome = match &fetched {
10780 Ok(verified) => match cached {
10781 Some(cached) if verified.version() < cached => Err((
10782 Older,
10783 format!(
10784 "the source served catalog v{}, older than the cached v{cached}",
10785 verified.version()
10786 ),
10787 )),
10788 _ => Ok(verified.version()),
10789 },
10790 Err((failure, error)) => Err((*failure, error.clone())),
10791 };
10792 catalog::record_check(
10793 &catalog::status_path(&self.config.state_root),
10794 url,
10795 outcome
10796 .as_ref()
10797 .map(|v| *v)
10798 .map_err(|(f, e)| (*f, e.as_str())),
10799 now_unix(),
10800 );
10801 fetched.map_err(|(_, error)| error)
10802 }
10803
10804 pub fn mark_catalog_loop_alive(&self) {
10806 catalog::record_loop_alive(&catalog::status_path(&self.config.state_root), now_unix());
10807 }
10808
10809 pub fn mark_daemon_alive_for_catalog(&self) {
10812 catalog::record_daemon_alive(&catalog::status_path(&self.config.state_root), now_unix());
10813 }
10814
10815 pub fn catalog_freshness(&self) -> catalog::CatalogFreshness {
10818 catalog::catalog_freshness(&catalog::load_status(&catalog::status_path(
10819 &self.config.state_root,
10820 )))
10821 }
10822
10823 pub async fn refresh_catalog_on_schedule(&self) -> Result<Option<usize>, String> {
10828 if matches!(self.update_prefs().policy, update_prefs::UpdatePolicy::Off) {
10829 catalog::record_off(&catalog::status_path(&self.config.state_root), now_unix());
10830 return Ok(None);
10831 }
10832 self.refresh_catalog_from(&catalog::catalog_url(), &catalog::catalog_public_key())
10833 .await
10834 }
10835
10836 async fn refresh_catalog_from(&self, url: &str, pubkey: &str) -> Result<Option<usize>, String> {
10838 let verified = self.fetch_checked_catalog(url, pubkey).await?;
10839 let path = crate::catalog::cache_path(&self.config.state_root);
10840 let cached = crate::catalog::load_doc(&path, Some(pubkey)).map(|doc| doc.version);
10841 if cached.is_some_and(|version| verified.version() <= version) {
10842 return Ok(None);
10843 }
10844 self.adopt_verified_catalog(&path, &verified, pubkey)
10845 .await
10846 .map(Some)
10847 }
10848
10849 async fn adopt_verified_catalog(
10856 &self,
10857 path: &Path,
10858 verified: &catalog::VerifiedCatalog,
10859 pubkey: &str,
10860 ) -> Result<usize, String> {
10861 let count = crate::catalog::install_if_newer(path, verified, pubkey).await?;
10862 let newest = crate::catalog::load_verified(path, Some(pubkey))
10863 .ok_or_else(|| format!("re-reading the verified catalog at {}", path.display()))?;
10864 let revoked = newest.revoked().to_vec();
10865 self.unlink_revoked_signed_models(&revoked).await;
10866 self.swap_signed_catalog(newest);
10867 self.clear_revoked_lane_defaults(&revoked).await;
10868 Ok(count)
10869 }
10870
10871 pub async fn enforce_catalog_revocations(&self) {
10876 let revoked = self.current_registry().revoked_ids();
10877 self.clear_revoked_lane_defaults(&revoked).await;
10878 }
10879
10880 async fn clear_revoked_lane_defaults(&self, revoked: &[String]) {
10888 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
10889 if revoked.is_empty() {
10890 return;
10891 }
10892 let _guard = self.concierge_action_lock.lock().await;
10893 let now = std::time::SystemTime::now()
10894 .duration_since(std::time::UNIX_EPOCH)
10895 .map(|d| d.as_secs())
10896 .unwrap_or(0);
10897 for lane in self.lane_defaults().defaults {
10898 if !revoked.contains(&lane.model_id) {
10899 continue;
10900 }
10901 match self.clear_lane_default(lane.project.as_deref(), lane.use_case) {
10902 Ok(true) => self.record_concierge_action(ConciergeActionEntry {
10903 seq: 0,
10904 kind: ConciergeActionKind::RevokedLaneCleared,
10905 model_id: lane.model_id.clone(),
10906 use_case: Some(lane.use_case),
10907 project: lane.project.clone(),
10908 prior_model_id: Some(lane.model_id.clone()),
10909 detail: "revoked by the signed model catalog".into(),
10910 timestamp: now,
10911 }),
10912 Ok(false) => {}
10913 Err(error) => {
10914 tracing::warn!(model = %lane.model_id, %error, "could not clear a revoked lane default")
10915 }
10916 }
10917 }
10918 for pin in self.revoked_pins(&self.current_registry()) {
10919 tracing::warn!(model = %pin.model_id, pin = %pin.pin, agents = ?pin.agents, "a pinned model was revoked by the signed catalog; re-point it");
10920 }
10921 }
10922
10923 async fn unlink_revoked_signed_models(&self, revoked: &[String]) {
10932 let registry = self.current_registry();
10933 let names_path = catalog::revoked_names_path(&self.config.state_root);
10934 let mut names = catalog::load_revoked_names(&names_path);
10935 let models_path = catalog::revoked_models_path(&self.config.state_root);
10936 let mut models = catalog::load_revoked_names(&models_path);
10937 names.retain(|_, id| revoked.contains(id));
10939 models.retain(|_, id| revoked.contains(id));
10940 let mut installed = Vec::new();
10941 for id in revoked {
10942 let Some(row) = registry.get(id) else {
10943 continue;
10944 };
10945 if registry.signed_origin_of(id).is_none() || registry.is_builtin(id) {
10946 continue;
10947 }
10948 if !retire::single_component(&row.name) {
10953 continue;
10954 }
10955 models.insert(row.name.clone(), id.clone());
10956 if row.is_local() && row.downloads_weights() {
10957 names.insert(row.name.clone(), id.clone());
10958 installed.push((id.clone(), row.name.clone()));
10959 }
10960 }
10961 for (path, record) in [(&names_path, &names), (&models_path, &models)] {
10962 if let Err(error) = catalog::save_revoked_names(path, record) {
10963 tracing::warn!(path = %path.display(), %error, "could not record revoked model names");
10964 }
10965 }
10966 let models_dir = self.config.models_dir.clone();
10967 for (id, name) in installed {
10968 let projection = models_dir.join(&name);
10969 if std::fs::symlink_metadata(&projection).is_err() || !retire::links_only(&projection) {
10970 continue;
10971 }
10972 match self.model_management.load_receipt(&id) {
10973 Ok(Some(receipt))
10974 if receipt.artifact_kind == model_management::ManagedArtifactKind::Symlink =>
10975 {
10976 if let Err(error) = self.remove_model_from_car(&id).await {
10977 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
10978 }
10979 }
10980 Ok(Some(_)) => {}
10981 Ok(None) => {
10982 if let Err(error) = retire::remove_links_only_tree(&projection, &models_dir) {
10983 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
10984 }
10985 }
10986 Err(error) => {
10987 tracing::warn!(%id, %error, "could not read a revoked model's receipt")
10988 }
10989 }
10990 }
10991 }
10992
10993 pub async fn discover_models(&self) -> Result<usize, String> {
11002 use crate::schema::{ModelSource, TrustTier};
11003 let template = self
11010 .unified_registry()
11011 .all()
11012 .filter(|m| {
11013 m.provider.eq_ignore_ascii_case("openai")
11014 && m.trust_tier == TrustTier::Curated
11015 && matches!(m.source, ModelSource::RemoteApi { .. })
11016 })
11017 .max_by(|a, b| {
11018 a.capabilities
11019 .len()
11020 .cmp(&b.capabilities.len())
11021 .then_with(|| a.id.cmp(&b.id))
11022 })
11023 .cloned();
11024 let Some(template) = template else {
11025 return Ok(0); };
11027 let (endpoint, api_key_env) = match &template.source {
11028 ModelSource::RemoteApi {
11029 endpoint,
11030 api_key_env,
11031 ..
11032 } => (endpoint.clone(), api_key_env.clone()),
11033 _ => return Ok(0),
11034 };
11035 let Some(models_url) = crate::discovery::models_url_from_endpoint(&endpoint) else {
11036 return Ok(0);
11037 };
11038 let key = match car_secrets::resolve_env_or_keychain(&api_key_env) {
11039 Some(k) if !k.is_empty() => k,
11040 _ => return Ok(0), };
11042
11043 let http = reqwest::Client::builder()
11047 .timeout(std::time::Duration::from_secs(20))
11048 .build()
11049 .map_err(|e| format!("discovery client: {e}"))?;
11050 let ids = crate::discovery::fetch_model_ids(&http, &models_url, &key).await?;
11051
11052 let cache = crate::discovery::cache_path(&self.config.state_models_dir());
11054 let mut cached = crate::discovery::load_cache(&cache);
11055 let mut have: std::collections::HashSet<String> =
11056 cached.iter().map(|m| m.id.clone()).collect();
11057 for id in ids {
11058 if !crate::discovery::is_chat_model(&id) {
11059 continue;
11060 }
11061 let schema = crate::discovery::discovered_schema("openai", &id, &template);
11062 if self.unified_registry().get(&schema.id).is_some() {
11068 continue;
11069 }
11070 if have.insert(schema.id.clone()) {
11071 cached.push(schema);
11072 }
11073 }
11074 let count = cached.len();
11075 crate::discovery::save_cache(&cache, &cached)?;
11076 Ok(count)
11077 }
11078
11079 pub fn lane_defaults(&self) -> crate::lane_defaults::LaneDefaults {
11081 self.lane_defaults_cache.read().unwrap().clone()
11082 }
11083
11084 pub fn lane_default(
11088 &self,
11089 project: Option<&str>,
11090 use_case: crate::intent::UseCase,
11091 ) -> Option<String> {
11092 self.lane_defaults_cache
11093 .read()
11094 .unwrap()
11095 .resolve(project, use_case)
11096 .map(str::to_string)
11097 }
11098
11099 fn model_is_excluded(
11100 exclude_set: &std::collections::HashSet<String>,
11101 registry: &UnifiedRegistry,
11102 model: &str,
11103 ) -> bool {
11104 registry
11105 .get(model)
11106 .or_else(|| registry.find_by_name(model))
11107 .map(|schema| exclude_set.contains(&schema.id))
11108 .unwrap_or_else(|| exclude_set.contains(model))
11109 }
11110
11111 fn lane_pin_for(
11117 &self,
11118 req: &GenerateRequest,
11119 routing_registry: &UnifiedRegistry,
11120 ) -> Option<String> {
11121 let task = req.intent.as_ref().and_then(|h| h.task)?;
11122 let use_case = crate::usage_profile::use_case_for_task_hint(task);
11123 let id = self.lane_default(None, use_case)?;
11124 let known = routing_registry
11129 .get(&id)
11130 .or_else(|| routing_registry.find_by_name(&id));
11131 match known {
11132 Some(s) if s.available_now() => Some(id),
11133 _ => None,
11134 }
11135 }
11136
11137 pub fn set_lane_default(
11140 &self,
11141 project: Option<String>,
11142 use_case: crate::intent::UseCase,
11143 model_id: &str,
11144 ) -> Result<(), String> {
11145 if self.current_registry().is_revoked(model_id) {
11148 return Err(format!(
11149 "{model_id} was revoked by the signed model catalog and cannot be a lane default"
11150 ));
11151 }
11152 let now = std::time::SystemTime::now()
11153 .duration_since(std::time::UNIX_EPOCH)
11154 .map(|d| d.as_secs())
11155 .unwrap_or(0);
11156 let mut defaults = self.lane_defaults_cache.write().unwrap();
11159 defaults.set(project, use_case, model_id.to_string(), now);
11160 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11161 .map_err(|e| e.to_string())
11162 }
11163
11164 pub fn clear_lane_default(
11167 &self,
11168 project: Option<&str>,
11169 use_case: crate::intent::UseCase,
11170 ) -> Result<bool, String> {
11171 let mut defaults = self.lane_defaults_cache.write().unwrap();
11172 let removed = defaults.clear(project, use_case);
11173 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11174 .map_err(|e| e.to_string())?;
11175 Ok(removed)
11176 }
11177
11178 pub async fn user_set_lane_default(
11186 &self,
11187 use_case: crate::intent::UseCase,
11188 model_id: &str,
11189 project: Option<String>,
11190 ) -> Result<(), String> {
11191 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11192 let _guard = self.concierge_action_lock.lock().await;
11193 let prior = self.lane_default(project.as_deref(), use_case);
11194 self.set_lane_default(project.clone(), use_case, model_id)?;
11195 self.record_concierge_action(ConciergeActionEntry {
11196 seq: 0,
11197 kind: ConciergeActionKind::SetDefault,
11198 model_id: model_id.to_string(),
11199 use_case: Some(use_case),
11200 project,
11201 prior_model_id: prior,
11202 detail: "user set lane default".into(),
11203 timestamp: now_unix(),
11204 });
11205 Ok(())
11206 }
11207
11208 pub async fn user_clear_lane_default(
11213 &self,
11214 use_case: crate::intent::UseCase,
11215 project: Option<String>,
11216 ) -> Result<bool, String> {
11217 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11218 let _guard = self.concierge_action_lock.lock().await;
11219 let prior = self.lane_default(project.as_deref(), use_case);
11220 let removed = self.clear_lane_default(project.as_deref(), use_case)?;
11221 if removed {
11222 self.record_concierge_action(ConciergeActionEntry {
11223 seq: 0,
11224 kind: ConciergeActionKind::ClearDefault,
11225 model_id: prior.clone().unwrap_or_default(),
11226 use_case: Some(use_case),
11227 project,
11228 prior_model_id: prior,
11229 detail: "user cleared lane default".into(),
11230 timestamp: now_unix(),
11231 });
11232 }
11233 Ok(removed)
11234 }
11235
11236 pub fn concierge_actions(
11238 &self,
11239 limit: usize,
11240 ) -> Vec<crate::action_ledger::ConciergeActionEntry> {
11241 crate::action_ledger::read_actions(&self.action_ledger_path(), limit)
11242 }
11243
11244 fn lane_defaults_path(&self) -> PathBuf {
11246 self.config
11247 .state_root
11248 .join(crate::lane_defaults::LANE_DEFAULTS_FILE)
11249 }
11250
11251 fn action_ledger_path(&self) -> PathBuf {
11254 self.config
11255 .state_root
11256 .join(crate::action_ledger::ACTION_LEDGER_FILE)
11257 }
11258
11259 fn record_concierge_action(&self, mut entry: crate::action_ledger::ConciergeActionEntry) {
11260 entry.seq = self
11261 .concierge_action_seq
11262 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
11263 if let Err(e) = crate::action_ledger::append_action(&self.action_ledger_path(), &entry) {
11264 tracing::debug!("record concierge action failed: {e}");
11265 }
11266 }
11267
11268 pub async fn apply_concierge(
11280 &self,
11281 use_case: crate::intent::UseCase,
11282 model_id: &str,
11283 project: Option<String>,
11284 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11285 self.with_pinned_registry(self.apply_concierge_unpinned(use_case, model_id, project))
11286 .await
11287 }
11288
11289 async fn apply_concierge_unpinned(
11290 &self,
11291 use_case: crate::intent::UseCase,
11292 model_id: &str,
11293 project: Option<String>,
11294 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11295 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11296 let now = || {
11297 std::time::SystemTime::now()
11298 .duration_since(std::time::UNIX_EPOCH)
11299 .map(|d| d.as_secs())
11300 .unwrap_or(0)
11301 };
11302 if let Some(schema) = self
11307 .list_schemas()
11308 .into_iter()
11309 .find(|s| s.id == model_id || s.name == model_id)
11310 {
11311 let serves = use_case
11312 .required_capabilities()
11313 .iter()
11314 .all(|c| schema.capabilities.contains(c));
11315 if !serves {
11316 return Err(format!(
11317 "model '{model_id}' does not serve the {use_case:?} lane"
11318 ));
11319 }
11320 }
11321
11322 if self
11330 .unified_registry()
11331 .existing_local_artifact(model_id)
11332 .is_none()
11333 {
11334 self.pull_model(model_id).await.map_err(|e| e.to_string())?;
11335 }
11336
11337 let _guard = self.concierge_action_lock.lock().await;
11340 let prior = self.lane_default(project.as_deref(), use_case);
11343 self.record_concierge_action(ConciergeActionEntry {
11344 seq: 0, kind: ConciergeActionKind::Install,
11346 model_id: model_id.to_string(),
11347 use_case: Some(use_case),
11348 project: project.clone(),
11349 prior_model_id: None,
11350 detail: "concierge apply: ensured model present".into(),
11351 timestamp: now(),
11352 });
11353
11354 self.set_lane_default(project.clone(), use_case, model_id)?;
11356 self.record_concierge_action(ConciergeActionEntry {
11357 seq: 0, kind: ConciergeActionKind::SetDefault,
11359 model_id: model_id.to_string(),
11360 use_case: Some(use_case),
11361 project: project.clone(),
11362 prior_model_id: prior.clone(),
11363 detail: "concierge apply: set lane default".into(),
11364 timestamp: now(),
11365 });
11366
11367 Ok(crate::action_ledger::ConciergeApplyResult {
11368 model_id: model_id.to_string(),
11369 use_case,
11370 installed: true,
11371 set_default: true,
11372 prior_model_id: prior,
11373 })
11374 }
11375
11376 pub async fn rollback_lane(
11381 &self,
11382 use_case: crate::intent::UseCase,
11383 project: Option<String>,
11384 ) -> Result<Option<String>, String> {
11385 self.rollback_lane_inner(use_case, project, None).await
11386 }
11387
11388 async fn rollback_lane_inner(
11394 &self,
11395 use_case: crate::intent::UseCase,
11396 project: Option<String>,
11397 expected_anchor_seq: Option<u64>,
11398 ) -> Result<Option<String>, String> {
11399 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11400 let _guard = self.concierge_action_lock.lock().await;
11401 let actions = self.concierge_actions(0);
11406 let anchor = actions.iter().rev().find(|a| {
11407 matches!(
11408 a.kind,
11409 ConciergeActionKind::SetDefault | ConciergeActionKind::Rollback
11410 ) && a.use_case == Some(use_case)
11411 && a.project == project
11412 });
11413 let set = match anchor {
11414 None => return Err("no prior set-default to roll back".into()),
11415 Some(a) if a.kind == ConciergeActionKind::Rollback => {
11416 return Err("already rolled back to the prior default; nothing to undo".into())
11417 }
11418 Some(a) => a,
11419 };
11420 if let Some(seq) = expected_anchor_seq {
11423 if set.seq != seq {
11424 return Err("lane default changed since the canary decision; not reverting".into());
11425 }
11426 }
11427 let prior = set.prior_model_id.clone();
11428 let now = std::time::SystemTime::now()
11429 .duration_since(std::time::UNIX_EPOCH)
11430 .map(|d| d.as_secs())
11431 .unwrap_or(0);
11432
11433 match &prior {
11434 Some(m) => self.set_lane_default(project.clone(), use_case, m)?,
11435 None => {
11436 self.clear_lane_default(project.as_deref(), use_case)?;
11437 }
11438 }
11439 let detail = match &prior {
11440 Some(_) => "concierge rollback: restored prior lane default",
11441 None => "concierge rollback: cleared lane default (no prior)",
11442 };
11443 self.record_concierge_action(ConciergeActionEntry {
11444 seq: 0, kind: ConciergeActionKind::Rollback,
11446 model_id: prior.clone().unwrap_or_default(),
11447 use_case: Some(use_case),
11448 project,
11449 prior_model_id: Some(set.model_id.clone()),
11450 detail: detail.into(),
11451 timestamp: now,
11452 });
11453 Ok(prior)
11454 }
11455
11456 pub async fn concierge_status(
11461 &self,
11462 inference_active: bool,
11463 ) -> crate::concierge::ConciergeStatus {
11464 const USAGE_WINDOW_SECS: u64 = 30 * 24 * 60 * 60;
11466
11467 let prefs = self.update_prefs();
11468 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
11469 let hw = crate::hardware::HardwareInfo::detect();
11470 let schemas = self.list_schemas();
11471 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11472 let now = std::time::SystemTime::now()
11473 .duration_since(std::time::UNIX_EPOCH)
11474 .map(|d| d.as_secs())
11475 .unwrap_or(0);
11476
11477 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
11478 let entries = crate::outcome::read_ledger(&ledger_path, 0);
11479 let usage =
11480 crate::usage_profile::UsageProfile::from_ledger(&entries, now, USAGE_WINDOW_SECS);
11481
11482 let decision = crate::concierge::evaluate_concierge(
11483 &refs,
11484 &hw,
11485 &usage,
11486 crate::intent::QualityTier::Balanced,
11487 &prefs,
11488 &state,
11489 now,
11490 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
11491 inference_active,
11492 );
11493
11494 let tracker = self.outcome_tracker.read().await;
11495 let models = tracker
11496 .export_profiles()
11497 .iter()
11498 .map(|p| crate::concierge::ModelHealth {
11499 model_id: p.model_id.clone(),
11500 calls: p.total_calls,
11501 success_rate: p.success_rate_resolved(),
11505 avg_latency_ms: p.avg_latency_ms(),
11506 quality: p.ema_quality,
11507 excluded: tracker.is_excluded(&p.model_id),
11508 })
11509 .collect();
11510 drop(tracker);
11511
11512 const STALE_VERIFY_SECS: u64 = 14 * 24 * 60 * 60;
11517 let actions = self.concierge_actions(0);
11518 let mut latest: std::collections::BTreeMap<
11519 crate::intent::UseCase,
11520 &crate::action_ledger::ConciergeActionEntry,
11521 > = std::collections::BTreeMap::new();
11522 for a in &actions {
11523 if a.project.is_none() {
11524 if let Some(uc) = a.use_case {
11525 latest.insert(uc, a);
11526 }
11527 }
11528 }
11529 let mut pending_verification = Vec::new();
11530 for (uc, a) in latest {
11531 if a.kind != crate::action_ledger::ConciergeActionKind::SetDefault {
11532 continue;
11533 }
11534 if now.saturating_sub(a.timestamp) < STALE_VERIFY_SECS {
11535 continue; }
11537 let resolved = entries
11538 .iter()
11539 .filter(|e| {
11540 e.timestamp >= a.timestamp
11541 && e.model_id == a.model_id
11542 && crate::usage_profile::use_case_for_task(e.task) == uc
11543 && e.success.is_some()
11544 })
11545 .count() as u64;
11546 if resolved < crate::concierge::CANARY_MIN_SAMPLES {
11547 pending_verification.push(crate::concierge::PendingVerification {
11548 use_case: uc,
11549 model_id: a.model_id.clone(),
11550 set_at: a.timestamp,
11551 resolved_samples: resolved,
11552 needed: crate::concierge::CANARY_MIN_SAMPLES,
11553 });
11554 }
11555 }
11556
11557 crate::concierge::ConciergeStatus {
11558 lanes: usage.active_lanes().into_iter().cloned().collect(),
11559 decision,
11560 models,
11561 pending_verification,
11562 catalog: self.catalog_freshness(),
11563 revoked_pins: self.revoked_pins(&self.current_registry()),
11564 }
11565 }
11566
11567 pub async fn detect_upgrades(&self) -> Vec<crate::upgrade::UpgradeFinding> {
11571 let prefs = self.update_prefs();
11572 let curated = self.unified_registry().available_upgrades();
11573 let schemas = self.list_schemas();
11574 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11575 let probe = crate::upgrade::HuggingFaceProbe::new();
11576 let now = std::time::SystemTime::now()
11577 .duration_since(std::time::UNIX_EPOCH)
11578 .map(|d| d.as_secs())
11579 .unwrap_or(0);
11580 let mut findings = crate::upgrade::detect_upgrades(
11581 curated,
11582 &refs,
11583 &prefs,
11584 &probe,
11585 &crate::upgrade::UpgradeCache::default_path(),
11586 now,
11587 crate::upgrade::DEFAULT_TTL_SECS,
11588 )
11589 .await;
11590 let lane_models: std::collections::HashSet<String> = self
11595 .lane_defaults()
11596 .defaults
11597 .into_iter()
11598 .map(|d| d.model_id)
11599 .collect();
11600 let in_use = |m: &ModelSchema| {
11601 lane_models.contains(&m.id)
11602 || self.model_management.last_used(&m.id).is_some_and(|at| {
11603 now.saturating_sub(at) < portfolio::PortfolioPolicy::DEFAULT_IDLE_AFTER_SECS
11604 })
11605 };
11606 let installed: Vec<&ModelSchema> = refs
11607 .iter()
11608 .copied()
11609 .filter(|m| {
11610 m.is_local()
11611 && m.downloads_weights()
11612 && m.weights_ready
11613 && self.model_management.car_enabled(&m.id).unwrap_or(true)
11614 && in_use(m)
11615 })
11616 .collect();
11617 let fit = |m: &ModelSchema| {
11618 let fit = self.model_fit(m);
11619 fit.platform_compatible.then_some(fit.fit)
11620 };
11621 let registry = self.unified_registry();
11622 let found = crate::upgrade::suggest_only_unless_builtin(
11623 crate::upgrade::benchmark_findings(&installed, &refs, &fit),
11624 &|id| registry.is_builtin(id),
11625 );
11626 for finding in found {
11627 let duplicate = findings
11628 .iter()
11629 .any(|f| f.from_id == finding.from_id && f.to_id == finding.to_id);
11630 if !duplicate {
11631 findings.push(finding);
11632 }
11633 }
11634 findings
11635 }
11636
11637 pub fn list_schemas(&self) -> Vec<ModelSchema> {
11640 self.catalog_registry_snapshot()
11641 .list()
11642 .into_iter()
11643 .cloned()
11644 .collect()
11645 }
11646
11647 pub fn catalog_snapshot(&self) -> Result<CatalogSnapshot, String> {
11651 CatalogSnapshot::new(self.unified_registry().list().into_iter().cloned())
11652 }
11653
11654 pub fn registered_schema(&self, id: &str) -> Option<ModelSchema> {
11659 self.unified_registry().registered_schema(id).cloned()
11660 }
11661
11662 pub fn list_models(&self) -> Vec<models::ModelInfo> {
11663 self.registry.list_models()
11664 }
11665
11666 pub fn knows_model(&self, name: &str) -> bool {
11679 self.model_schema(name).is_some()
11680 }
11681
11682 pub fn model_schema(&self, name: &str) -> Option<ModelSchema> {
11700 self.resolve_schema(name)
11701 }
11702
11703 fn first_installed_local_model(&self, needs_tools: bool) -> Option<String> {
11716 let registry = self.unified_registry();
11724 let mut candidates: Vec<_> = registry
11725 .all()
11726 .filter(|s| s.is_local() && !s.is_vllm_mlx())
11733 .filter(|s| !s.is_foundation_models() || s.available)
11749 .filter(|s| s.has_capability(ModelCapability::Generate))
11750 .filter(|s| !needs_tools || s.has_capability(ModelCapability::ToolUse))
11754 .filter(|s| self.model_management.car_enabled(&s.id).unwrap_or(false))
11755 .filter(|s| self.unified_registry().ready_without_download(&s.id) == Some(true))
11756 .collect();
11757 candidates.sort_by_key(|s| s.size_mb());
11759
11760 if let Some(avail) = crate::hardware::available_ram_mb() {
11772 const WORKING_RESERVE_MB: u64 = 2048;
11773 candidates.retain(|s| s.size_mb().saturating_add(WORKING_RESERVE_MB) <= avail);
11774 }
11775
11776 candidates.first().map(|s| s.name.clone())
11777 }
11778
11779 pub async fn pull_model(&self, name: &str) -> Result<std::path::PathBuf, InferenceError> {
11781 self.pull_model_with_progress(name, &crate::download::ProgressSink::none())
11782 .await
11783 }
11784
11785 pub async fn pull_model_with_progress(
11790 &self,
11791 name: &str,
11792 sink: &crate::download::ProgressSink,
11793 ) -> Result<std::path::PathBuf, InferenceError> {
11794 self.with_pinned_registry(async {
11795 let pulled = self.pull_model_with_progress_unpinned(name, sink).await;
11796 if pulled.is_ok() {
11797 self.keep_pulled_row_if_withdrawn(name);
11798 }
11799 pulled
11800 })
11801 .await
11802 }
11803
11804 async fn pull_model_with_progress_unpinned(
11805 &self,
11806 name: &str,
11807 sink: &crate::download::ProgressSink,
11808 ) -> Result<std::path::PathBuf, InferenceError> {
11809 let schema = self
11810 .resolve_schema_name_first(name)
11811 .ok_or_else(|| InferenceError::ModelNotFound(name.to_string()))?;
11812 let _mutation = self.model_management.begin_mutation(&schema.id)?;
11813 if let Some(receipt) = self.model_management.load_receipt(&schema.id)? {
11814 let _ = self.model_management.can_remove(&schema.id)?;
11818 self.model_management.clear_tombstone(&schema.id)?;
11819 return Ok(receipt.managed_path);
11820 }
11821 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
11822 return Ok(receipt.managed_path);
11823 }
11824 let expected_managed = self.model_management.models_dir().join(&schema.name);
11825 if std::fs::symlink_metadata(&expected_managed).is_ok() {
11826 return Err(InferenceError::InferenceFailed(format!(
11827 "model {} has a pre-existing unreceipted artifact at {}; use models.adopt or move it before pulling",
11828 schema.id,
11829 expected_managed.display()
11830 )));
11831 }
11832 let staging = self.model_management.create_install_staging(&schema.id)?;
11833 let installed = match self
11834 .unified_registry()
11835 .ensure_local_with_progress_staged(&schema.id, sink, &staging)
11836 .await
11837 {
11838 Ok(installed) => installed,
11839 Err(error) => {
11840 self.model_management.discard_install_staging(&staging);
11841 return Err(error);
11842 }
11843 };
11844 let installed_is_staging = match (installed.canonicalize(), staging.canonicalize()) {
11845 (Ok(installed), Ok(staging)) => installed == staging,
11846 _ => false,
11847 };
11848 let generation = self
11849 .resource_policy_generation
11850 .load(std::sync::atomic::Ordering::Acquire);
11851 let managed_path = self.model_management.models_dir().join(&schema.name);
11852 let managed = if installed_is_staging {
11853 let receipt = self.model_management.install_receipt_for_publication(
11854 &schema.id,
11855 model_source_identity(&schema),
11856 generation,
11857 false,
11858 managed_path,
11859 model_management::ManagedArtifactKind::Directory,
11860 &staging,
11861 )?;
11862 self.model_management
11863 .begin_install_intent(receipt, Some(&staging))?;
11864 self.model_management
11865 .publish_install_staging(&schema.id, &staging, &schema.name)?
11866 } else {
11867 self.model_management.discard_install_staging(&staging);
11868 let receipt = self.model_management.install_receipt_for_publication(
11869 &schema.id,
11870 model_source_identity(&schema),
11871 generation,
11872 false,
11873 managed_path,
11874 model_management::ManagedArtifactKind::Symlink,
11875 &installed,
11876 )?;
11877 self.model_management.begin_install_intent(receipt, None)?;
11878 self.model_management
11879 .materialize_managed_projection(&schema.name, &installed)?
11880 };
11881 let receipt = self
11882 .model_management
11883 .resume_install_intent(&schema.id)?
11884 .ok_or_else(|| {
11885 InferenceError::InferenceFailed(format!(
11886 "model {} publication completed without a durable install intent",
11887 schema.id
11888 ))
11889 })?;
11890 debug_assert_eq!(receipt.managed_path, managed);
11891 Ok(receipt.managed_path)
11892 }
11893
11894 pub async fn adopt_model_into_car(
11898 &self,
11899 model_id: &str,
11900 ) -> Result<model_management::InstallReceipt, InferenceError> {
11901 let schema = self
11902 .resolve_schema(model_id)
11903 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
11904 if !schema.is_local() || !schema.downloads_weights() {
11905 return Err(InferenceError::InferenceFailed(format!(
11906 "model {} is not a CAR-manageable local artifact",
11907 schema.id
11908 )));
11909 }
11910 let existing = self
11911 .unified_registry()
11912 .existing_local_artifact(&schema.id)
11913 .ok_or_else(|| {
11914 InferenceError::InferenceFailed(format!(
11915 "model {} has no usable local artifact to adopt",
11916 schema.id
11917 ))
11918 })?;
11919 let _mutation = self.model_management.begin_mutation(&schema.id)?;
11920 let existing_is_managed_symlink = std::fs::symlink_metadata(&existing)
11921 .is_ok_and(|metadata| metadata.file_type().is_symlink())
11922 && existing.parent().is_some_and(|parent| {
11923 parent.canonicalize().ok() == self.model_management.models_dir().canonicalize().ok()
11924 });
11925 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
11926 return Ok(receipt);
11927 }
11928 let generation = self
11929 .resource_policy_generation
11930 .load(std::sync::atomic::Ordering::Acquire);
11931 let managed = if existing_is_managed_symlink {
11932 existing
11933 } else {
11934 let managed = self.model_management.adopted_projection_path(&schema.id)?;
11935 let receipt = self.model_management.install_receipt_for_publication(
11936 &schema.id,
11937 model_source_identity(&schema),
11938 generation,
11939 true,
11940 managed,
11941 model_management::ManagedArtifactKind::Symlink,
11942 &existing,
11943 )?;
11944 self.model_management.begin_install_intent(receipt, None)?;
11945 self.model_management
11946 .materialize_adopted_projection(&schema.id, &existing)?
11947 };
11948 if existing_is_managed_symlink {
11949 self.model_management
11950 .record_managed_artifact(
11951 &schema.id,
11952 model_source_identity(&schema),
11953 None,
11954 generation,
11955 true,
11956 managed,
11957 )
11958 .map_err(InferenceError::from)
11959 } else {
11960 self.model_management
11961 .resume_install_intent(&schema.id)?
11962 .ok_or_else(|| {
11963 InferenceError::InferenceFailed(format!(
11964 "model {} adoption completed without a durable install intent",
11965 schema.id
11966 ))
11967 })
11968 }
11969 }
11970
11971 pub async fn remove_model_from_car(
11974 &self,
11975 model_id: &str,
11976 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
11977 self.remove_model_from_car_inner(model_id, false).await
11978 }
11979
11980 async fn remove_model_from_car_inner(
11984 &self,
11985 model_id: &str,
11986 maintenance_held: bool,
11987 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
11988 let schema = self
11989 .resolve_schema(model_id)
11990 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
11991 if !schema.is_local() || !schema.downloads_weights() {
11992 return Err(InferenceError::InferenceFailed(format!(
11993 "model {} is not a CAR-owned local artifact",
11994 schema.id
11995 )));
11996 }
11997 if self.model_management.load_receipt(&schema.id)?.is_none() {
11998 if self.model_management.car_enabled(&schema.id)? {
11999 return Err(model_management::ModelManagementError::MissingReceipt {
12000 model_id: schema.id.clone(),
12001 }
12002 .into());
12003 }
12004 let generation = self
12008 .resource_policy_generation
12009 .load(std::sync::atomic::Ordering::Acquire)
12010 .saturating_add(1);
12011 return self
12012 .model_management
12013 .begin_mutation(&schema.id)?
12014 .remove(generation)
12015 .map_err(InferenceError::from);
12016 }
12017 let mutation = self.model_management.begin_mutation(&schema.id)?;
12018 let _maintenance = if maintenance_held {
12019 None
12020 } else {
12021 Some(
12022 self.prepare_local_model_removal(&schema.id)
12023 .await
12024 .map_err(|error| InferenceError::InferenceFailed(error.to_string()))?,
12025 )
12026 };
12027 let generation = self
12028 .resource_policy_generation
12029 .load(std::sync::atomic::Ordering::Acquire)
12030 .saturating_add(1);
12031 mutation.remove(generation).map_err(InferenceError::from)
12032 }
12033
12034 pub fn update_prefs(&self) -> crate::update_prefs::UpdatePreferences {
12039 let cwd = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("."));
12040 crate::update_prefs::UpdatePreferences::load_effective(&cwd).unwrap_or_else(|error| {
12041 tracing::warn!(%error, "update preferences unreadable; acting as notify");
12044 crate::update_prefs::UpdatePreferences {
12045 policy: crate::update_prefs::UpdatePolicy::Notify,
12046 policy_chosen: true,
12047 ..Default::default()
12048 }
12049 })
12050 }
12051
12052 pub fn set_update_prefs(
12054 &self,
12055 prefs: &crate::update_prefs::UpdatePreferences,
12056 ) -> Result<(), InferenceError> {
12057 prefs.save().map_err(InferenceError::InferenceFailed)
12058 }
12059
12060 #[deprecated(note = "use async remove_model_from_car for receipt-backed safe removal")]
12064 pub fn remove_model(&self, name: &str) -> Result<(), InferenceError> {
12065 Err(InferenceError::InferenceFailed(format!(
12066 "legacy removal for {name} is disabled; use async receipt-backed remove_model_from_car"
12067 )))
12068 }
12069
12070 pub fn register_model(&mut self, schema: ModelSchema) {
12076 self.unified_registry_mut().register(schema);
12077 }
12078
12079 pub fn register_user_model(&mut self, schema: ModelSchema) {
12081 self.unified_registry_mut().register_user_model(schema);
12082 }
12083
12084 pub async fn discover_vllm_mlx_models(&mut self) -> usize {
12087 let config = vllm_mlx::VllmMlxConfig::default();
12088 if !config.auto_discover {
12089 return 0;
12090 }
12091 vllm_mlx::discover_and_register(&config, self.unified_registry_mut()).await
12092 }
12093
12094 pub fn outcome_tracker(&self) -> Arc<RwLock<OutcomeTracker>> {
12096 self.outcome_tracker.clone()
12097 }
12098
12099 async fn auto_save_outcomes(&self) {
12107 const OUTCOME_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
12108
12109 if crate::offload::is_offload_worker() {
12117 return;
12118 }
12119
12120 let due = {
12122 let last = self.last_outcome_flush.lock().unwrap();
12123 last.is_none_or(|t| t.elapsed() >= OUTCOME_FLUSH_INTERVAL)
12124 };
12125 if due {
12126 match self.persist_outcomes().await {
12127 Ok(did) => {
12128 if did {
12129 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12130 }
12131 }
12132 Err(e) => tracing::debug!("auto-save outcomes failed: {}", e),
12133 }
12134 }
12135
12136 if let Err(e) = self.save_key_pool_stats().await {
12137 tracing::debug!("auto-save key pool stats failed: {}", e);
12138 }
12139 }
12140
12141 async fn persist_outcomes(&self) -> Result<bool, std::io::Error> {
12147 const PENDING_TTL_SECS: u64 = 300;
12148 let entries = {
12152 let mut tracker = self.outcome_tracker.write().await;
12153 tracker.sweep_pending(PENDING_TTL_SECS);
12154 tracker.drain_ledger()
12155 };
12156 let mut did = false;
12157 let ledger_disabled = std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some();
12162 if !entries.is_empty() && !ledger_disabled {
12163 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12164 let _guard = self.ledger_io_lock.lock().await;
12165 crate::outcome::append_ledger_entries(&ledger_path, &entries)?;
12166 did = true;
12167 }
12168 let profiles_path = self.config.state_models_dir().join("outcome_profiles.json");
12170 let wrote = {
12171 let mut tracker = self.outcome_tracker.write().await;
12172 tracker.save_if_dirty(&profiles_path)?
12173 };
12174 Ok(did || wrote)
12175 }
12176
12177 pub async fn save_outcomes(&self) -> Result<(), std::io::Error> {
12182 let tracker = self.outcome_tracker.read().await;
12183 let path = self.config.state_models_dir().join("outcome_profiles.json");
12184 tracker.save_to_file(&path)
12185 }
12186
12187 pub async fn flush_outcomes(&self) -> Result<bool, std::io::Error> {
12193 self.flush_outcomes_inner().await
12194 }
12195
12196 pub async fn reconcile_outcome_ledger(
12212 &self,
12213 ) -> Result<outcome::LedgerReconciliation, std::io::Error> {
12214 if crate::offload::is_offload_worker()
12215 || std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some()
12216 || !self.outcome_profiles_loaded
12217 {
12218 return Ok(outcome::LedgerReconciliation::default());
12219 }
12220 let models_dir = self.config.state_models_dir();
12221 let ledger = models_dir.join("outcome_ledger.jsonl");
12222 let _ = std::fs::remove_file(ledger.with_extension("jsonl.migrated"));
12225 self.flush_outcomes_inner().await?;
12226 let seeds = quality_seeds(&models_dir);
12227 let _io = self.ledger_io_lock.lock().await;
12228 let registry = &self.unified_registry();
12229 let is_registered = |id: &str| registry.get(id).is_some();
12230 let mut tracker = self.outcome_tracker.write().await;
12231 let plan = outcome::plan_reconcile(&ledger, &tracker, &is_registered, &seeds);
12232 if plan.ledger.is_none() && plan.profiles.is_none() {
12233 return Ok(plan.report);
12234 }
12235 if let Some(entries) = &plan.ledger {
12236 outcome::write_reconciled_ledger(&ledger, entries)?;
12237 }
12238 if let Some(profiles) = plan.profiles {
12239 tracker.commit_reconciled(profiles);
12240 tracker.save_to_file(&models_dir.join("outcome_profiles.json"))?;
12241 }
12242 tracing::info!(
12243 reclassified = plan.report.reclassified,
12244 unregistered = plan.report.unregistered,
12245 profiles_rebuilt = plan.report.profiles_rebuilt,
12246 profiles_removed = plan.report.profiles_removed,
12247 "reconciled the outcome ledger with model profiles"
12248 );
12249 Ok(plan.report)
12250 }
12251
12252 async fn flush_outcomes_inner(&self) -> Result<bool, std::io::Error> {
12253 let did = self.persist_outcomes().await?;
12254 if did {
12255 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12256 }
12257 Ok(did)
12258 }
12259
12260 pub async fn prune_outcome_ledger(&self, max_entries: usize) -> std::io::Result<()> {
12263 let path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12264 let _guard = self.ledger_io_lock.lock().await;
12265 crate::outcome::prune_ledger(&path, max_entries)
12266 }
12267
12268 pub async fn save_key_pool_stats(&self) -> Result<(), std::io::Error> {
12270 let path = self.config.state_models_dir().join("key_pool_stats.json");
12271 self.remote_backend.key_pool.save_stats(&path).await
12272 }
12273
12274 pub async fn key_pool_stats(
12276 &self,
12277 ) -> std::collections::HashMap<String, Vec<key_pool::KeyStats>> {
12278 self.remote_backend.key_pool.all_stats().await
12279 }
12280
12281 pub async fn export_profiles(&self) -> Vec<ModelProfile> {
12283 let tracker = self.outcome_tracker.read().await;
12284 tracker.export_profiles()
12285 }
12286
12287 pub fn outcome_scoreboard(&self) -> crate::scoreboard::Scoreboard {
12295 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12296 let entries = crate::outcome::read_ledger(&ledger_path, 0);
12297 crate::calibration::ShadowCalibration::from_ledger(&entries).emit();
12301 crate::scoreboard::Scoreboard::from_ledger(&entries, |id| {
12302 let s = self.resolve_schema(id)?;
12303 scoreboard_price_model(&s)
12304 })
12305 }
12306
12307 pub async fn import_profiles(&self, profiles: Vec<ModelProfile>) {
12309 let mut tracker = self.outcome_tracker.write().await;
12310 tracker.import_profiles(profiles);
12311 }
12312
12313 pub async fn prepare_speech_runtime(&self) -> Result<PathBuf, InferenceError> {
12336 self.ensure_speech_runtime().await.map(|r| r.root)
12342 }
12343
12344 pub fn set_speech_policy(&mut self, policy: SpeechPolicy) {
12346 self.speech_policy = policy;
12347 }
12348
12349 pub fn set_routing_config(&mut self, config: RoutingConfig) {
12350 self.adaptive_router.set_config(config);
12351 }
12352
12353 pub async fn install_curated_speech(
12355 &mut self,
12356 ) -> Result<Vec<SpeechInstallReport>, InferenceError> {
12357 if let Err(error) = self.prepare_speech_runtime().await {
12370 tracing::warn!(
12371 %error,
12372 "managed speech runtime could not be provisioned; installing the \
12373 models that do not depend on it"
12374 );
12375 }
12376 let schemas = self.list_schemas();
12377 let mut repos = Vec::new();
12378 for schema in &schemas {
12379 if !schema.is_mlx() || !schema.tags.iter().any(|tag| tag == "speech") {
12380 continue;
12381 }
12382 if !cfg!(all(
12397 target_os = "macos",
12398 target_arch = "aarch64",
12399 not(car_skip_mlx)
12400 )) {
12401 continue;
12402 }
12403 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
12404 if !repos.iter().any(|existing: &String| existing == hf_repo) {
12405 repos.push(hf_repo.clone());
12406 }
12407 }
12408 }
12409
12410 let mut installed = Vec::new();
12411 for repo in repos {
12412 let model_for_record = self
12413 .unified_registry()
12414 .all()
12415 .find(|m| matches!(&m.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo))
12416 .map(|m| m.id.clone())
12417 .unwrap_or_else(|| repo.clone());
12418 let _guard = crate::download::acquire_model_lock(&model_for_record).await;
12421 let (snapshot, fetched) =
12422 crate::retire::collecting_fetches(download_hf_repo_snapshot(&repo)).await;
12423 crate::retire::record_download(
12424 &self.config.state_root,
12425 &crate::hf_cache::hub_dir(),
12426 &model_for_record,
12427 &fetched,
12428 );
12429 let (snapshot_path, files_downloaded) = snapshot?;
12430 let name = schemas
12431 .iter()
12432 .find(|schema| {
12433 matches!(&schema.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo)
12434 })
12435 .map(|schema| schema.name.clone())
12436 .unwrap_or_else(|| repo.clone());
12437 installed.push(SpeechInstallReport {
12438 name,
12439 hf_repo: repo,
12440 snapshot_path,
12441 files_downloaded,
12442 });
12443 }
12444
12445 for schema in &schemas {
12449 if !schema.tags.iter().any(|tag| tag == "speech") {
12450 continue;
12451 }
12452 if let ModelSource::WhisperCpp { model } = &schema.source {
12453 let model = model.clone();
12454 let name = schema.name.clone();
12455 let path = tokio::task::spawn_blocking(move || car_whisper::ensure_model(&model))
12456 .await
12457 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))?
12458 .map_err(|e| {
12459 InferenceError::InferenceFailed(format!("whisper model fetch: {e}"))
12460 })?;
12461 installed.push(SpeechInstallReport {
12462 name,
12463 hf_repo: "ggerganov/whisper.cpp".to_string(),
12464 snapshot_path: path,
12465 files_downloaded: 1,
12466 });
12467 }
12468 }
12469
12470 self.unified_registry_mut().refresh_availability();
12471 Ok(installed)
12472 }
12473
12474 pub fn speech_health(&self) -> SpeechHealthReport {
12476 let local_stt_default =
12477 self.speech_health_default_name(ModelCapability::SpeechToText, true, false);
12478 let local_tts_default =
12479 self.speech_health_default_name(ModelCapability::TextToSpeech, true, false);
12480 let remote_stt_default =
12481 self.speech_health_default_name(ModelCapability::SpeechToText, false, true);
12482 let remote_tts_default =
12483 self.speech_health_default_name(ModelCapability::TextToSpeech, false, true);
12484
12485 let mut local_models = Vec::new();
12486 let mut remote_models = Vec::new();
12487 for schema in self.list_schemas() {
12488 let capability = if schema.has_capability(ModelCapability::SpeechToText) {
12489 Some(ModelCapability::SpeechToText)
12490 } else if schema.has_capability(ModelCapability::TextToSpeech) {
12491 Some(ModelCapability::TextToSpeech)
12492 } else {
12493 None
12494 };
12495 let Some(capability) = capability else {
12496 continue;
12497 };
12498
12499 let selected_by_default = local_stt_default
12500 .as_ref()
12501 .is_some_and(|name| name == &schema.name)
12502 || local_tts_default
12503 .as_ref()
12504 .is_some_and(|name| name == &schema.name)
12505 || remote_stt_default
12506 .as_ref()
12507 .is_some_and(|name| name == &schema.name)
12508 || remote_tts_default
12509 .as_ref()
12510 .is_some_and(|name| name == &schema.name);
12511
12512 let health = SpeechModelHealth {
12513 id: schema.id.clone(),
12514 name: schema.name.clone(),
12515 provider: schema.provider.clone(),
12516 capability,
12517 is_local: schema.is_local(),
12518 available: schema.available,
12519 cached: speech_model_cached(&schema),
12520 selected_by_default,
12521 source: speech_model_source_label(&schema),
12522 };
12523 if schema.is_local() {
12524 local_models.push(health);
12525 } else {
12526 remote_models.push(health);
12527 }
12528 }
12529
12530 let runtime = {
12537 let rt =
12538 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
12539 SpeechRuntimeHealth {
12540 root: rt.root.clone(),
12541 installed: rt.is_ready(),
12542 python: rt.python.clone(),
12543 stt_command: rt.stt_program.clone(),
12544 tts_command: rt.tts_program.clone(),
12545 configured_python: std::env::var("CAR_SPEECH_PYTHON")
12546 .ok()
12547 .filter(|value| !value.trim().is_empty()),
12548 detected_python: detect_speech_python(),
12549 }
12550 };
12551
12552 SpeechHealthReport {
12553 runtime,
12554 local_models,
12555 remote_models,
12556 elevenlabs_configured: crate::tasks::transcribe::provider_configured_for_passive_status(
12560 "ELEVENLABS_API_KEY",
12561 )
12562 || crate::tasks::synthesize::provider_configured_for_passive_status(
12563 "ELEVENLABS_API_KEY",
12564 ),
12565 prefer_local: self.speech_policy.prefer_local,
12566 allow_remote_fallback: self.speech_policy.allow_remote_fallback,
12567 preferred_local_stt: self.speech_policy.preferred_local_stt.clone(),
12568 preferred_local_tts: self.speech_policy.preferred_local_tts.clone(),
12569 preferred_remote_stt: self.speech_policy.preferred_remote_stt.clone(),
12570 preferred_remote_tts: self.speech_policy.preferred_remote_tts.clone(),
12571 local_stt_default,
12572 local_tts_default,
12573 remote_stt_default,
12574 remote_tts_default,
12575 }
12576 }
12577
12578 pub async fn model_health(&self) -> ModelHealthReport {
12581 self.model_health_with(false).await
12582 }
12583
12584 pub async fn model_health_for_explicit_status(&self) -> ModelHealthReport {
12594 self.model_health_with(true).await
12595 }
12596
12597 async fn model_health_with(&self, authoritative_credentials: bool) -> ModelHealthReport {
12598 let schemas: Vec<ModelSchema> = self
12599 .catalog_registry_snapshot_with(authoritative_credentials)
12600 .list()
12601 .into_iter()
12602 .cloned()
12603 .collect();
12604 let total_models = schemas.len();
12605 let available_models = schemas
12606 .iter()
12607 .filter(|schema| schema.available_now())
12608 .count();
12609 let local_models = schemas.iter().filter(|schema| schema.is_local()).count();
12610 let remote_models = total_models.saturating_sub(local_models);
12611
12612 let defaults = vec![
12613 self.model_default_health(
12614 ModelCapability::Generate,
12615 self.preferred_model_for_capability(ModelCapability::Generate)
12616 .unwrap_or(&self.config.generation_model),
12617 ),
12618 self.model_default_health(
12619 ModelCapability::Embed,
12620 self.preferred_model_for_capability(ModelCapability::Embed)
12621 .unwrap_or(&self.config.embedding_model),
12622 ),
12623 self.model_default_health(
12624 ModelCapability::Classify,
12625 self.preferred_model_for_capability(ModelCapability::Classify)
12626 .unwrap_or(&self.config.classification_model),
12627 ),
12628 ];
12629
12630 let credential_memo: std::sync::Mutex<std::collections::HashMap<String, bool>> =
12633 std::sync::Mutex::new(std::collections::HashMap::new());
12634 let credential = |env_var: &str| -> bool {
12635 if !authoritative_credentials {
12643 return std::env::var(env_var).is_ok_and(|value| !value.trim().is_empty());
12644 }
12645 let mut memo = credential_memo
12646 .lock()
12647 .unwrap_or_else(|poisoned| poisoned.into_inner());
12648 if let Some(known) = memo.get(env_var) {
12649 return *known;
12650 }
12651 let present = crate::registry::credential_present_without_reading_it(env_var);
12652 memo.insert(env_var.to_string(), present);
12653 present
12654 };
12655
12656 let mut providers = std::collections::BTreeMap::new();
12657 for schema in &schemas {
12658 let entry =
12659 providers
12660 .entry(schema.provider.clone())
12661 .or_insert_with(|| ProviderAccumulator {
12662 configured: false,
12663 local_models: 0,
12664 remote_models: 0,
12665 available_models: 0,
12666 capabilities: std::collections::HashSet::new(),
12667 });
12668
12669 entry.configured |= model_source_configured(schema, &credential);
12670 if schema.is_local() {
12671 entry.local_models += 1;
12672 } else {
12673 entry.remote_models += 1;
12674 }
12675 if schema.available_now() {
12676 entry.available_models += 1;
12677 }
12678 for capability in &schema.capabilities {
12679 entry.capabilities.insert(*capability);
12680 }
12681 }
12682
12683 let providers = providers
12684 .into_iter()
12685 .map(|(provider, acc)| ModelProviderHealth {
12686 provider,
12687 configured: acc.configured,
12688 local_models: acc.local_models,
12689 remote_models: acc.remote_models,
12690 available_models: acc.available_models,
12691 capabilities: sort_capabilities(acc.capabilities.into_iter().collect()),
12692 })
12693 .collect();
12694
12695 let capabilities = all_model_capabilities()
12696 .into_iter()
12697 .map(|capability| {
12698 let relevant: Vec<&ModelSchema> = schemas
12699 .iter()
12700 .filter(|schema| schema.has_capability(capability))
12701 .collect();
12702 let available: Vec<&ModelSchema> = relevant
12703 .iter()
12704 .copied()
12705 .filter(|schema| schema.available_now())
12706 .collect();
12707 ModelCapabilityHealth {
12708 capability,
12709 total_models: relevant.len(),
12710 available_models: available.len(),
12711 local_available_models: available
12712 .iter()
12713 .filter(|schema| schema.is_local())
12714 .count(),
12715 remote_available_models: available
12716 .iter()
12717 .filter(|schema| !schema.is_local())
12718 .count(),
12719 }
12720 })
12721 .collect();
12722
12723 let routing = self.routing_scenarios().await;
12724 let routing_config = self.adaptive_router.config().clone();
12725 let benchmark_priors =
12726 load_benchmark_prior_health(&self.config.state_models_dir(), &schemas);
12727
12728 ModelHealthReport {
12729 total_models,
12730 available_models,
12731 local_models,
12732 remote_models,
12733 defaults,
12734 providers,
12735 capabilities,
12736 routing_prefer_local: routing_config.prefer_local,
12737 routing_quality_first_cold_start: routing_config.quality_first_cold_start,
12738 routing_min_observations: routing_config.min_observations,
12739 routing_bootstrap_min_task_observations: routing_config.bootstrap_min_task_observations,
12740 routing_bootstrap_quality_floor: routing_config.bootstrap_quality_floor,
12741 routing_quality_weight: routing_config.quality_weight,
12742 routing_latency_weight: routing_config.latency_weight,
12743 routing_cost_weight: routing_config.cost_weight,
12744 routing_scenarios: routing,
12745 benchmark_priors,
12746 speech: self.speech_health(),
12747 }
12748 }
12749
12750 async fn routing_scenarios(&self) -> Vec<RoutingScenarioHealth> {
12751 let tracker = self.outcome_tracker.read().await;
12752 let config = self.adaptive_router.config().clone();
12753 let scenarios = [
12754 (
12755 "interactive_text",
12756 "Summarize the benefits of local-first AI routing in two sentences.",
12757 "text",
12758 RoutingWorkload::Interactive,
12759 false,
12760 false,
12761 ),
12762 (
12763 "background_code",
12764 "Write a Python function named fibonacci(n) that returns the nth Fibonacci number.",
12765 "code",
12766 RoutingWorkload::Background,
12767 false,
12768 false,
12769 ),
12770 (
12771 "interactive_tool_use",
12772 "Use the provided weather tool to get the weather for Boston.",
12773 "tool_use",
12774 RoutingWorkload::Interactive,
12775 true,
12776 false,
12777 ),
12778 (
12779 "interactive_vision",
12780 "What is in this image? Answer in one word.",
12781 "vision",
12782 RoutingWorkload::Interactive,
12783 false,
12784 true,
12785 ),
12786 ];
12787
12788 let routing_registry = self.catalog_registry_snapshot();
12795
12796 scenarios
12797 .into_iter()
12798 .map(
12799 |(name, prompt, task_family, workload, has_tools, has_vision)| {
12800 let decision = self.adaptive_router.route_context_aware(
12801 prompt,
12802 0,
12803 &routing_registry,
12804 &tracker,
12805 has_tools,
12806 has_vision,
12807 workload,
12808 );
12809 let quality_first_cold_start = if has_tools || has_vision {
12810 config.quality_first_cold_start
12811 } else if task_family == "code"
12812 && matches!(workload, RoutingWorkload::Background)
12813 {
12814 false
12815 } else {
12816 config.quality_first_cold_start
12817 };
12818 RoutingScenarioHealth {
12819 name: name.to_string(),
12820 task_family: task_family.to_string(),
12821 workload,
12822 has_tools,
12823 has_vision,
12824 prefer_local: if task_family == "speech" {
12825 self.speech_policy.prefer_local
12826 } else {
12827 config.prefer_local
12828 },
12829 quality_first_cold_start,
12830 bootstrap_min_task_observations: config.bootstrap_min_task_observations,
12831 bootstrap_quality_floor: config.bootstrap_quality_floor,
12832 model_id: decision.model_id,
12833 model_name: decision.model_name,
12834 reason: decision.reason,
12835 strategy: decision.strategy,
12836 }
12837 },
12838 )
12839 .collect()
12840 }
12841
12842 pub async fn smoke_test_speech(
12844 &self,
12845 local: bool,
12846 remote: bool,
12847 ) -> Result<SpeechSmokeReport, InferenceError> {
12848 let mut report = SpeechSmokeReport::default();
12849
12850 if local {
12851 let tts = self
12852 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
12853 .ok_or_else(|| {
12854 InferenceError::InferenceFailed(
12855 "no local text-to-speech model available".into(),
12856 )
12857 })?;
12858 let stt = self
12859 .preferred_speech_schema(ModelCapability::SpeechToText, true, false)
12860 .ok_or_else(|| {
12861 InferenceError::InferenceFailed(
12862 "no local speech-to-text model available".into(),
12863 )
12864 })?;
12865 report.local = Some(
12866 self.run_speech_smoke_path("local", &tts, &stt, "Testing CAR local speech path.")
12867 .await?,
12868 );
12869 } else {
12870 report.skipped.push("local".to_string());
12871 }
12872
12873 if remote {
12874 let tts = self
12875 .preferred_speech_schema(ModelCapability::TextToSpeech, false, true)
12876 .ok_or_else(|| {
12877 InferenceError::InferenceFailed(
12878 "no remote text-to-speech model available".into(),
12879 )
12880 })?;
12881 let stt = self
12882 .preferred_speech_schema(ModelCapability::SpeechToText, false, true)
12883 .ok_or_else(|| {
12884 InferenceError::InferenceFailed(
12885 "no remote speech-to-text model available".into(),
12886 )
12887 })?;
12888 report.remote = Some(
12889 self.run_speech_smoke_path("remote", &tts, &stt, "Testing CAR remote speech path.")
12890 .await?,
12891 );
12892 } else {
12893 report.skipped.push("remote".to_string());
12894 }
12895
12896 Ok(report)
12897 }
12898
12899 fn speech_candidates(
12900 &self,
12901 capability: ModelCapability,
12902 explicit: Option<&str>,
12903 ) -> Result<Vec<ModelSchema>, InferenceError> {
12904 if let Some(model) = explicit {
12905 let schema = self
12906 .resolve_schema(model)
12907 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
12908 if !schema.has_capability(capability) {
12909 return Err(InferenceError::InferenceFailed(format!(
12910 "model {} does not support {:?}",
12911 schema.name, capability
12912 )));
12913 }
12914 return Ok(vec![schema]);
12915 }
12916
12917 let mut candidates: Vec<ModelSchema> = self
12918 .unified_registry()
12919 .query(&ModelFilter {
12920 capabilities: vec![capability],
12921 ..Default::default()
12922 })
12923 .into_iter()
12924 .cloned()
12925 .collect();
12926
12927 if candidates.is_empty() {
12928 return Err(InferenceError::InferenceFailed(format!(
12929 "no models registered for capability {:?}",
12930 capability
12931 )));
12932 }
12933
12934 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
12935 if !self.speech_policy.allow_remote_fallback
12936 && candidates.iter().any(|model| model.is_local())
12937 {
12938 candidates.retain(|model| model.is_local());
12939 }
12940
12941 Ok(candidates)
12942 }
12943
12944 #[allow(dead_code)] fn resolve_external_hf_repo(
12950 &self,
12951 explicit: Option<&str>,
12952 capability: ModelCapability,
12953 ) -> Option<String> {
12954 let id = explicit?;
12955 let schema = self.resolve_schema(id)?;
12956 if !schema.has_capability(capability) {
12957 return Some(id.to_string());
12958 }
12959 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
12960 return Some(hf_repo.clone());
12961 }
12962 Some(id.to_string())
12963 }
12964
12965 fn media_generation_candidates(
12966 &self,
12967 capability: ModelCapability,
12968 explicit: Option<&str>,
12969 ) -> Result<Vec<ModelSchema>, InferenceError> {
12970 if let Some(model) = explicit {
12971 let schema = self
12972 .resolve_schema(model)
12973 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
12974 if !schema.has_capability(capability) {
12975 return Err(InferenceError::InferenceFailed(format!(
12976 "model {} does not support {:?}",
12977 schema.name, capability
12978 )));
12979 }
12980 return Ok(vec![schema]);
12981 }
12982
12983 let mut candidates: Vec<ModelSchema> = self
12984 .unified_registry()
12985 .query(&ModelFilter {
12986 capabilities: vec![capability],
12987 local_only: true,
12988 ..Default::default()
12989 })
12990 .into_iter()
12991 .cloned()
12992 .collect();
12993 candidates.sort_by_key(|schema| (!schema.available, schema.size_mb()));
12994 if candidates.is_empty() {
12995 return Err(InferenceError::InferenceFailed(format!(
12996 "no models registered for capability {:?}",
12997 capability
12998 )));
12999 }
13000 Ok(candidates)
13001 }
13002
13003 fn preferred_speech_schema(
13004 &self,
13005 capability: ModelCapability,
13006 local_only: bool,
13007 remote_only: bool,
13008 ) -> Option<ModelSchema> {
13009 let available_only = remote_only;
13010 let mut candidates: Vec<ModelSchema> = self
13011 .unified_registry()
13012 .query(&ModelFilter {
13013 capabilities: vec![capability],
13014 available_only,
13015 ..Default::default()
13016 })
13017 .into_iter()
13018 .filter(|schema| {
13019 (!local_only || schema.is_local()) && (!remote_only || schema.is_remote())
13020 })
13021 .cloned()
13022 .collect();
13023 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
13024 candidates.into_iter().next()
13025 }
13026
13027 fn speech_health_default_name(
13028 &self,
13029 capability: ModelCapability,
13030 local_only: bool,
13031 remote_only: bool,
13032 ) -> Option<String> {
13033 let preferred = match capability {
13034 ModelCapability::SpeechToText if local_only => {
13035 self.speech_policy.preferred_local_stt.as_ref()
13036 }
13037 ModelCapability::SpeechToText if remote_only => {
13038 self.speech_policy.preferred_remote_stt.as_ref()
13039 }
13040 ModelCapability::TextToSpeech if local_only => {
13041 self.speech_policy.preferred_local_tts.as_ref()
13042 }
13043 ModelCapability::TextToSpeech if remote_only => {
13044 self.speech_policy.preferred_remote_tts.as_ref()
13045 }
13046 _ => None,
13047 };
13048
13049 preferred
13050 .filter(|name| {
13051 self.unified_registry().list().iter().any(|schema| {
13052 schema.name == **name
13053 && schema.has_capability(capability)
13054 && (!local_only || schema.is_local())
13055 && (!remote_only || schema.is_remote())
13056 })
13057 })
13058 .cloned()
13059 .or_else(|| {
13060 self.preferred_speech_schema(capability, local_only, remote_only)
13061 .map(|schema| schema.name)
13062 })
13063 }
13064
13065 fn model_default_health(
13066 &self,
13067 capability: ModelCapability,
13068 configured_model: &str,
13069 ) -> ModelDefaultHealth {
13070 let schema = self.resolve_schema_name_first(configured_model);
13071
13072 ModelDefaultHealth {
13073 capability,
13074 configured_model: configured_model.to_string(),
13075 available: schema.as_ref().is_some_and(ModelSchema::available_now),
13076 is_local: schema.as_ref().is_some_and(ModelSchema::is_local),
13077 provider: schema.map(|model| model.provider),
13078 }
13079 }
13080
13081 fn speech_sort_key(
13082 &self,
13083 capability: ModelCapability,
13084 model: &ModelSchema,
13085 ) -> (u8, u8, u8, u8, u64, u64) {
13086 let policy_preference = match capability {
13087 ModelCapability::SpeechToText if model.is_local() => {
13088 self.speech_policy.preferred_local_stt.as_ref()
13089 }
13090 ModelCapability::SpeechToText => self.speech_policy.preferred_remote_stt.as_ref(),
13091 ModelCapability::TextToSpeech if model.is_local() => {
13092 self.speech_policy.preferred_local_tts.as_ref()
13093 }
13094 ModelCapability::TextToSpeech => self.speech_policy.preferred_remote_tts.as_ref(),
13095 _ => None,
13096 };
13097 let local_rank = if self.speech_policy.prefer_local {
13098 if model.is_local() {
13099 0
13100 } else {
13101 1
13102 }
13103 } else if model.is_remote() {
13104 0
13105 } else {
13106 1
13107 };
13108 let availability_rank = if model.available {
13109 0
13110 } else if model.is_local() {
13111 1
13112 } else {
13113 2
13114 };
13115 let policy_rank: u8 = if policy_preference.is_some_and(|preferred| preferred == &model.name)
13116 {
13117 0
13118 } else {
13119 1
13120 };
13121 let speech_rank = match capability {
13122 ModelCapability::TextToSpeech => {
13138 if model.name == "Kokoro-82M-bf16" {
13139 0
13140 } else if model.name == "Kokoro-82M-6bit" {
13141 1
13142 } else if model.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
13143 3
13146 } else {
13147 2
13148 }
13149 }
13150 ModelCapability::SpeechToText => {
13151 if model.name == "Parakeet-TDT-0.6B-v3-MLX" {
13152 0
13153 } else {
13154 1
13155 }
13156 }
13157 _ => 0,
13158 };
13159 let latency_rank = model.performance.latency_p50_ms.unwrap_or(u64::MAX);
13160 let size_rank = model.cost.size_mb.unwrap_or(u64::MAX);
13161 (
13162 local_rank,
13163 availability_rank,
13164 policy_rank,
13165 speech_rank,
13166 latency_rank,
13167 size_rank,
13168 )
13169 }
13170
13171 async fn run_speech_smoke_path(
13172 &self,
13173 path: &str,
13174 tts: &ModelSchema,
13175 stt: &ModelSchema,
13176 text: &str,
13177 ) -> Result<SpeechSmokePathReport, InferenceError> {
13178 let work_dir = temp_work_dir(&format!("speech-smoke-{path}"))?;
13179 let audio_path = work_dir.join(format!("{path}.wav"));
13180 let synth = self
13181 .synthesize(SynthesizeRequest {
13182 text: text.to_string(),
13183 model: Some(tts.name.clone()),
13184 voice: default_speech_voice(tts),
13185 language: Some("en".to_string()),
13186 output_path: Some(audio_path.display().to_string()),
13187 ..SynthesizeRequest::default()
13188 })
13189 .await?;
13190 let transcript = self
13191 .transcribe(TranscribeRequest {
13192 audio_path: synth.audio_path.clone(),
13193 model: Some(stt.name.clone()),
13194 language: Some("en".to_string()),
13195 prompt: None,
13196 timestamps: false,
13197 })
13198 .await?;
13199
13200 Ok(SpeechSmokePathReport {
13201 path: path.to_string(),
13202 tts_model: synth.model_used.unwrap_or_else(|| tts.name.clone()),
13203 stt_model: transcript.model_used.unwrap_or_else(|| stt.name.clone()),
13204 audio_path: PathBuf::from(synth.audio_path),
13205 transcript: transcript.text,
13206 })
13207 }
13208
13209 async fn ensure_speech_runtime(&self) -> Result<SpeechRuntime, InferenceError> {
13210 let mut guard = self.speech_runtime.lock().await;
13211 if let Some(runtime) = guard.as_ref() {
13212 if runtime.is_ready() {
13213 return Ok(runtime.clone());
13214 }
13215 }
13216
13217 let runtime =
13218 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
13219 if !runtime.is_ready() {
13220 bootstrap_speech_runtime(&runtime).await?;
13221 }
13222 if !runtime.is_ready() {
13223 return Err(InferenceError::InferenceFailed(format!(
13224 "managed speech runtime is not ready at {}",
13225 runtime.root.display()
13226 )));
13227 }
13228
13229 *guard = Some(runtime.clone());
13230 Ok(runtime)
13231 }
13232
13233 async fn transcribe_whisper(
13242 &self,
13243 schema: &ModelSchema,
13244 model: &str,
13245 req: &TranscribeRequest,
13246 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13247 ) -> Result<TranscribeResult, InferenceError> {
13248 let model = model.to_string();
13249 let model_path = car_whisper::ensure_model(&model)
13250 .map_err(|e| InferenceError::InferenceFailed(format!("whisper download: {e}")))?;
13251 let measured_bytes = backend_cache::estimate_model_size(&model_path);
13252 let reservation = reservation.ok_or_else(|| {
13253 InferenceError::InferenceFailed("local Whisper path missing admission".into())
13254 })?;
13255 reservation
13256 .reconcile_measured_weights(measured_bytes)
13257 .map_err(InferenceError::from)?;
13258 let detached_lease = reservation.detached_lease();
13259 let language = req.language.clone().unwrap_or_else(|| "auto".to_string());
13261 let audio_path = std::path::PathBuf::from(&req.audio_path);
13262 let name = schema.name.clone();
13263 let req_language = req.language.clone();
13264 let text =
13265 run_admitted_blocking(detached_lease, move || -> Result<String, InferenceError> {
13266 let stt = car_whisper::WhisperStt::load_from_path(&model_path, &language)
13268 .map_err(|e| InferenceError::InferenceFailed(format!("whisper load: {e}")))?;
13269 stt.transcribe_file(&audio_path).map_err(|e| {
13270 InferenceError::InferenceFailed(format!("whisper transcribe: {e}"))
13271 })
13272 })
13273 .await
13274 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))??;
13275 Ok(TranscribeResult::text_only(text, Some(name), req_language))
13276 }
13277
13278 async fn transcribe_local_mlx(
13279 &self,
13280 schema: &ModelSchema,
13281 req: &TranscribeRequest,
13282 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13283 ) -> Result<TranscribeResult, InferenceError> {
13284 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13285 let reservation = reservation.ok_or_else(|| {
13286 InferenceError::InferenceFailed("local STT path missing admission".into())
13287 })?;
13288 reservation
13289 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13290 .map_err(InferenceError::from)?;
13291 self.transcribe_via_speech_runtime(schema, req, reservation.detached_lease())
13300 .await
13301 }
13302
13303 async fn transcribe_via_speech_runtime(
13310 &self,
13311 schema: &ModelSchema,
13312 req: &TranscribeRequest,
13313 detached_lease: resource_policy::DetachedLocalLease,
13314 ) -> Result<TranscribeResult, InferenceError> {
13315 {
13316 let runtime = self.ensure_speech_runtime().await?;
13317 let hf_repo = match &schema.source {
13318 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13319 _ => {
13320 return Err(InferenceError::InferenceFailed(format!(
13321 "speech runtime needs an MLX model repo; {} is not one",
13322 schema.id
13323 )))
13324 }
13325 };
13326 let output_dir = temp_work_dir("stt")?;
13327 let output_prefix = output_dir.join("transcript");
13328 let mut args = vec![
13329 "--model".to_string(),
13330 hf_repo,
13331 "--audio".to_string(),
13332 req.audio_path.clone(),
13333 "--output-path".to_string(),
13334 output_prefix.display().to_string(),
13335 "--format".to_string(),
13336 "json".to_string(),
13337 ];
13338 if let Some(language) = &req.language {
13339 args.push("--language".to_string());
13340 args.push(normalize_lang_code(language));
13341 }
13342 if let Some(prompt) = &req.prompt {
13343 args.push("--context".to_string());
13344 args.push(prompt.clone());
13345 }
13346 if req.timestamps {
13347 args.push("--verbose".to_string());
13348 }
13349
13350 let output =
13351 run_mlx_audio_command(&runtime, "stt.generate", &args, detached_lease).await?;
13352 let text = read_transcription_result(&output_prefix)?
13353 .or_else(|| extract_text_from_payload(&output.stdout))
13354 .ok_or_else(|| {
13355 InferenceError::InferenceFailed(format!(
13356 "mlx-audio transcription returned no text: {}",
13357 output.stderr
13358 ))
13359 })?;
13360
13361 Ok(TranscribeResult {
13362 text,
13363 model_used: Some(schema.name.clone()),
13364 language: req.language.clone(),
13365 words: Vec::new(),
13366 routing_explanation: None,
13367 })
13368 }
13369 }
13370
13371 async fn synthesize_local_mlx(
13372 &self,
13373 schema: &ModelSchema,
13374 req: &SynthesizeRequest,
13375 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13376 ) -> Result<SynthesizeResult, InferenceError> {
13377 let requested = req.requested_advanced_controls();
13382 let repo_supports_advanced = match &schema.source {
13383 ModelSource::Mlx { hf_repo, .. } => hf_repo.to_ascii_lowercase().contains("qwen3-tts"),
13384 _ => false,
13385 };
13386 if !requested.is_empty() && !repo_supports_advanced {
13387 if req.strict_capabilities {
13388 return Err(InferenceError::InferenceFailed(format!(
13389 "model {name} does not support Qwen3-TTS advanced controls {requested:?}; \
13390 route to a Qwen3-TTS model or set strict_capabilities = false to degrade",
13391 name = schema.name,
13392 )));
13393 }
13394 tracing::warn!(
13395 model = %schema.name,
13396 fields = ?requested,
13397 "Qwen3-TTS advanced controls set on non-Qwen3-TTS backend — ignored \
13398 (set strict_capabilities=true to error instead)"
13399 );
13400 }
13401
13402 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13403 let reservation = reservation.ok_or_else(|| {
13404 InferenceError::InferenceFailed("local TTS path missing admission".into())
13405 })?;
13406 reservation
13407 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13408 .map_err(InferenceError::from)?;
13409
13410 self.synthesize_via_speech_runtime(schema, req, reservation.detached_lease())
13420 .await
13421 }
13422
13423 async fn synthesize_via_speech_runtime(
13436 &self,
13437 schema: &ModelSchema,
13438 req: &SynthesizeRequest,
13439 detached_lease: resource_policy::DetachedLocalLease,
13440 ) -> Result<SynthesizeResult, InferenceError> {
13441 let runtime = self.ensure_speech_runtime().await?;
13442 let primary_hf_repo = match &schema.source {
13443 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13444 _ => {
13445 return Err(InferenceError::InferenceFailed(format!(
13446 "speech runtime needs an MLX model repo; {} is not one",
13447 schema.id
13448 )))
13449 }
13450 };
13451 let (produced, model_used) = match self
13452 .synthesize_local_mlx_repo(
13453 &runtime,
13454 &primary_hf_repo,
13455 schema.name.as_str(),
13456 req,
13457 detached_lease.clone(),
13458 )
13459 .await
13460 {
13461 Ok(result) => result,
13462 Err(primary_err)
13463 if primary_hf_repo == "mlx-community/Kokoro-82M-6bit"
13464 && kokoro_runtime_fallback_enabled() =>
13465 {
13466 let fallback_repo = "mlx-community/Kokoro-82M-bf16";
13467 let fallback_name = "Kokoro-82M-bf16";
13468 match self
13469 .synthesize_local_mlx_repo(
13470 &runtime,
13471 fallback_repo,
13472 fallback_name,
13473 req,
13474 detached_lease.clone(),
13475 )
13476 .await
13477 {
13478 Ok(result) => result,
13479 Err(fallback_err) => {
13480 return Err(InferenceError::InferenceFailed(format!(
13481 "{primary_err}; fallback {fallback_name} also failed: {fallback_err}"
13482 )));
13483 }
13484 }
13485 }
13486 Err(err) => return Err(err),
13487 };
13488 let final_path =
13489 materialize_audio_output(&produced, req.output_path.as_deref(), &req.format)?;
13490
13491 Ok(SynthesizeResult {
13492 audio_path: final_path.display().to_string(),
13493 media_type: media_type_for_format(&req.format),
13494 model_used: Some(model_used),
13495 voice_used: req.voice.clone(),
13496 routing_explanation: None,
13497 })
13498 }
13499
13500 async fn synthesize_local_mlx_repo(
13501 &self,
13502 runtime: &SpeechRuntime,
13503 hf_repo: &str,
13504 model_name: &str,
13505 req: &SynthesizeRequest,
13506 detached_lease: resource_policy::DetachedLocalLease,
13507 ) -> Result<(PathBuf, String), InferenceError> {
13508 let output_dir = temp_work_dir("tts")?;
13509 let mut args = vec![
13510 "--model".to_string(),
13511 hf_repo.to_string(),
13512 "--text".to_string(),
13513 req.text.clone(),
13514 "--output_path".to_string(),
13515 output_dir.display().to_string(),
13516 ];
13517 if let Some(voice) = &req.voice {
13518 args.push("--voice".to_string());
13519 args.push(voice.clone());
13520 }
13521 if let Some(speed) = req.speed {
13522 args.push("--speed".to_string());
13523 args.push(speed.to_string());
13524 }
13525 let repo_lower = hf_repo.to_ascii_lowercase();
13526 if repo_lower.contains("kokoro") {
13527 args.push("--lang_code".to_string());
13528 args.push(kokoro_lang_code(req.language.as_deref()).to_string());
13529 } else if let Some(language) = &req.language {
13530 args.push("--lang_code".to_string());
13531 args.push(normalize_lang_code(language));
13532 }
13533
13534 if repo_lower.contains("qwen3-tts") {
13540 if let Some(ref_audio) = &req.reference_audio_path {
13541 args.push("--ref_audio".to_string());
13542 args.push(ref_audio.clone());
13543 }
13544 if let Some(ref_text) = &req.reference_text {
13545 args.push("--ref_text".to_string());
13546 args.push(ref_text.clone());
13547 }
13548 if let Some(instruct) = &req.voice_instruction {
13549 args.push("--instruct".to_string());
13550 args.push(instruct.clone());
13551 }
13552 }
13553
13554 let output = if repo_lower.contains("kokoro") {
13555 let device = std::env::var("CAR_SPEECH_KOKORO_DEVICE")
13556 .or_else(|_| std::env::var("CAR_SPEECH_MLX_DEVICE"))
13557 .unwrap_or_else(|_| "cpu".to_string());
13558 let extra_env = vec![
13559 ("MLX_DEVICE".to_string(), device),
13561 ("PYTORCH_ENABLE_MPS_FALLBACK".to_string(), "1".to_string()),
13563 ];
13564 run_mlx_audio_command_with_env(
13565 runtime,
13566 "tts.generate",
13567 &args,
13568 &extra_env,
13569 detached_lease,
13570 )
13571 .await?
13572 } else {
13573 run_mlx_audio_command(runtime, "tts.generate", &args, detached_lease).await?
13574 };
13575 let produced = find_audio_file(&output_dir)?.ok_or_else(|| {
13576 let hint = if repo_lower.contains("kokoro") {
13577 ". Kokoro models may crash on GPU — try CAR_SPEECH_KOKORO_DEVICE=cpu or use the default Qwen3-TTS model"
13578 } else {
13579 ""
13580 };
13581 InferenceError::InferenceFailed(format!(
13582 "mlx-audio synthesis produced no audio file: {}{}",
13583 output.stderr, hint
13584 ))
13585 })?;
13586 Ok((produced, model_name.to_string()))
13587 }
13588
13589 async fn transcribe_elevenlabs(
13590 &self,
13591 schema: &ModelSchema,
13592 req: &TranscribeRequest,
13593 ) -> Result<TranscribeResult, InferenceError> {
13594 let (endpoint, api_key) = elevenlabs_auth(
13595 schema,
13596 crate::tasks::transcribe::resolve_provider_credential_for_request,
13597 )?;
13598 let file_name = Path::new(&req.audio_path)
13599 .file_name()
13600 .and_then(|f| f.to_str())
13601 .unwrap_or("audio.wav")
13602 .to_string();
13603 let audio_bytes = tokio::fs::read(&req.audio_path).await?;
13604 let file_part = Part::bytes(audio_bytes).file_name(file_name);
13605 let mut form = Form::new()
13606 .text("model_id", schema.name.clone())
13607 .part("file", file_part);
13608 if let Some(language) = &req.language {
13609 form = form.text("language_code", language.clone());
13610 }
13611
13612 let resp = self
13613 .remote_backend
13614 .client
13615 .post(format!(
13616 "{}/v1/speech-to-text",
13617 endpoint.trim_end_matches('/')
13618 ))
13619 .header("xi-api-key", api_key)
13620 .multipart(form)
13621 .send()
13622 .await
13623 .map_err(|e| {
13624 self.remote_backend
13625 .request_error("ElevenLabs STT request failed", &e)
13626 })?;
13627 let status = resp.status();
13628 let body = resp.text().await.map_err(|e| {
13629 InferenceError::InferenceFailed(format!("read ElevenLabs STT body: {e}"))
13630 })?;
13631 if !status.is_success() {
13632 return Err(InferenceError::InferenceFailed(format!(
13633 "ElevenLabs STT returned {status}: {body}"
13634 )));
13635 }
13636 let payload: serde_json::Value = serde_json::from_str(&body).map_err(|e| {
13637 InferenceError::InferenceFailed(format!("parse ElevenLabs STT response: {e}"))
13638 })?;
13639 let text = payload
13640 .get("text")
13641 .and_then(|v| v.as_str())
13642 .map(str::to_string)
13643 .ok_or_else(|| {
13644 InferenceError::InferenceFailed("ElevenLabs STT response missing text".into())
13645 })?;
13646
13647 Ok(TranscribeResult {
13648 text,
13649 model_used: Some(schema.name.clone()),
13650 language: payload
13651 .get("language_code")
13652 .and_then(|v| v.as_str())
13653 .map(str::to_string),
13654 words: Vec::new(),
13655 routing_explanation: None,
13656 })
13657 }
13658
13659 async fn synthesize_elevenlabs(
13660 &self,
13661 schema: &ModelSchema,
13662 req: &SynthesizeRequest,
13663 ) -> Result<SynthesizeResult, InferenceError> {
13664 let requested = req.requested_advanced_controls();
13668 if !requested.is_empty() {
13669 if req.strict_capabilities {
13670 return Err(InferenceError::InferenceFailed(format!(
13671 "ElevenLabs backend does not support Qwen3-TTS advanced controls \
13672 {requested:?}; route to a Qwen3-TTS model or set strict_capabilities = false"
13673 )));
13674 }
13675 tracing::warn!(
13676 model = %schema.name,
13677 fields = ?requested,
13678 "Qwen3-TTS advanced controls ignored by ElevenLabs backend"
13679 );
13680 }
13681 let (endpoint, api_key) = elevenlabs_auth(
13682 schema,
13683 crate::tasks::synthesize::resolve_provider_credential_for_request,
13684 )?;
13685 let voice_id = req
13686 .voice
13687 .clone()
13688 .unwrap_or_else(|| "JBFqnCBsd6RMkjVDRZzb".to_string());
13689 let output_format = elevenlabs_output_format(&req.format);
13690 let url = format!(
13691 "{}/v1/text-to-speech/{}?output_format={}",
13692 endpoint.trim_end_matches('/'),
13693 voice_id,
13694 output_format
13695 );
13696
13697 let mut body = serde_json::json!({
13698 "text": req.text,
13699 "model_id": schema.name,
13700 });
13701 if let Some(language) = &req.language {
13702 body["language_code"] = serde_json::Value::String(language.clone());
13703 }
13704
13705 let resp = self
13706 .remote_backend
13707 .client
13708 .post(url)
13709 .header("xi-api-key", api_key)
13710 .header("Content-Type", "application/json")
13711 .json(&body)
13712 .send()
13713 .await
13714 .map_err(|e| {
13715 self.remote_backend
13716 .request_error("ElevenLabs TTS request failed", &e)
13717 })?;
13718 let status = resp.status();
13719 let audio = resp.bytes().await.map_err(|e| {
13720 InferenceError::InferenceFailed(format!("read ElevenLabs TTS body: {e}"))
13721 })?;
13722 if !status.is_success() {
13723 let err_body = String::from_utf8_lossy(&audio);
13724 return Err(InferenceError::InferenceFailed(format!(
13725 "ElevenLabs TTS returned {status}: {err_body}"
13726 )));
13727 }
13728
13729 let final_path = requested_or_temp_output(req.output_path.as_deref(), &req.format)?;
13730 ensure_parent_dir(&final_path)?;
13731 tokio::fs::write(&final_path, &audio).await?;
13732
13733 Ok(SynthesizeResult {
13734 audio_path: final_path.display().to_string(),
13735 media_type: media_type_for_format(&req.format),
13736 model_used: Some(schema.name.clone()),
13737 voice_used: Some(voice_id),
13738 routing_explanation: None,
13739 })
13740 }
13741}
13742
13743#[derive(Default)]
13744struct ProviderAccumulator {
13745 configured: bool,
13746 local_models: usize,
13747 remote_models: usize,
13748 available_models: usize,
13749 capabilities: std::collections::HashSet<ModelCapability>,
13750}
13751
13752struct CommandOutput {
13757 stdout: String,
13758 stderr: String,
13759}
13760
13761#[derive(Debug, Clone)]
13762struct SpeechRuntime {
13763 root: PathBuf,
13764 python: PathBuf,
13765 stt_program: PathBuf,
13766 tts_program: PathBuf,
13767}
13768
13769impl SpeechRuntime {
13770 fn new(root: PathBuf) -> Self {
13771 let python = managed_venv::interpreter(&root);
13772 let stt_program = managed_venv::venv_program(&root, "mlx_audio.stt.generate");
13773 let tts_program = managed_venv::venv_program(&root, "mlx_audio.tts.generate");
13774 Self {
13775 root,
13776 python,
13777 stt_program,
13778 tts_program,
13779 }
13780 }
13781
13782 fn is_ready(&self) -> bool {
13783 managed_venv::interpreter_healthy(&self.root)
13788 && self.stt_program.exists()
13789 && self.tts_program.exists()
13790 }
13791
13792 fn command_for(&self, subcommand: &str) -> Result<&Path, InferenceError> {
13793 match subcommand {
13794 "stt.generate" => Ok(&self.stt_program),
13795 "tts.generate" => Ok(&self.tts_program),
13796 _ => Err(InferenceError::InferenceFailed(format!(
13797 "unknown speech subcommand: {subcommand}"
13798 ))),
13799 }
13800 }
13801}
13802
13803async fn run_mlx_audio_command(
13804 runtime: &SpeechRuntime,
13805 subcommand: &str,
13806 args: &[String],
13807 detached_lease: resource_policy::DetachedLocalLease,
13808) -> Result<CommandOutput, InferenceError> {
13809 run_mlx_audio_command_with_env(runtime, subcommand, args, &[], detached_lease).await
13810}
13811
13812async fn run_admitted_blocking<F, T>(
13816 detached_lease: resource_policy::DetachedLocalLease,
13817 operation: F,
13818) -> Result<T, tokio::task::JoinError>
13819where
13820 F: FnOnce() -> T + Send + 'static,
13821 T: Send + 'static,
13822{
13823 tokio::task::spawn_blocking(move || {
13824 let _detached_lease = detached_lease;
13825 operation()
13826 })
13827 .await
13828}
13829
13830struct DetachedSpeechProcess {
13831 child: Option<tokio::process::Child>,
13832 lease: Option<resource_policy::DetachedLocalLease>,
13833}
13834
13835impl Drop for DetachedSpeechProcess {
13836 fn drop(&mut self) {
13837 let Some(mut child) = self.child.take() else {
13838 return;
13839 };
13840 let lease = self.lease.take();
13841 let _ = child.start_kill();
13842 if tokio::runtime::Handle::try_current().is_ok() {
13843 tokio::spawn(async move {
13844 let _ = child.wait().await;
13845 drop(lease);
13846 });
13847 } else {
13848 std::mem::forget((child, lease));
13851 }
13852 }
13853}
13854
13855async fn run_mlx_audio_command_with_env(
13856 runtime: &SpeechRuntime,
13857 subcommand: &str,
13858 args: &[String],
13859 envs: &[(String, String)],
13860 detached_lease: resource_policy::DetachedLocalLease,
13861) -> Result<CommandOutput, InferenceError> {
13862 let program = runtime.command_for(subcommand)?;
13863 let mut command = Command::new(program);
13864 command.args(args);
13865 crate::hf_cache::apply_to(&mut command);
13866 for (key, value) in envs {
13867 command.env(key, value);
13868 }
13869 command
13870 .stdout(std::process::Stdio::piped())
13871 .stderr(std::process::Stdio::piped())
13872 .kill_on_drop(true);
13873 let child = command
13874 .spawn()
13875 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
13876 let mut owned = DetachedSpeechProcess {
13877 child: Some(child),
13878 lease: Some(detached_lease),
13879 };
13880 let mut stdout = owned
13881 .child
13882 .as_mut()
13883 .expect("speech child owned")
13884 .stdout
13885 .take()
13886 .ok_or_else(|| {
13887 InferenceError::InferenceFailed(format!("{} stdout unavailable", program.display()))
13888 })?;
13889 let mut stderr = owned
13890 .child
13891 .as_mut()
13892 .expect("speech child owned")
13893 .stderr
13894 .take()
13895 .ok_or_else(|| {
13896 InferenceError::InferenceFailed(format!("{} stderr unavailable", program.display()))
13897 })?;
13898 let stdout_reader = tokio::spawn(async move {
13899 let mut bytes = Vec::new();
13900 stdout.read_to_end(&mut bytes).await.map(|_| bytes)
13901 });
13902 let stderr_reader = tokio::spawn(async move {
13903 let mut bytes = Vec::new();
13904 stderr.read_to_end(&mut bytes).await.map(|_| bytes)
13905 });
13906 let status = owned
13907 .child
13908 .as_mut()
13909 .expect("speech child owned")
13910 .wait()
13911 .await
13912 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
13913 owned.child.take();
13914 owned.lease.take();
13915 let stdout = stdout_reader
13916 .await
13917 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout join: {err}")))?
13918 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout: {err}")))?;
13919 let stderr = stderr_reader
13920 .await
13921 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr join: {err}")))?
13922 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr: {err}")))?;
13923
13924 if status.success() {
13925 Ok(CommandOutput {
13926 stdout: String::from_utf8_lossy(&stdout).to_string(),
13927 stderr: String::from_utf8_lossy(&stderr).to_string(),
13928 })
13929 } else {
13930 Err(InferenceError::InferenceFailed(format!(
13931 "{} exited with {}: {}",
13932 program.display(),
13933 status,
13934 String::from_utf8_lossy(&stderr)
13935 )))
13936 }
13937}
13938
13939async fn bootstrap_speech_runtime(runtime: &SpeechRuntime) -> Result<(), InferenceError> {
13940 let python = select_speech_python()?;
13941
13942 let outcome = managed_venv::ensure_venv(&runtime.root, &python)
13948 .await
13949 .map_err(|err| InferenceError::InferenceFailed(err.to_string()))?;
13950 if outcome == managed_venv::VenvOutcome::Recreated {
13951 tracing::warn!(
13952 root = %runtime.root.display(),
13953 "managed speech runtime had an unusable interpreter; rebuilt it \
13954 (installed packages were discarded and are being reinstalled)"
13955 );
13956 }
13957
13958 run_command(
13959 "uv",
13960 &[
13961 "pip".to_string(),
13962 "install".to_string(),
13963 "--python".to_string(),
13964 runtime.python.display().to_string(),
13965 speech_runtime_mlx_audio_spec(),
13966 "misaki[en]".to_string(),
13967 speech_runtime_spacy_model_spec(),
13968 ],
13969 )
13970 .await?;
13971
13972 Ok(())
13973}
13974
13975async fn run_command(program: &str, args: &[String]) -> Result<(), InferenceError> {
13976 let output = Command::new(program)
13977 .args(args)
13978 .output()
13979 .await
13980 .map_err(|err| InferenceError::InferenceFailed(format!("{program}: {err}")))?;
13981
13982 if output.status.success() {
13983 Ok(())
13984 } else {
13985 Err(InferenceError::InferenceFailed(format!(
13986 "{} exited with {}: {}",
13987 program,
13988 output.status,
13989 String::from_utf8_lossy(&output.stderr)
13990 )))
13991 }
13992}
13993
13994fn select_speech_python() -> Result<String, InferenceError> {
13995 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
13996 if !path.trim().is_empty() {
13997 return Ok(path);
13998 }
13999 }
14000
14001 for candidate in ["python3.13", "python3.12", "python3.11"] {
14002 if command_in_path(candidate) {
14003 return Ok(candidate.to_string());
14004 }
14005 }
14006
14007 Ok(SPEECH_RUNTIME_FALLBACK_PYTHON.to_string())
14017}
14018
14019const SPEECH_RUNTIME_FALLBACK_PYTHON: &str = "3.12";
14025
14026fn detect_speech_python() -> Option<String> {
14027 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14028 if !path.trim().is_empty() {
14029 return Some(path);
14030 }
14031 }
14032
14033 ["python3.13", "python3.12", "python3.11"]
14034 .into_iter()
14035 .find(|candidate| command_in_path(candidate))
14036 .map(str::to_string)
14037}
14038
14039fn speech_runtime_root_from_models_dir(_models_dir: &Path) -> PathBuf {
14040 if let Ok(path) = std::env::var("CAR_SPEECH_RUNTIME_DIR") {
14041 if !path.trim().is_empty() {
14042 return PathBuf::from(path);
14043 }
14044 }
14045
14046 std::env::var_os("HOME")
14047 .or_else(|| std::env::var_os("USERPROFILE"))
14048 .map(PathBuf::from)
14049 .unwrap_or_else(|| PathBuf::from("."))
14050 .join(".car")
14051 .join("speech-runtime")
14052}
14053
14054fn command_in_path(name: &str) -> bool {
14055 std::env::var_os("PATH")
14056 .map(|paths| {
14057 std::env::split_paths(&paths).any(|dir| {
14058 let path = dir.join(name);
14059 path.exists() && path.is_file()
14060 })
14061 })
14062 .unwrap_or(false)
14063}
14064
14065fn speech_model_cached(schema: &ModelSchema) -> bool {
14066 match &schema.source {
14067 ModelSource::Mlx { hf_repo, .. } => huggingface_repo_has_snapshot(hf_repo),
14068 ModelSource::WhisperCpp { model } => car_whisper::model_cached(model),
14069 ModelSource::WindowsSpeech {} => cfg!(target_os = "windows"),
14071 ModelSource::Proprietary { .. } => false,
14074 _ => false,
14075 }
14076}
14077
14078fn model_source_configured(schema: &ModelSchema, credential: &dyn Fn(&str) -> bool) -> bool {
14098 match &schema.source {
14099 ModelSource::RemoteApi {
14100 protocol: ApiProtocol::OpenRouter,
14101 ..
14102 } => crate::openrouter::credential_source().is_some(),
14103 ModelSource::RemoteApi {
14104 api_key_env,
14105 api_key_envs,
14106 ..
14107 } => credential(api_key_env) || api_key_envs.iter().any(|env_var| credential(env_var)),
14108 ModelSource::Proprietary { auth, .. } => match auth {
14109 ProprietaryAuth::ApiKeyEnv { env_var } => credential(env_var),
14110 ProprietaryAuth::BearerTokenEnv { env_var } => credential(env_var),
14111 ProprietaryAuth::OAuth2Pkce { .. } => matches!(
14112 car_auth::credential_authority_hint().state,
14113 car_auth::CredentialAuthorityState::Configured
14114 ),
14115 ProprietaryAuth::ChatGptSubscription {} => {
14116 credential(car_auth::CODEX_AUTH_RECORD_KEY)
14117 || crate::registry::passive_codex_oauth_available()
14118 }
14119 },
14120 ModelSource::VllmMlx { .. } => {
14121 std::env::var("VLLM_MLX_ENDPOINT").is_ok() || schema.available
14122 }
14123 ModelSource::Ollama { .. } => schema.available,
14124 ModelSource::CodexCli { .. } => crate::backend::codex_cli::is_available(),
14125 ModelSource::Mlx { .. }
14126 | ModelSource::ManagedVllmMlx { .. }
14127 | ModelSource::Local { .. }
14128 | ModelSource::WhisperCpp { .. } => true,
14129 ModelSource::WindowsSpeech {} => schema.available,
14131 ModelSource::AppleFoundationModels { .. } => schema.available,
14132 ModelSource::Delegated { .. } => true,
14137 }
14138}
14139
14140fn scoreboard_price_model(schema: &ModelSchema) -> Option<crate::scoreboard::PriceModel> {
14141 if schema.is_subscription_billed() {
14142 return None;
14143 }
14144 match (schema.cost.input_per_mtok, schema.cost.output_per_mtok) {
14145 (Some(input_per_mtok), Some(output_per_mtok)) => Some(crate::scoreboard::PriceModel {
14149 input_per_mtok,
14150 output_per_mtok,
14151 cache: schema.cache_rates(),
14152 is_estimate: !schema.cost.pricing_tiers.is_empty(),
14153 }),
14154 _ => None,
14155 }
14156}
14157
14158fn all_model_capabilities() -> [ModelCapability; 13] {
14159 [
14160 ModelCapability::Generate,
14161 ModelCapability::Embed,
14162 ModelCapability::Classify,
14163 ModelCapability::Code,
14164 ModelCapability::Reasoning,
14165 ModelCapability::Summarize,
14166 ModelCapability::ToolUse,
14167 ModelCapability::MultiToolCall,
14168 ModelCapability::Vision,
14169 ModelCapability::SpeechToText,
14170 ModelCapability::TextToSpeech,
14171 ModelCapability::ImageGeneration,
14172 ModelCapability::VideoGeneration,
14173 ]
14174}
14175
14176fn sort_capabilities(mut capabilities: Vec<ModelCapability>) -> Vec<ModelCapability> {
14177 capabilities.sort_by_key(|capability| {
14178 all_model_capabilities()
14179 .iter()
14180 .position(|candidate| candidate == capability)
14181 .unwrap_or(usize::MAX)
14182 });
14183 capabilities
14184}
14185
14186fn speech_model_source_label(schema: &ModelSchema) -> String {
14187 match &schema.source {
14188 ModelSource::Mlx { hf_repo, .. } => format!("mlx:{hf_repo}"),
14189 ModelSource::ManagedVllmMlx { hf_repo, .. } => {
14190 format!("managed-vllm-mlx:{hf_repo}")
14191 }
14192 ModelSource::WhisperCpp { model } => format!("whisper:{model}"),
14193 ModelSource::WindowsSpeech {} => "windows-speech".to_string(),
14194 ModelSource::Proprietary {
14195 provider, endpoint, ..
14196 } => format!("proprietary:{provider}:{endpoint}"),
14197 ModelSource::RemoteApi { endpoint, .. } => format!("remote:{endpoint}"),
14198 ModelSource::CodexCli { model } => format!("codex-cli:{model}"),
14199 ModelSource::Local { hf_repo, .. } => format!("local:{hf_repo}"),
14200 ModelSource::VllmMlx {
14201 endpoint,
14202 model_name,
14203 } => format!("vllm-mlx:{endpoint}:{model_name}"),
14204 ModelSource::Ollama { model_tag, host } => format!("ollama:{host}:{model_tag}"),
14205 ModelSource::AppleFoundationModels { use_case } => {
14206 format!(
14207 "apple-foundation:{}",
14208 use_case.as_deref().unwrap_or("default")
14209 )
14210 }
14211 ModelSource::Delegated { hint } => {
14212 format!("delegated:{}", hint.as_deref().unwrap_or("(none)"))
14213 }
14214 }
14215}
14216
14217fn rerank_prompt(instruction: &str, query: &str, document: &str) -> String {
14225 const SYSTEM: &str = "Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".";
14226 format!(
14227 "<|im_start|>system\n{SYSTEM}<|im_end|>\n\
14228 <|im_start|>user\n<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {document}<|im_end|>\n\
14229 <|im_start|>assistant\n<think>\n\n</think>\n\n"
14230 )
14231}
14232
14233fn parse_api_returned_status(err: &str) -> Option<u16> {
14257 let lower = err.to_ascii_lowercase();
14258 let idx = lower.find("api returned ")?;
14259 let digits: String = lower[idx + "api returned ".len()..]
14260 .chars()
14261 .take_while(|c| c.is_ascii_digit())
14262 .take(3)
14263 .collect();
14264 digits.parse().ok()
14265}
14266
14267pub const NO_BACKEND_RECOVERY_MARKER: &str = "no inference backend is available.";
14280
14281const AUTO_GRACE_FILE: &str = "concierge-auto-since";
14284const AUTO_GRACE_SECS: u64 = 7 * 24 * 60 * 60;
14286
14287#[doc(hidden)]
14291pub const LOCAL_TOOL_MODEL_PULL_ID: &str =
14292 if cfg!(all(
14295 target_os = "macos",
14296 target_arch = "aarch64",
14297 not(car_skip_mlx)
14298 )) {
14299 "mlx/qwen3-4b:4bit"
14300 } else {
14301 "qwen/qwen3-4b:q4_k_m"
14302 };
14303
14304#[doc(hidden)]
14311pub fn no_backend_recovery_hint(underlying: &str) -> Option<String> {
14312 let is_no_backend = underlying.contains("no credential")
14313 || underlying.contains("model not found")
14314 || underlying.contains("no models available")
14315 || underlying.contains("no inference runner");
14316 if !is_no_backend {
14317 return None;
14318 }
14319 Some(format!(
14325 "{NO_BACKEND_RECOVERY_MARKER} To use Parslee's hosted models, run:\n \
14326 car auth login\n\
14327 Or, for a model you name yourself with --model, install a local \
14328 tool-capable one:\n \
14329 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
14330 (underlying error: {underlying})"
14331 ))
14332}
14333
14334pub fn is_auth_rejection_message(underlying: &str) -> bool {
14352 let l = underlying.to_ascii_lowercase();
14353 l.contains("org lookup failed")
14354 || l.contains("authentication required")
14355 || (l.contains("401") && l.contains("unauthorized"))
14356 || (l.contains("403") && l.contains("forbidden"))
14357 || l.contains("invalid_grant")
14358 || l.contains("token expired")
14359}
14360
14361pub fn is_auth_failure_message(message: &str) -> bool {
14368 let lower = message.to_ascii_lowercase();
14369 AUTH_FAILURE_MESSAGE_MARKERS
14370 .iter()
14371 .any(|marker| lower.contains(marker))
14372 || is_auth_rejection_message(message)
14373}
14374
14375fn record_auth_dead_lane(slot: &mut Option<String>, candidate: &str, error: &str) {
14384 if slot.is_none() && is_auth_rejection_message(error) {
14385 *slot = Some(candidate.to_string());
14386 }
14387}
14388
14389fn is_local_last_resort(appended_id: Option<&str>, candidate_id: &str) -> bool {
14394 appended_id == Some(candidate_id)
14395}
14396
14397fn record_generation_span(
14413 model: &str,
14414 latency_ms: u64,
14415 ttft_ms: Option<u64>,
14416 usage: Option<&TokenUsage>,
14417) {
14418 let span = tracing::Span::current();
14419 span.record("model", model);
14420 span.record("gen_ai.response.model", model);
14421 span.record("latency_ms", latency_ms);
14422 if let Some(ttft) = ttft_ms {
14423 span.record("ttft_ms", ttft);
14424 }
14425 if let Some(u) = usage {
14426 span.record("prompt_tokens", u.prompt_tokens);
14427 span.record("completion_tokens", u.completion_tokens);
14428 span.record("gen_ai.usage.input_tokens", u.prompt_tokens);
14429 span.record("gen_ai.usage.output_tokens", u.completion_tokens);
14430 }
14431}
14432
14433fn report_local_last_resort_served(
14434 appended_id: Option<&str>,
14435 candidate_id: &str,
14436 resolved_id: &str,
14437) -> bool {
14438 let served = is_local_last_resort(appended_id, candidate_id);
14439 if served {
14440 tracing::warn!(
14441 local_model = %resolved_id,
14442 "on-device last-resort model served inference turn"
14443 );
14444 }
14445 served
14446}
14447
14448fn auth_expired_recovery_hint(underlying: &str) -> Option<String> {
14458 if !is_auth_rejection_message(underlying) {
14459 return None;
14460 }
14461 Some(format!(
14462 "your Parslee session has expired or was rejected — re-authenticate:\n \
14463 car auth login\n\
14464 Or, for a model you name yourself with --model, install a local \
14465 tool-capable one:\n \
14466 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
14467 (underlying error: {underlying})"
14468 ))
14469}
14470
14471fn score_from_rerank_output(text: &str, model_name: &str) -> f32 {
14472 let normalized: String = text
14477 .to_ascii_lowercase()
14478 .chars()
14479 .map(|c| if c.is_ascii_alphanumeric() { c } else { ' ' })
14480 .collect();
14481 for tok in normalized.split_ascii_whitespace().take(5) {
14482 match tok {
14483 "yes" => return 1.0,
14484 "no" => return 0.0,
14485 _ => continue,
14486 }
14487 }
14488 tracing::warn!(
14489 model = %model_name,
14490 output = %text,
14491 "rerank: first tokens contain neither `yes` nor `no`; returning neutral 0.5"
14492 );
14493 0.5
14494}
14495
14496fn default_speech_voice(schema: &ModelSchema) -> Option<String> {
14497 if schema.provider == "elevenlabs" {
14498 Some("JBFqnCBsd6RMkjVDRZzb".to_string())
14499 } else if schema.name == "Kokoro-82M-6bit" || schema.name == "Kokoro-82M-bf16" {
14500 Some("af_heart".to_string())
14501 } else if schema.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
14502 Some("Chelsie".to_string())
14503 } else {
14504 None
14505 }
14506}
14507
14508#[allow(dead_code)] fn huggingface_repo_has_snapshot(repo_id: &str) -> bool {
14510 crate::registry::latest_huggingface_repo_snapshot(repo_id).is_some()
14511}
14512
14513#[allow(dead_code)] fn count_files_recursive(root: &Path) -> usize {
14515 let Ok(entries) = std::fs::read_dir(root) else {
14516 return 0;
14517 };
14518 entries
14519 .filter_map(Result::ok)
14520 .map(|entry| entry.path())
14521 .map(|path| {
14522 if path.is_dir() {
14523 count_files_recursive(&path)
14524 } else if path.is_file() {
14525 1
14526 } else {
14527 0
14528 }
14529 })
14530 .sum()
14531}
14532
14533async fn download_hf_repo_snapshot(repo_id: &str) -> Result<(PathBuf, usize), InferenceError> {
14534 let api = crate::hf_cache::api()
14535 .with_progress(false)
14536 .build()
14537 .map_err(|e| InferenceError::DownloadFailed(format!("init hf api: {e}")))?;
14538 let repo = api.model(repo_id.to_string());
14539 let info = repo
14540 .info()
14541 .await
14542 .map_err(|e| InferenceError::DownloadFailed(format!("{repo_id}: {e}")))?;
14543
14544 let snapshot_path = crate::hf_cache::repo_dir(repo_id)
14545 .join("snapshots")
14546 .join(&info.sha);
14547 let mut downloaded = 0usize;
14548 for sibling in &info.siblings {
14549 let local_path = snapshot_path.join(&sibling.rfilename);
14550 if crate::download::cache_file_usable(&local_path) {
14558 downloaded += 1;
14559 continue;
14560 }
14561 let _ = std::fs::remove_file(&local_path);
14566 repo.download(&sibling.rfilename).await.map_err(|e| {
14567 InferenceError::DownloadFailed(format!("{repo_id}/{}: {e}", sibling.rfilename))
14568 })?;
14569 crate::retire::note_fetched(repo_id, &sibling.rfilename, Some(info.sha.clone()));
14570 downloaded += 1;
14571 }
14572
14573 Ok((snapshot_path, downloaded))
14574}
14575
14576fn temp_work_dir(prefix: &str) -> Result<PathBuf, InferenceError> {
14577 let unique = SystemTime::now()
14578 .duration_since(UNIX_EPOCH)
14579 .map_err(|e| InferenceError::InferenceFailed(format!("clock error: {e}")))?
14580 .as_nanos();
14581 let dir = std::env::temp_dir().join(format!("car-inference-{prefix}-{unique}"));
14582 std::fs::create_dir_all(&dir)?;
14583 Ok(dir)
14584}
14585
14586fn ensure_parent_dir(path: &Path) -> Result<(), InferenceError> {
14587 if let Some(parent) = path.parent() {
14588 std::fs::create_dir_all(parent)?;
14589 }
14590 Ok(())
14591}
14592
14593fn requested_or_temp_output(
14594 output_path: Option<&str>,
14595 format: &str,
14596) -> Result<PathBuf, InferenceError> {
14597 if let Some(path) = output_path {
14598 return Ok(PathBuf::from(path));
14599 }
14600 let dir = temp_work_dir("audio-out")?;
14601 Ok(dir.join(format!("speech.{format}")))
14602}
14603
14604#[allow(dead_code)] fn requested_or_temp_media_output(
14606 output_path: Option<&str>,
14607 format: &str,
14608 stem: &str,
14609) -> Result<PathBuf, InferenceError> {
14610 if let Some(path) = output_path {
14611 return Ok(PathBuf::from(path));
14612 }
14613 let dir = temp_work_dir(&format!("{stem}-out"))?;
14614 Ok(dir.join(format!("{stem}.{format}")))
14615}
14616
14617fn materialize_audio_output(
14618 produced: &Path,
14619 requested: Option<&str>,
14620 format: &str,
14621) -> Result<PathBuf, InferenceError> {
14622 if let Some(path) = requested {
14623 let dest = PathBuf::from(path);
14624 ensure_parent_dir(&dest)?;
14625 std::fs::copy(produced, &dest)?;
14626 Ok(dest)
14627 } else {
14628 let dest = requested_or_temp_output(None, format)?;
14629 ensure_parent_dir(&dest)?;
14630 std::fs::copy(produced, &dest)?;
14631 Ok(dest)
14632 }
14633}
14634
14635#[cfg(target_os = "windows")]
14641fn winrt_synthesize_wav(text: &str, voice: &str, rate: f64) -> Result<Vec<u8>, InferenceError> {
14642 use windows::core::HSTRING;
14643 use windows::Media::SpeechSynthesis::SpeechSynthesizer;
14644 use windows::Storage::Streams::DataReader;
14645
14646 let err = |m: String| InferenceError::InferenceFailed(m);
14647 let synth =
14648 SpeechSynthesizer::new().map_err(|e| err(format!("SpeechSynthesizer::new: {e}")))?;
14649 if let Ok(opts) = synth.Options() {
14650 let _ = opts.SetSpeakingRate(rate.clamp(0.5, 6.0));
14651 }
14652 if !voice.is_empty() {
14653 if let Ok(all) = SpeechSynthesizer::AllVoices() {
14654 let want = voice.to_lowercase();
14655 let count = all.Size().unwrap_or(0);
14656 for i in 0..count {
14657 if let Ok(info) = all.GetAt(i) {
14658 if let Ok(name) = info.DisplayName() {
14659 if name.to_string_lossy().to_lowercase().contains(&want) {
14660 let _ = synth.SetVoice(&info);
14661 break;
14662 }
14663 }
14664 }
14665 }
14666 }
14667 }
14668 let stream = synth
14669 .SynthesizeTextToStreamAsync(&HSTRING::from(text))
14670 .map_err(|e| err(format!("SynthesizeTextToStreamAsync: {e}")))?
14671 .get()
14672 .map_err(|e| err(format!("synthesize await: {e}")))?;
14673 let size = stream
14674 .Size()
14675 .map_err(|e| err(format!("stream size: {e}")))?;
14676 let input = stream
14677 .GetInputStreamAt(0)
14678 .map_err(|e| err(format!("input stream: {e}")))?;
14679 let reader =
14680 DataReader::CreateDataReader(&input).map_err(|e| err(format!("data reader: {e}")))?;
14681 reader
14682 .LoadAsync(size as u32)
14683 .map_err(|e| err(format!("load async: {e}")))?
14684 .get()
14685 .map_err(|e| err(format!("load await: {e}")))?;
14686 let mut buf = vec![0u8; size as usize];
14687 reader
14688 .ReadBytes(&mut buf)
14689 .map_err(|e| err(format!("read bytes: {e}")))?;
14690 Ok(buf)
14691}
14692
14693#[allow(dead_code)] fn read_transcription_result(output_prefix: &Path) -> Result<Option<String>, InferenceError> {
14695 let candidates = [
14696 output_prefix.with_extension("json"),
14697 output_prefix.to_path_buf(),
14698 ];
14699
14700 for path in candidates {
14701 if path.exists() {
14702 let contents = std::fs::read_to_string(path)?;
14703 if let Some(text) = extract_text_from_payload(&contents) {
14704 return Ok(Some(text));
14705 }
14706 }
14707 }
14708
14709 Ok(None)
14710}
14711
14712#[allow(dead_code)] fn extract_text_from_payload(payload: &str) -> Option<String> {
14714 let value: serde_json::Value = serde_json::from_str(payload).ok()?;
14715 if let Some(text) = value.get("text").and_then(|v| v.as_str()) {
14716 return Some(text.to_string());
14717 }
14718 if let Some(transcripts) = value.get("transcripts").and_then(|v| v.as_array()) {
14719 let joined = transcripts
14720 .iter()
14721 .filter_map(|item| item.get("text").and_then(|v| v.as_str()))
14722 .collect::<Vec<_>>()
14723 .join("\n");
14724 if !joined.is_empty() {
14725 return Some(joined);
14726 }
14727 }
14728 if let Some(items) = value.as_array() {
14729 let joined = items
14730 .iter()
14731 .filter_map(|item| {
14732 item.get("text")
14733 .or_else(|| item.get("Content"))
14734 .and_then(|v| v.as_str())
14735 })
14736 .collect::<Vec<_>>()
14737 .join(" ");
14738 if !joined.is_empty() {
14739 return Some(joined);
14740 }
14741 }
14742 None
14743}
14744
14745#[allow(dead_code)] fn find_audio_file(output_dir: &Path) -> Result<Option<PathBuf>, InferenceError> {
14747 let mut audio_files = Vec::new();
14748 collect_audio_files(output_dir, &mut audio_files)?;
14749 audio_files.sort();
14750 Ok(audio_files.into_iter().next())
14751}
14752
14753#[allow(dead_code)] fn collect_audio_files(dir: &Path, audio_files: &mut Vec<PathBuf>) -> Result<(), InferenceError> {
14755 for entry in std::fs::read_dir(dir)? {
14756 let path = entry?.path();
14757 if path.is_dir() {
14758 collect_audio_files(&path, audio_files)?;
14759 } else if matches!(
14760 path.extension().and_then(|ext| ext.to_str()),
14761 Some("wav" | "mp3" | "flac" | "pcm" | "m4a")
14762 ) {
14763 audio_files.push(path);
14764 }
14765 }
14766 Ok(())
14767}
14768
14769fn media_type_for_format(format: &str) -> String {
14770 match format.to_ascii_lowercase().as_str() {
14771 "mp3" => "audio/mpeg".to_string(),
14772 "flac" => "audio/flac".to_string(),
14773 "pcm" => "audio/L16".to_string(),
14774 "m4a" => "audio/mp4".to_string(),
14775 _ => "audio/wav".to_string(),
14776 }
14777}
14778
14779fn kokoro_lang_code(language: Option<&str>) -> &'static str {
14780 match language.unwrap_or("en").to_ascii_lowercase().as_str() {
14781 "en-gb" | "british" | "british english" => "b",
14782 "ja" | "japanese" => "j",
14783 "zh" | "zh-cn" | "mandarin" | "chinese" => "z",
14784 "es" | "spanish" => "e",
14785 "fr" | "french" => "f",
14786 _ => "a",
14787 }
14788}
14789
14790#[allow(dead_code)] fn normalize_lang_code(language: &str) -> String {
14792 match language.to_ascii_lowercase().as_str() {
14793 "english" | "en-us" | "en_us" => "en".to_string(),
14794 "spanish" => "es".to_string(),
14795 "french" => "fr".to_string(),
14796 "japanese" => "ja".to_string(),
14797 "chinese" | "mandarin" => "zh".to_string(),
14798 other => match other {
14799 "en" | "es" | "fr" | "ja" | "zh" => other.to_string(),
14800 _ => "en".to_string(),
14801 },
14802 }
14803}
14804
14805fn elevenlabs_auth(
14806 schema: &ModelSchema,
14807 resolve_credential: fn(&str) -> Option<String>,
14808) -> Result<(String, String), InferenceError> {
14809 match &schema.source {
14810 ModelSource::Proprietary {
14811 endpoint,
14812 auth: schema::ProprietaryAuth::ApiKeyEnv { env_var },
14813 ..
14814 } => {
14815 let key = resolve_credential(env_var).ok_or_else(|| {
14816 InferenceError::InferenceFailed(format!(
14817 "missing API key {env_var}; set the environment variable or \
14818 store it with `car secrets put {env_var}`"
14819 ))
14820 })?;
14821 Ok((endpoint.clone(), key))
14822 }
14823 _ => Err(InferenceError::InferenceFailed(format!(
14824 "model {} is not an ElevenLabs proprietary model",
14825 schema.id
14826 ))),
14827 }
14828}
14829
14830fn elevenlabs_output_format(format: &str) -> &'static str {
14831 match format.to_ascii_lowercase().as_str() {
14832 "mp3" => "mp3_44100_128",
14833 "pcm" => "pcm_16000",
14834 _ => "wav_44100",
14835 }
14836}
14837
14838fn quality_seeds(
14842 state_models_dir: &Path,
14843) -> std::collections::HashMap<String, outcome::QualitySeed> {
14844 let mut seeds = std::collections::HashMap::new();
14845 for path in benchmark_priors_paths(state_models_dir) {
14846 let Ok(priors) = routing_ext::load_benchmark_priors(&path) else {
14847 continue;
14848 };
14849 for (model_id, prior) in priors {
14850 seeds.insert(
14851 model_id,
14852 outcome::QualitySeed {
14853 overall: Some(prior.overall_score),
14854 tasks: prior.task_scores,
14855 },
14856 );
14857 }
14858 }
14859 seeds
14860}
14861
14862fn benchmark_priors_paths(state_models_dir: &Path) -> Vec<PathBuf> {
14867 let mut paths = Vec::new();
14868
14869 let direct = state_models_dir.join("benchmark_priors.json");
14870 if !paths.contains(&direct) {
14871 paths.push(direct);
14872 }
14873
14874 if let Some(parent) = state_models_dir.parent() {
14875 let parent_path = parent.join("benchmark_priors.json");
14876 if !paths.contains(&parent_path) {
14877 paths.push(parent_path);
14878 }
14879 }
14880
14881 if let Some(path) = std::env::var_os("CAR_BENCHMARK_PRIORS_PATH") {
14882 let path = PathBuf::from(path);
14883 if !paths.contains(&path) {
14884 paths.push(path);
14885 }
14886 }
14887
14888 paths
14889}
14890
14891fn load_benchmark_prior_health(
14892 state_models_dir: &Path,
14893 schemas: &[ModelSchema],
14894) -> Vec<ModelBenchmarkPriorHealth> {
14895 let mut priors = std::collections::BTreeMap::new();
14896 for path in benchmark_priors_paths(state_models_dir) {
14897 let Ok(loaded) = routing_ext::load_benchmark_priors(&path) else {
14898 continue;
14899 };
14900 for (model_id, prior) in loaded {
14901 let model_name = schemas
14902 .iter()
14903 .find(|schema| schema.id == model_id)
14904 .map(|schema| schema.name.clone());
14905 priors.insert(
14906 model_id.clone(),
14907 ModelBenchmarkPriorHealth {
14908 model_id,
14909 model_name,
14910 overall_score: prior.overall_score,
14911 overall_latency_ms: prior.overall_latency_ms,
14912 task_scores: prior.task_scores,
14913 task_latency_ms: prior.task_latency_ms,
14914 source_path: path.clone(),
14915 },
14916 );
14917 }
14918 }
14919
14920 priors.into_values().collect()
14921}
14922
14923fn kokoro_runtime_fallback_enabled() -> bool {
14924 std::env::var("CAR_SPEECH_KOKORO_FALLBACK")
14925 .ok()
14926 .map(|value| {
14927 !matches!(
14928 value.trim().to_ascii_lowercase().as_str(),
14929 "0" | "false" | "off"
14930 )
14931 })
14932 .unwrap_or(true)
14933}
14934
14935fn speech_runtime_mlx_audio_spec() -> String {
14936 std::env::var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC")
14937 .ok()
14938 .filter(|value| !value.trim().is_empty())
14939 .unwrap_or_else(|| "mlx-audio==0.4.2".to_string())
14940}
14941
14942fn speech_runtime_spacy_model_spec() -> String {
14943 std::env::var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC")
14944 .ok()
14945 .filter(|value| !value.trim().is_empty())
14946 .unwrap_or_else(|| {
14947 "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl".to_string()
14948 })
14949}
14950
14951#[cfg(test)]
14952pub(crate) fn run_in_isolated_test_process(test_name: &str, sentinel: &str) -> bool {
14953 static CHILD_PROCESS_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
14954
14955 if std::env::var_os(sentinel).is_some() {
14956 return true;
14957 }
14958
14959 let _child_process = CHILD_PROCESS_MUTEX
14960 .lock()
14961 .unwrap_or_else(std::sync::PoisonError::into_inner);
14962 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
14963 command
14964 .arg("--exact")
14965 .arg(test_name)
14966 .arg("--nocapture")
14967 .arg("--test-threads=1")
14968 .env(sentinel, "1");
14969 for name in [
14970 car_home::ENV_VAR,
14971 "CAR_SECRETS_FILE_DIR",
14972 "CAR_TEST_NATIVE_KEYCHAIN",
14973 "CAR_AUTH_LOCK_PATH",
14974 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
14975 car_auth::PARSLEE_API_BASE_KEY,
14976 crate::openrouter::API_KEY_ENV,
14977 "OPENAI_API_KEY",
14978 "ANTHROPIC_API_KEY",
14979 "GOOGLE_API_KEY",
14980 "ELEVENLABS_API_KEY",
14981 "SSL_CERT_FILE",
14982 "SSL_CERT_DIR",
14983 ] {
14984 command.env_remove(name);
14985 }
14986 const CHILD_BUDGET: std::time::Duration = std::time::Duration::from_secs(300);
14993 let mut stdout = tempfile::tempfile().expect("create isolated-test stdout capture");
14994 let mut stderr = tempfile::tempfile().expect("create isolated-test stderr capture");
14995 command
14996 .stdin(std::process::Stdio::null())
14997 .stdout(stdout.try_clone().expect("clone stdout capture"))
14998 .stderr(stderr.try_clone().expect("clone stderr capture"));
14999 #[cfg(unix)]
15000 {
15001 use std::os::unix::process::CommandExt;
15002 command.process_group(0);
15003 }
15004
15005 let mut child = command.spawn().expect("spawn isolated inference test");
15006 let deadline = std::time::Instant::now() + CHILD_BUDGET;
15007 let mut timed_out = false;
15008 let status = loop {
15009 match child.try_wait().expect("poll isolated inference test") {
15010 Some(status) => break status,
15011 None if std::time::Instant::now() < deadline => {
15012 std::thread::sleep(std::time::Duration::from_millis(50));
15013 }
15014 None => {
15015 timed_out = true;
15016 #[cfg(unix)]
15017 unsafe {
15018 libc::kill(-(child.id() as i32), libc::SIGKILL);
15019 }
15020 let _ = child.kill();
15021 break child
15022 .wait()
15023 .expect("reap timed-out isolated inference test");
15024 }
15025 }
15026 };
15027
15028 use std::io::{Read, Seek};
15029 stdout.rewind().expect("rewind isolated-test stdout");
15030 stderr.rewind().expect("rewind isolated-test stderr");
15031 let mut stdout_bytes = Vec::new();
15032 let mut stderr_bytes = Vec::new();
15033 stdout
15034 .read_to_end(&mut stdout_bytes)
15035 .expect("read isolated-test stdout");
15036 stderr
15037 .read_to_end(&mut stderr_bytes)
15038 .expect("read isolated-test stderr");
15039 assert!(
15040 !timed_out,
15041 "isolated {test_name} exceeded {}s and was killed\nstdout:\n{}\nstderr:\n{}",
15042 CHILD_BUDGET.as_secs(),
15043 String::from_utf8_lossy(&stdout_bytes),
15044 String::from_utf8_lossy(&stderr_bytes),
15045 );
15046 assert!(
15047 status.success(),
15048 "isolated {test_name} failed\nstdout:\n{}\nstderr:\n{}",
15049 String::from_utf8_lossy(&stdout_bytes),
15050 String::from_utf8_lossy(&stderr_bytes),
15051 );
15052 false
15053}
15054
15055fn pinned_by(agents: &[String]) -> String {
15057 let quoted: Vec<String> = agents.iter().map(|a| format!("\"{a}\"")).collect();
15058 match quoted.as_slice() {
15059 [one] => format!("pinned by the agent {one}"),
15060 _ => format!("pinned by the agents {}", quoted.join(", ")),
15061 }
15062}
15063
15064#[cfg(test)]
15065mod tests {
15066
15067 #[test]
15077 fn every_successful_generation_records_its_span() {
15078 let source = include_str!("lib.rs");
15079 let start = source
15080 .find("async fn generate_tracked_inner")
15081 .expect("the tracked-inference entry point");
15082 let open = source[start..].find('{').expect("function body") + start;
15083 let mut depth = 0usize;
15084 let mut end = open;
15085 for (offset, ch) in source[open..].char_indices() {
15086 match ch {
15087 '{' => depth += 1,
15088 '}' => {
15089 depth -= 1;
15090 if depth == 0 {
15091 end = open + offset;
15092 break;
15093 }
15094 }
15095 _ => {}
15096 }
15097 }
15098 let body = &source[start..end];
15099 let returns = body.matches("return Ok(InferenceResult {").count();
15100 let records = body.matches("record_generation_span(").count();
15101 assert!(
15102 returns >= 3,
15103 "control: the function still has several success returns, got {returns}"
15104 );
15105 assert_eq!(
15106 returns, records,
15107 "every successful return must record its generation span: \
15108 {returns} returns, {records} recorder calls"
15109 );
15110 }
15111 use super::*;
15112
15113 #[test]
15114 fn classify_defaults_to_jev_only_while_it_is_available() {
15115 let catalog: Vec<ModelSchema> =
15116 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
15117 let mut jev = catalog
15118 .iter()
15119 .find(|s| s.id == "parslee/jev")
15120 .unwrap()
15121 .clone();
15122 let local = catalog
15123 .iter()
15124 .find(|s| s.id == "mlx/qwen3-0.6b:6bit")
15125 .unwrap()
15126 .clone();
15127 jev.available = false;
15128 assert_eq!(
15129 default_system_one_classifier([&jev, &local].into_iter()),
15130 None,
15131 "signed out: keep the local default"
15132 );
15133 jev.available = true;
15134 assert_eq!(
15135 default_system_one_classifier([&local, &jev].into_iter()).as_deref(),
15136 Some("parslee/jev")
15137 );
15138 let chat = catalog
15140 .iter()
15141 .find(|s| s.id == "parslee/fast")
15142 .unwrap()
15143 .clone();
15144 assert_eq!(default_system_one_classifier([&chat].into_iter()), None);
15145 }
15146
15147 #[tokio::test]
15148 async fn generate_pinned_to_a_system_one_row_is_refused_streaming_included() {
15149 let dir = tempfile::tempdir().unwrap();
15150 let engine = InferenceEngine::new(test_config(dir.path().to_path_buf()));
15151 let req = || GenerateRequest {
15152 prompt: "hello".into(),
15153 model: Some("parslee/jev".into()),
15154 ..Default::default()
15155 };
15156 let refused = |e: &InferenceError| {
15157 matches!(
15158 e,
15159 InferenceError::UnsupportedMode {
15160 backend: "parslee-system-one",
15161 ..
15162 }
15163 )
15164 };
15165 match engine.generate_tracked(req()).await {
15166 Err(e) => assert!(refused(&e), "{e}"),
15167 Ok(_) => panic!("generate on a System One row must be refused"),
15168 }
15169 match engine.generate_tracked_stream(req()).await {
15170 Err(e) => assert!(refused(&e), "{e}"),
15171 Ok(_) => panic!("streaming generate on a System One row must be refused"),
15172 }
15173 }
15174 use std::ffi::OsString;
15175 use tempfile::TempDir;
15176
15177 #[test]
15184 fn local_models_keep_the_callers_default_output_budget() {
15185 use crate::tasks::generate::DEFAULT_MAX_TOKENS;
15186
15187 let catalog = crate::registry::builtin_catalog();
15188 let local = catalog
15189 .iter()
15190 .find(|s| s.id == "mlx/qwen3-8b:4bit")
15191 .expect("mlx/qwen3-8b:4bit is a builtin catalog entry");
15192
15193 assert!(local.is_local(), "mlx/* is decoded in-process");
15194 assert_eq!(
15195 local.effective_max_output(),
15196 32_768,
15197 "the budget that made the reported turn ~24 minutes long"
15198 );
15199 assert_eq!(
15200 resolved_max_tokens(DEFAULT_MAX_TOKENS, local),
15201 DEFAULT_MAX_TOKENS,
15202 "a local model must keep the budget the caller actually asked for"
15203 );
15204
15205 let mut remote = catalog
15208 .iter()
15209 .find(|s| !s.is_local())
15210 .expect("the builtin catalog ships remote models")
15211 .clone();
15212 remote.context_length = local.context_length;
15213 remote.max_output_tokens = local.max_output_tokens;
15214 assert!(!remote.is_local());
15215 assert_eq!(
15216 resolved_max_tokens(DEFAULT_MAX_TOKENS, &remote),
15217 32_768,
15218 "remote models still get their advertised output budget"
15219 );
15220
15221 assert_eq!(resolved_max_tokens(512, local), 512);
15223 assert_eq!(resolved_max_tokens(512, &remote), 512);
15224
15225 let codex = catalog
15226 .iter()
15227 .find(|s| s.id == "openai/gpt-5.6-sol:high")
15228 .expect("subscription-backed Codex row is builtin");
15229 assert_eq!(
15230 resolved_max_tokens(DEFAULT_MAX_TOKENS, codex),
15231 DEFAULT_MAX_TOKENS,
15232 "an approximate instruction must not widen the default to 128K tokens"
15233 );
15234
15235 if let Some(vllm) = catalog.iter().find(|s| s.is_vllm_mlx()) {
15241 assert!(vllm.is_local(), "vLLM-MLX runs on this machine");
15242 assert!(
15243 !vllm.decodes_in_process(),
15244 "but CAR does not decode it token by token"
15245 );
15246 assert_eq!(
15247 resolved_max_tokens(DEFAULT_MAX_TOKENS, vllm),
15248 vllm.effective_max_output(),
15249 );
15250 }
15251 }
15252
15253 #[test]
15256 fn a_ceiling_stop_fails_instead_of_retrying() {
15257 use EmptyPassAction::*;
15258 const CEILING: Option<&str> = Some(LOCAL_DECODE_TIMEOUT_STOP_REASON);
15259
15260 assert_eq!(
15263 classify_empty_pass(true, CEILING, "", true),
15264 FailDecodeCeiling
15265 );
15266 assert_eq!(
15267 classify_empty_pass(false, CEILING, "", true),
15268 FailDecodeCeiling
15269 );
15270 assert_eq!(
15272 classify_empty_pass(true, CEILING, " \n ", true),
15273 FailDecodeCeiling
15274 );
15275
15276 assert_eq!(
15279 classify_empty_pass(true, Some("length"), "", true),
15280 RetryWithoutThinking
15281 );
15282 assert_eq!(
15283 classify_empty_pass(true, None, "", true),
15284 RetryWithoutThinking
15285 );
15286
15287 assert_eq!(classify_empty_pass(false, Some("stop"), "", true), Accept);
15290
15291 assert_eq!(classify_empty_pass(true, CEILING, "partial", true), Accept);
15294 assert_eq!(classify_empty_pass(true, CEILING, "", false), Accept);
15295
15296 assert_ne!(LOCAL_DECODE_TIMEOUT_STOP_REASON, "timeout");
15300 assert_eq!(
15301 classify_empty_pass(true, Some("timeout"), "", true),
15302 RetryWithoutThinking
15303 );
15304 }
15305
15306 #[test]
15310 fn decode_deadline_and_heartbeat_predicates() {
15311 use std::time::Duration;
15312 let limit = Duration::from_secs(300);
15313
15314 assert!(!deadline_exceeded(Duration::from_secs(86_400), None));
15316
15317 assert!(!deadline_exceeded(Duration::from_secs(299), Some(limit)));
15318 assert!(deadline_exceeded(limit, Some(limit)));
15321 assert!(deadline_exceeded(Duration::from_secs(301), Some(limit)));
15322
15323 let every = Duration::from_secs(10);
15324 assert!(!heartbeat_due(
15326 Duration::from_secs(9),
15327 Duration::ZERO,
15328 every
15329 ));
15330 assert!(heartbeat_due(
15331 Duration::from_secs(10),
15332 Duration::ZERO,
15333 every
15334 ));
15335 assert!(!heartbeat_due(
15337 Duration::from_secs(11),
15338 Duration::from_secs(10),
15339 every
15340 ));
15341 assert!(heartbeat_due(
15342 Duration::from_secs(20),
15343 Duration::from_secs(10),
15344 every
15345 ));
15346 assert!(!heartbeat_due(
15348 Duration::from_secs(5),
15349 Duration::from_secs(10),
15350 every
15351 ));
15352 }
15353
15354 #[test]
15355 fn decode_timeout_parses_with_a_safe_fallback() {
15356 let default = std::time::Duration::from_secs(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS);
15357 assert_eq!(parse_decode_timeout(None), Some(default));
15358 assert_eq!(
15359 parse_decode_timeout(Some(" 45 ")),
15360 Some(std::time::Duration::from_secs(45))
15361 );
15362 assert_eq!(
15363 parse_decode_timeout(Some("0")),
15364 None,
15365 "0 disables the ceiling"
15366 );
15367 assert_eq!(
15368 parse_decode_timeout(Some("banana")),
15369 Some(default),
15370 "garbage must fall back to the default, never silently disable it"
15371 );
15372 }
15373
15374 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15379 #[test]
15380 fn local_decode_stops_at_the_wall_clock_ceiling() {
15381 struct NeverStops;
15384
15385 impl crate::backend::local::TextDecoder for NeverStops {
15386 fn encode(&self, _text: &str) -> Result<Vec<u32>, InferenceError> {
15387 Ok(vec![1, 2, 3])
15388 }
15389 fn decode(&self, tokens: &[u32]) -> Result<String, InferenceError> {
15390 Ok("x".repeat(tokens.len()))
15391 }
15392 fn forward(
15393 &mut self,
15394 _tokens: &[u32],
15395 _pos: usize,
15396 ) -> Result<Vec<f32>, InferenceError> {
15397 std::thread::sleep(std::time::Duration::from_millis(5));
15398 let mut logits = vec![0.0f32; 16];
15399 logits[7] = 10.0;
15400 Ok(logits)
15401 }
15402 fn eos_ids(&self) -> Vec<u32> {
15403 vec![0]
15404 }
15405 fn context_length(&self) -> usize {
15406 4096
15407 }
15408 fn clear_kv_cache(&mut self) {}
15409 }
15410
15411 let params = GenerateParams {
15414 max_tokens: 100_000,
15415 temperature: 0.0,
15416 ..Default::default()
15417 };
15418 let ceiling = std::time::Duration::from_millis(300);
15419
15420 let mut backend = NeverStops;
15421 let started = std::time::Instant::now();
15422 let generated = match InferenceEngine::drive_generation_with_timeout(
15423 &mut backend,
15424 "anything",
15425 ¶ms,
15426 Some(ceiling),
15427 ) {
15428 Ok(generated) => generated,
15429 Err(e) => panic!(
15431 "a decode cut short by the ceiling returns its partial text, not an error: {}",
15432 e.into_inner()
15433 ),
15434 };
15435 let elapsed = started.elapsed();
15436
15437 assert_eq!(
15438 generated.stop_reason.as_deref(),
15439 Some(LOCAL_DECODE_TIMEOUT_STOP_REASON),
15440 "the caller must be able to tell a deadline stop from a clean finish"
15441 );
15442 assert!(
15443 InferenceResult {
15444 text: generated.text.clone(),
15445 bounding_boxes: Vec::new(),
15446 tool_calls: vec![],
15447 trace_id: String::new(),
15448 model_used: String::new(),
15449 model_identity: Default::default(),
15450 latency_ms: 0,
15451 time_to_first_token_ms: None,
15452 usage: None,
15453 provider_output_items: Vec::new(),
15454 thinking: Vec::new(),
15455 stop_reason: generated.stop_reason.clone(),
15456 auth_fallback_from: None,
15457 local_last_resort: false,
15458 fallback_from: Vec::new(),
15459 }
15460 .was_truncated(),
15461 "a ceiling stop is a cut-short answer, not a complete one"
15462 );
15463 assert!(
15464 generated.completion_tokens > 0,
15465 "the partial response is kept, not discarded"
15466 );
15467 assert!(
15468 generated.completion_tokens < params.max_tokens,
15469 "the loop stopped on the clock, not by exhausting the budget"
15470 );
15471 assert!(
15472 elapsed < std::time::Duration::from_secs(30),
15473 "bounded in wall clock; took {elapsed:?}"
15474 );
15475 }
15476
15477 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
15478
15479 impl RestoredEnvironment {
15480 fn capture(names: &[&'static str]) -> Self {
15481 Self(
15482 names
15483 .iter()
15484 .map(|name| (*name, std::env::var_os(name)))
15485 .collect(),
15486 )
15487 }
15488 }
15489
15490 impl Drop for RestoredEnvironment {
15491 fn drop(&mut self) {
15492 for (name, value) in &self.0 {
15493 unsafe {
15494 match value {
15495 Some(value) => std::env::set_var(name, value),
15496 None => std::env::remove_var(name),
15497 }
15498 }
15499 }
15500 }
15501 }
15502
15503 struct FixtureLocalOffload {
15504 emit_done: bool,
15505 }
15506
15507 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15512 struct ExactPinCaptureOffload {
15513 dispatched_models: std::sync::Mutex<Vec<String>>,
15514 }
15515
15516 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15517 impl ExactPinCaptureOffload {
15518 fn new() -> Self {
15519 Self {
15520 dispatched_models: std::sync::Mutex::new(Vec::new()),
15521 }
15522 }
15523
15524 fn dispatched_models(&self) -> Vec<String> {
15525 self.dispatched_models.lock().unwrap().clone()
15526 }
15527 }
15528
15529 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15530 #[async_trait::async_trait]
15531 impl crate::offload::LocalGenerationOffload for ExactPinCaptureOffload {
15532 async fn generate(
15533 &self,
15534 request: GenerateRequest,
15535 ) -> Result<InferenceResult, InferenceError> {
15536 let model_id = request.model.expect("resolved worker model");
15537 self.dispatched_models
15538 .lock()
15539 .unwrap()
15540 .push(model_id.clone());
15541 Ok(InferenceResult {
15542 text: "exact pin".into(),
15543 tool_calls: vec![],
15544 bounding_boxes: vec![],
15545 trace_id: "worker-trace".into(),
15546 model_used: model_id,
15547 model_identity: Default::default(),
15548 latency_ms: 0,
15549 time_to_first_token_ms: None,
15550 usage: None,
15551 provider_output_items: vec![],
15552 thinking: vec![],
15553 stop_reason: Some("stop".into()),
15554 auth_fallback_from: None,
15555 local_last_resort: false,
15556 fallback_from: Vec::new(),
15557 })
15558 }
15559
15560 async fn stream(
15561 &self,
15562 request: GenerateRequest,
15563 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15564 let model_id = request.model.expect("resolved worker model");
15565 self.dispatched_models.lock().unwrap().push(model_id);
15566 let (tx, rx) = tokio::sync::mpsc::channel(2);
15567 tx.send(StreamEvent::Done {
15568 text: "exact pin".into(),
15569 tool_calls: vec![],
15570 })
15571 .await
15572 .unwrap();
15573 Ok(rx)
15574 }
15575 }
15576
15577 struct ReleaseRetryOffload {
15578 calls: std::sync::Mutex<Vec<String>>,
15579 model_id: String,
15580 resident: bool,
15581 release_acknowledged: bool,
15582 }
15583
15584 struct SequencedRetryProbe {
15585 calls: Arc<std::sync::atomic::AtomicUsize>,
15586 first_available_mb: u64,
15587 subsequent_available_mb: u64,
15588 }
15589
15590 impl crate::resource_policy::LiveMemoryProbe for SequencedRetryProbe {
15591 fn available_memory_mb(
15592 &self,
15593 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
15594 let call = self
15595 .calls
15596 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
15597 Ok(Some(if call == 0 {
15598 self.first_available_mb
15599 } else {
15600 self.subsequent_available_mb
15601 }))
15602 }
15603 }
15604
15605 #[async_trait::async_trait]
15606 impl crate::offload::LocalGenerationOffload for ReleaseRetryOffload {
15607 async fn generate(
15608 &self,
15609 _request: GenerateRequest,
15610 ) -> Result<InferenceResult, InferenceError> {
15611 unreachable!("release-and-retry fixture")
15612 }
15613
15614 async fn stream(
15615 &self,
15616 _request: GenerateRequest,
15617 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15618 unreachable!("release-and-retry fixture")
15619 }
15620
15621 async fn resident_models(&self) -> Vec<String> {
15622 self.resident
15623 .then(|| self.model_id.clone())
15624 .into_iter()
15625 .collect()
15626 }
15627
15628 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
15629 Some(format!("worker:retry-fixture:{model_id}"))
15630 }
15631
15632 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15633 self.calls
15634 .lock()
15635 .unwrap()
15636 .push(format!("release:{model_id}"));
15637 Ok(self.release_acknowledged)
15638 }
15639 }
15640
15641 struct RetiringReleaseOffload {
15642 calls: std::sync::Mutex<Vec<String>>,
15643 model_id: String,
15644 allocation_id: String,
15645 resident: std::sync::atomic::AtomicBool,
15646 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
15647 }
15648
15649 #[async_trait::async_trait]
15650 impl crate::offload::LocalGenerationOffload for RetiringReleaseOffload {
15651 async fn generate(
15652 &self,
15653 _request: GenerateRequest,
15654 ) -> Result<InferenceResult, InferenceError> {
15655 unreachable!("retiring-release fixture")
15656 }
15657
15658 async fn stream(
15659 &self,
15660 _request: GenerateRequest,
15661 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15662 unreachable!("retiring-release fixture")
15663 }
15664
15665 async fn resident_models(&self) -> Vec<String> {
15666 self.resident
15667 .load(std::sync::atomic::Ordering::Acquire)
15668 .then(|| self.model_id.clone())
15669 .into_iter()
15670 .collect()
15671 }
15672
15673 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15674 Some(self.allocation_id.clone())
15675 }
15676
15677 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15678 self.calls
15679 .lock()
15680 .unwrap()
15681 .push(format!("release:{model_id}"));
15682 self.coordinator
15683 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
15684 self.coordinator
15685 .finish_teardown_allocation(model_id, &self.allocation_id);
15686 self.resident
15687 .store(false, std::sync::atomic::Ordering::Release);
15688 Ok(true)
15689 }
15690 }
15691
15692 #[tokio::test]
15693 async fn release_and_retry_releases_resident_worker_after_live_memory_refusal() {
15694 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15695 let root = tempfile::tempdir().unwrap();
15696 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15697 let schema = crate::registry::builtin_catalog()
15698 .into_iter()
15699 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15700 .unwrap();
15701 let allocation_id = format!("worker:retry-fixture:{}", schema.id);
15702 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15703 let coordinator = Arc::new(
15704 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15705 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15706 metal_mac_for_fit(32),
15707 Arc::new(SequencedRetryProbe {
15708 calls: probe_calls.clone(),
15709 first_available_mb: 1_000,
15710 subsequent_available_mb: 1_048_576,
15711 }),
15712 ),
15713 );
15714 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
15715 engine.local_admission = coordinator.clone();
15716 let offload = Arc::new(RetiringReleaseOffload {
15717 calls: std::sync::Mutex::new(Vec::new()),
15718 model_id: schema.id.clone(),
15719 allocation_id,
15720 resident: std::sync::atomic::AtomicBool::new(true),
15721 coordinator,
15722 });
15723 crate::offload::set_local_offload(Some(offload.clone()));
15724
15725 let result = engine
15726 .reserve_local_request_with_worker_retry(&schema, 9_000)
15727 .await;
15728 crate::offload::set_local_offload(None);
15729
15730 assert!(
15731 result.is_ok(),
15732 "release followed by a second probe should admit"
15733 );
15734 assert_eq!(
15735 *offload.calls.lock().unwrap(),
15736 vec![format!("release:{}", schema.id)]
15737 );
15738 assert_eq!(
15739 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
15740 2,
15741 "admission must probe exactly once before and once after release"
15742 );
15743 }
15744
15745 #[tokio::test]
15746 async fn release_and_retry_ignores_sibling_allocation_after_exact_worker_retirement() {
15747 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15748 let root = tempfile::tempdir().unwrap();
15749 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15750 let schema = crate::registry::builtin_catalog()
15751 .into_iter()
15752 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15753 .unwrap();
15754 let worker_allocation_id = format!("worker:sibling-test:{}", schema.id);
15755 let sibling_allocation_id = format!("cache:sibling-test:{}", schema.id);
15756 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15757 let coordinator = Arc::new(
15758 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15759 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15760 metal_mac_for_fit(32),
15761 Arc::new(SequencedRetryProbe {
15762 calls: probe_calls.clone(),
15763 first_available_mb: 1_000,
15764 subsequent_available_mb: 1_048_576,
15765 }),
15766 ),
15767 );
15768 coordinator.mark_resident_allocation(&schema.id, &worker_allocation_id, 4_800);
15769 coordinator.mark_resident_allocation(&schema.id, &sibling_allocation_id, 500);
15770 engine.local_admission = coordinator.clone();
15771 let offload = Arc::new(RetiringReleaseOffload {
15772 calls: std::sync::Mutex::new(Vec::new()),
15773 model_id: schema.id.clone(),
15774 allocation_id: worker_allocation_id.clone(),
15775 resident: std::sync::atomic::AtomicBool::new(true),
15776 coordinator: coordinator.clone(),
15777 });
15778 crate::offload::set_local_offload(Some(offload.clone()));
15779
15780 let result = engine
15781 .reserve_local_request_with_worker_retry(&schema, 9_000)
15782 .await;
15783 crate::offload::set_local_offload(None);
15784
15785 assert!(
15786 result.is_ok(),
15787 "an unrelated resident allocation must not block the worker retry"
15788 );
15789 assert_eq!(
15790 coordinator.resident_allocation_ids(&schema.id),
15791 vec![sibling_allocation_id]
15792 );
15793 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 2);
15794 assert_eq!(
15795 *offload.calls.lock().unwrap(),
15796 vec![format!("release:{}", schema.id)]
15797 );
15798 }
15799
15800 struct UnretiredReleaseOffload {
15801 calls: std::sync::Mutex<Vec<String>>,
15802 model_id: String,
15803 allocation_id: String,
15804 resident: std::sync::atomic::AtomicBool,
15805 }
15806
15807 #[async_trait::async_trait]
15808 impl crate::offload::LocalGenerationOffload for UnretiredReleaseOffload {
15809 async fn generate(
15810 &self,
15811 _request: GenerateRequest,
15812 ) -> Result<InferenceResult, InferenceError> {
15813 unreachable!("unretired-release fixture")
15814 }
15815
15816 async fn stream(
15817 &self,
15818 _request: GenerateRequest,
15819 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15820 unreachable!("unretired-release fixture")
15821 }
15822
15823 async fn resident_models(&self) -> Vec<String> {
15824 self.resident
15825 .load(std::sync::atomic::Ordering::Acquire)
15826 .then(|| self.model_id.clone())
15827 .into_iter()
15828 .collect()
15829 }
15830
15831 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15832 Some(self.allocation_id.clone())
15833 }
15834
15835 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15836 self.calls
15837 .lock()
15838 .unwrap()
15839 .push(format!("release:{model_id}"));
15840 self.resident
15841 .store(false, std::sync::atomic::Ordering::Release);
15842 Ok(true)
15843 }
15844 }
15845
15846 #[tokio::test]
15847 async fn release_and_retry_rejects_ack_without_accounting_retirement() {
15848 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15849 let warnings_before =
15850 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
15851 let root = tempfile::tempdir().unwrap();
15852 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15853 let schema = crate::registry::builtin_catalog()
15854 .into_iter()
15855 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15856 .unwrap();
15857 let allocation_id = format!("worker:phantom:{}", schema.id);
15858 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15859 let coordinator = Arc::new(
15860 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15861 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15862 metal_mac_for_fit(32),
15863 Arc::new(SequencedRetryProbe {
15864 calls: probe_calls.clone(),
15865 first_available_mb: 1_000,
15866 subsequent_available_mb: 8_000,
15867 }),
15868 ),
15869 );
15870 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
15871 engine.local_admission = coordinator;
15872 let offload = Arc::new(UnretiredReleaseOffload {
15873 calls: std::sync::Mutex::new(Vec::new()),
15874 model_id: schema.id.clone(),
15875 allocation_id,
15876 resident: std::sync::atomic::AtomicBool::new(true),
15877 });
15878 crate::offload::set_local_offload(Some(offload.clone()));
15879
15880 let result = engine
15881 .reserve_local_request_with_worker_retry(&schema, 9_000)
15882 .await;
15883 crate::offload::set_local_offload(None);
15884
15885 assert!(matches!(
15886 result,
15887 Err(InferenceError::LocalResourceBlocked { preflight, .. })
15888 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
15889 ));
15890 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
15891 assert_eq!(
15892 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
15893 warnings_before + 1,
15894 "the refused retry must emit the stale-residency warning"
15895 );
15896 assert!(engine.local_admission.is_resident(&schema.id));
15897 assert!(offload.resident_models().await.is_empty());
15898 assert_eq!(
15899 *offload.calls.lock().unwrap(),
15900 vec![format!("release:{}", schema.id)]
15901 );
15902 }
15903
15904 struct PendingTeardownAckOffload {
15905 calls: std::sync::Mutex<Vec<String>>,
15906 model_id: String,
15907 allocation_id: String,
15908 resident: std::sync::atomic::AtomicBool,
15909 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
15910 }
15911
15912 #[async_trait::async_trait]
15913 impl crate::offload::LocalGenerationOffload for PendingTeardownAckOffload {
15914 async fn generate(
15915 &self,
15916 _request: GenerateRequest,
15917 ) -> Result<InferenceResult, InferenceError> {
15918 unreachable!("pending-teardown ACK fixture")
15919 }
15920
15921 async fn stream(
15922 &self,
15923 _request: GenerateRequest,
15924 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15925 unreachable!("pending-teardown ACK fixture")
15926 }
15927
15928 async fn resident_models(&self) -> Vec<String> {
15929 self.resident
15930 .load(std::sync::atomic::Ordering::Acquire)
15931 .then(|| self.model_id.clone())
15932 .into_iter()
15933 .collect()
15934 }
15935
15936 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15937 Some(self.allocation_id.clone())
15938 }
15939
15940 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15941 self.calls
15942 .lock()
15943 .unwrap()
15944 .push(format!("release:{model_id}"));
15945 self.coordinator
15946 .mark_teardown_pending_allocation_with_charge(
15947 model_id,
15948 &self.allocation_id,
15949 4_800 * 1024 * 1024,
15950 );
15951 self.resident
15952 .store(false, std::sync::atomic::Ordering::Release);
15953 Ok(true)
15954 }
15955 }
15956
15957 #[tokio::test]
15958 async fn release_and_retry_rejects_ack_with_exact_worker_pending_teardown() {
15959 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15960 let warnings_before =
15961 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
15962 let root = tempfile::tempdir().unwrap();
15963 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15964 let schema = crate::registry::builtin_catalog()
15965 .into_iter()
15966 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15967 .unwrap();
15968 let allocation_id = format!("worker:pending-test:{}", schema.id);
15969 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15970 let coordinator = Arc::new(
15971 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15972 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15973 metal_mac_for_fit(32),
15974 Arc::new(SequencedRetryProbe {
15975 calls: probe_calls.clone(),
15976 first_available_mb: 1_000,
15977 subsequent_available_mb: 8_000,
15978 }),
15979 ),
15980 );
15981 engine.local_admission = coordinator.clone();
15982 let offload = Arc::new(PendingTeardownAckOffload {
15983 calls: std::sync::Mutex::new(Vec::new()),
15984 model_id: schema.id.clone(),
15985 allocation_id: allocation_id.clone(),
15986 resident: std::sync::atomic::AtomicBool::new(true),
15987 coordinator: coordinator.clone(),
15988 });
15989 crate::offload::set_local_offload(Some(offload.clone()));
15990
15991 let result = engine
15992 .reserve_local_request_with_worker_retry(&schema, 9_000)
15993 .await;
15994 crate::offload::set_local_offload(None);
15995
15996 assert!(matches!(
15997 result,
15998 Err(InferenceError::LocalResourceBlocked { preflight, .. })
15999 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16000 ));
16001 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16002 assert_eq!(
16003 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
16004 warnings_before + 1,
16005 "the pending exact allocation must emit the fail-closed warning"
16006 );
16007 assert_eq!(
16008 coordinator.resident_allocation_ids(&schema.id),
16009 vec![allocation_id]
16010 );
16011 assert!(offload.resident_models().await.is_empty());
16012 assert_eq!(
16013 *offload.calls.lock().unwrap(),
16014 vec![format!("release:{}", schema.id)]
16015 );
16016 }
16017
16018 struct MissingAllocationIdOffload {
16019 calls: std::sync::Mutex<Vec<String>>,
16020 model_id: String,
16021 }
16022
16023 #[async_trait::async_trait]
16024 impl crate::offload::LocalGenerationOffload for MissingAllocationIdOffload {
16025 async fn generate(
16026 &self,
16027 _request: GenerateRequest,
16028 ) -> Result<InferenceResult, InferenceError> {
16029 unreachable!("missing allocation-id fixture")
16030 }
16031
16032 async fn stream(
16033 &self,
16034 _request: GenerateRequest,
16035 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16036 unreachable!("missing allocation-id fixture")
16037 }
16038
16039 async fn resident_models(&self) -> Vec<String> {
16040 vec![self.model_id.clone()]
16041 }
16042
16043 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16044 self.calls
16045 .lock()
16046 .unwrap()
16047 .push(format!("release:{model_id}"));
16048 Ok(true)
16049 }
16050 }
16051
16052 #[tokio::test]
16053 async fn release_and_retry_without_scoped_allocation_id_fails_closed() {
16054 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16055 let root = tempfile::tempdir().unwrap();
16056 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16057 let schema = crate::registry::builtin_catalog()
16058 .into_iter()
16059 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16060 .unwrap();
16061 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16062 engine.local_admission = Arc::new(
16063 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16064 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16065 metal_mac_for_fit(32),
16066 Arc::new(SequencedRetryProbe {
16067 calls: probe_calls.clone(),
16068 first_available_mb: 1_000,
16069 subsequent_available_mb: 8_000,
16070 }),
16071 ),
16072 );
16073 let offload = Arc::new(MissingAllocationIdOffload {
16074 calls: std::sync::Mutex::new(Vec::new()),
16075 model_id: schema.id.clone(),
16076 });
16077 crate::offload::set_local_offload(Some(offload.clone()));
16078
16079 let result = engine
16080 .reserve_local_request_with_worker_retry(&schema, 9_000)
16081 .await;
16082 crate::offload::set_local_offload(None);
16083
16084 assert!(matches!(
16085 result,
16086 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16087 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16088 ));
16089 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16090 assert!(offload.calls.lock().unwrap().is_empty());
16091 }
16092
16093 struct ReplacingResidentOffload {
16094 calls: std::sync::Mutex<Vec<String>>,
16095 model_id: String,
16096 allocation_id: String,
16097 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16098 }
16099
16100 #[async_trait::async_trait]
16101 impl crate::offload::LocalGenerationOffload for ReplacingResidentOffload {
16102 async fn generate(
16103 &self,
16104 _request: GenerateRequest,
16105 ) -> Result<InferenceResult, InferenceError> {
16106 unreachable!("replacement-resident fixture")
16107 }
16108
16109 async fn stream(
16110 &self,
16111 _request: GenerateRequest,
16112 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16113 unreachable!("replacement-resident fixture")
16114 }
16115
16116 async fn resident_models(&self) -> Vec<String> {
16117 vec![self.model_id.clone()]
16118 }
16119
16120 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16121 Some(self.allocation_id.clone())
16122 }
16123
16124 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16125 self.calls
16126 .lock()
16127 .unwrap()
16128 .push(format!("release:{model_id}"));
16129 self.coordinator
16133 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
16134 self.coordinator
16135 .finish_teardown_allocation(model_id, &self.allocation_id);
16136 self.coordinator
16137 .mark_resident_allocation(model_id, &self.allocation_id, 4_800);
16138 Ok(true)
16139 }
16140 }
16141
16142 #[tokio::test]
16143 async fn release_and_retry_same_id_replacement_blocks_retry_without_erasing_accounting() {
16144 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16145 let root = tempfile::tempdir().unwrap();
16146 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16147 let schema = crate::registry::builtin_catalog()
16148 .into_iter()
16149 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16150 .unwrap();
16151 let allocation_id = format!("worker:reused-scope:{}", schema.id);
16152 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16153 let coordinator = Arc::new(
16154 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16155 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16156 metal_mac_for_fit(32),
16157 Arc::new(SequencedRetryProbe {
16158 calls: probe_calls.clone(),
16159 first_available_mb: 1_000,
16160 subsequent_available_mb: 8_000,
16161 }),
16162 ),
16163 );
16164 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
16165 engine.local_admission = coordinator.clone();
16166 let offload = Arc::new(ReplacingResidentOffload {
16167 calls: std::sync::Mutex::new(Vec::new()),
16168 model_id: schema.id.clone(),
16169 allocation_id,
16170 coordinator,
16171 });
16172 crate::offload::set_local_offload(Some(offload.clone()));
16173
16174 let result = engine
16175 .reserve_local_request_with_worker_retry(&schema, 9_000)
16176 .await;
16177 crate::offload::set_local_offload(None);
16178
16179 assert!(matches!(
16180 result,
16181 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16182 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16183 ));
16184 assert!(engine.local_admission.is_resident(&schema.id));
16185 assert_eq!(
16186 *offload.calls.lock().unwrap(),
16187 vec![format!("release:{}", schema.id)]
16188 );
16189 assert_eq!(
16190 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
16191 1,
16192 "replacement residency must stop the retry before a second probe"
16193 );
16194 }
16195
16196 #[tokio::test]
16197 async fn release_and_retry_nonresident_worker_returns_original_refusal() {
16198 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16199 let root = tempfile::tempdir().unwrap();
16200 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16201 let schema = crate::registry::builtin_catalog()
16202 .into_iter()
16203 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16204 .unwrap();
16205 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16206 engine.local_admission = Arc::new(
16207 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16208 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16209 metal_mac_for_fit(32),
16210 Arc::new(SequencedRetryProbe {
16211 calls: probe_calls.clone(),
16212 first_available_mb: 1_000,
16213 subsequent_available_mb: 1_048_576,
16214 }),
16215 ),
16216 );
16217 let offload = Arc::new(ReleaseRetryOffload {
16218 calls: std::sync::Mutex::new(Vec::new()),
16219 model_id: schema.id.clone(),
16220 resident: false,
16221 release_acknowledged: true,
16222 });
16223 crate::offload::set_local_offload(Some(offload.clone()));
16224
16225 let result = engine
16226 .reserve_local_request_with_worker_retry(&schema, 9_000)
16227 .await;
16228 crate::offload::set_local_offload(None);
16229
16230 assert!(matches!(
16231 result,
16232 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16233 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16234 ));
16235 assert!(offload.calls.lock().unwrap().is_empty());
16236 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16237 }
16238
16239 #[tokio::test]
16240 async fn release_and_retry_unacknowledged_release_does_not_reprobe() {
16241 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16242 let root = tempfile::tempdir().unwrap();
16243 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16244 let schema = crate::registry::builtin_catalog()
16245 .into_iter()
16246 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16247 .unwrap();
16248 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16249 engine.local_admission = Arc::new(
16250 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16251 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16252 metal_mac_for_fit(32),
16253 Arc::new(SequencedRetryProbe {
16254 calls: probe_calls.clone(),
16255 first_available_mb: 1_000,
16256 subsequent_available_mb: 1_048_576,
16257 }),
16258 ),
16259 );
16260 let offload = Arc::new(ReleaseRetryOffload {
16261 calls: std::sync::Mutex::new(Vec::new()),
16262 model_id: schema.id.clone(),
16263 resident: true,
16264 release_acknowledged: false,
16265 });
16266 crate::offload::set_local_offload(Some(offload.clone()));
16267
16268 let result = engine
16269 .reserve_local_request_with_worker_retry(&schema, 9_000)
16270 .await;
16271 crate::offload::set_local_offload(None);
16272
16273 assert!(matches!(
16274 result,
16275 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16276 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16277 ));
16278 assert_eq!(
16279 *offload.calls.lock().unwrap(),
16280 vec![format!("release:{}", schema.id)]
16281 );
16282 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16283 }
16284
16285 #[tokio::test]
16286 async fn release_and_retry_allowed_first_verdict_never_releases() {
16287 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16288 let root = tempfile::tempdir().unwrap();
16289 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16290 let schema = crate::registry::builtin_catalog()
16291 .into_iter()
16292 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16293 .unwrap();
16294 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16295 engine.local_admission = Arc::new(
16296 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16297 crate::resource_policy::ResourcePolicy::local_focused(),
16298 metal_mac_for_fit(64),
16299 Arc::new(SequencedRetryProbe {
16300 calls: probe_calls.clone(),
16301 first_available_mb: 1_048_576,
16302 subsequent_available_mb: 1_048_576,
16303 }),
16304 ),
16305 );
16306 let offload = Arc::new(ReleaseRetryOffload {
16307 calls: std::sync::Mutex::new(Vec::new()),
16308 model_id: schema.id.clone(),
16309 resident: true,
16310 release_acknowledged: true,
16311 });
16312 crate::offload::set_local_offload(Some(offload.clone()));
16313
16314 let result = engine
16315 .reserve_local_request_with_worker_retry(&schema, 9_000)
16316 .await;
16317 crate::offload::set_local_offload(None);
16318
16319 assert!(result.is_ok());
16320 assert!(offload.calls.lock().unwrap().is_empty());
16321 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16322 }
16323
16324 struct ReconcileOffload {
16325 calls: std::sync::Mutex<Vec<String>>,
16326 release_acknowledged: bool,
16327 }
16328
16329 #[async_trait::async_trait]
16330 impl crate::offload::LocalGenerationOffload for ReconcileOffload {
16331 async fn generate(
16332 &self,
16333 _request: GenerateRequest,
16334 ) -> Result<InferenceResult, InferenceError> {
16335 unreachable!("residency reconciliation fixture")
16336 }
16337
16338 async fn stream(
16339 &self,
16340 _request: GenerateRequest,
16341 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16342 unreachable!("residency reconciliation fixture")
16343 }
16344
16345 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
16346 self.calls
16347 .lock()
16348 .unwrap()
16349 .push(format!("allocation:{model_id}"));
16350 Some(format!("worker:{model_id}:generation-7"))
16351 }
16352
16353 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16354 self.calls
16355 .lock()
16356 .unwrap()
16357 .push(format!("release:{model_id}"));
16358 Ok(self.release_acknowledged)
16359 }
16360 }
16361
16362 fn reconciliation_reservation() -> (
16363 Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16364 crate::resource_policy::LocalLoadReservation,
16365 ) {
16366 struct FixedProbe;
16367 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
16368 fn available_memory_mb(
16369 &self,
16370 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
16371 Ok(Some(24_000))
16372 }
16373 }
16374 let coordinator = Arc::new(
16375 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16376 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16377 crate::hardware::HardwareInfo {
16378 total_ram_mb: 32 * 1024,
16379 ..crate::hardware::HardwareInfo::detect()
16380 },
16381 Arc::new(FixedProbe),
16382 ),
16383 );
16384 let reservation = coordinator
16385 .reserve_measured_host_allocation(
16386 "expected/model",
16387 "expected/model#request",
16388 1024 * 1024,
16389 0,
16390 )
16391 .unwrap();
16392 (coordinator, reservation)
16393 }
16394
16395 #[tokio::test]
16396 async fn worker_residency_reconciliation_releases_mismatched_ack_before_returning_error() {
16397 let offload = ReconcileOffload {
16398 calls: std::sync::Mutex::new(Vec::new()),
16399 release_acknowledged: true,
16400 };
16401 let (_coordinator, mut reservation) = reconciliation_reservation();
16402 let error = InferenceEngine::reconcile_worker_residency(
16403 &offload,
16404 "expected/model",
16405 &crate::offload::LocalWorkerResidency {
16406 model_id: "wrong/model".into(),
16407 measured_weights_bytes: 2 * 1024 * 1024,
16408 },
16409 backend_cache::BackendRetention::Resident,
16410 &mut reservation,
16411 )
16412 .await
16413 .unwrap_err();
16414 assert!(error.to_string().contains("wrong/model"));
16415 assert_eq!(
16416 *offload.calls.lock().unwrap(),
16417 vec!["release:wrong/model".to_string()]
16418 );
16419 }
16420
16421 #[tokio::test]
16422 async fn worker_residency_reconciliation_publishes_exact_owner_for_matching_ack() {
16423 let offload = ReconcileOffload {
16424 calls: std::sync::Mutex::new(Vec::new()),
16425 release_acknowledged: false,
16426 };
16427 let (coordinator, mut reservation) = reconciliation_reservation();
16428 InferenceEngine::reconcile_worker_residency(
16429 &offload,
16430 "expected/model",
16431 &crate::offload::LocalWorkerResidency {
16432 model_id: "expected/model".into(),
16433 measured_weights_bytes: 2 * 1024 * 1024,
16434 },
16435 backend_cache::BackendRetention::Resident,
16436 &mut reservation,
16437 )
16438 .await
16439 .unwrap();
16440 drop(reservation);
16441 assert_eq!(
16442 *offload.calls.lock().unwrap(),
16443 vec!["allocation:expected/model".to_string()]
16444 );
16445 assert!(coordinator.is_resident("expected/model"));
16446 }
16447
16448 #[async_trait::async_trait]
16449 impl crate::offload::LocalGenerationOffload for FixtureLocalOffload {
16450 async fn generate(
16451 &self,
16452 _request: GenerateRequest,
16453 ) -> Result<InferenceResult, InferenceError> {
16454 unreachable!("streaming fixture")
16455 }
16456
16457 async fn stream(
16458 &self,
16459 _request: GenerateRequest,
16460 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16461 unreachable!("admission-aware streaming fixture")
16462 }
16463
16464 async fn stream_admitted(
16465 &self,
16466 request: GenerateRequest,
16467 admission: crate::offload::LocalWorkerAdmission,
16468 ) -> Result<crate::offload::LocalOffloadStream, InferenceError> {
16469 let (tx, rx) = tokio::sync::mpsc::channel(4);
16470 let emit_done = self.emit_done;
16471 tokio::spawn(async move {
16472 let _ = tx.send(StreamEvent::TextDelta("local answer".into())).await;
16473 if emit_done {
16474 let _ = tx
16475 .send(StreamEvent::Done {
16476 text: "local answer".into(),
16477 tool_calls: vec![],
16478 })
16479 .await;
16480 }
16481 });
16482 Ok(crate::offload::LocalOffloadStream {
16483 events: rx,
16484 residency: crate::offload::LocalWorkerResidency {
16485 model_id: request.model.unwrap_or_else(|| "fixture/local".into()),
16486 measured_weights_bytes: admission.measured_weights_bytes,
16487 },
16488 retention: backend_cache::BackendRetention::Resident,
16489 })
16490 }
16491 }
16492
16493 fn pin_test_local_admission_memory(engine: &mut InferenceEngine) {
16501 let policy = engine.local_admission.policy();
16502 let mut hardware = HardwareInfo::detect();
16503 hardware.total_ram_mb = hardware.total_ram_mb.max(16 * 1024);
16504 hardware.max_model_mb = hardware.max_model_mb.max(16 * 1024);
16505 engine.local_admission = Arc::new(resource_policy::LocalAdmissionCoordinator::with_probe(
16506 policy,
16507 hardware,
16508 Arc::new(resource_policy::FixedLiveMemoryProbe::known(1_048_576)),
16509 ));
16510 }
16511
16512 fn install_small_local_fixture(engine: &InferenceEngine) -> String {
16513 let schema = engine
16514 .unified_registry()
16515 .find_by_name("Qwen3-0.6B")
16516 .expect("small built-in local model")
16517 .clone();
16518 let model_dir = engine.config.models_dir.join(&schema.name);
16519 std::fs::create_dir_all(&model_dir).unwrap();
16520 std::fs::write(model_dir.join("model.gguf"), b"fixture").unwrap();
16521 std::fs::write(model_dir.join("tokenizer.json"), b"{}").unwrap();
16522 schema.id
16523 }
16524
16525 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16526 fn install_exact_pin_equivalent_fixture(engine: &InferenceEngine) -> String {
16527 let gguf = engine
16528 .unified_registry()
16529 .get("qwen/qwen3-0.6b:q8_0")
16530 .expect("GGUF fixture row")
16531 .clone();
16532 let mlx = engine
16533 .unified_registry()
16534 .get("mlx/qwen3-0.6b:6bit")
16535 .expect("MLX equivalent fixture row")
16536 .clone();
16537
16538 let gguf_dir = engine.config.models_dir.join(&gguf.name);
16539 std::fs::create_dir_all(&gguf_dir).unwrap();
16540 std::fs::write(gguf_dir.join("model.gguf"), b"fixture").unwrap();
16541 std::fs::write(gguf_dir.join("tokenizer.json"), b"{}").unwrap();
16542
16543 let mlx_dir = engine.config.models_dir.join(&mlx.name);
16544 std::fs::create_dir_all(&mlx_dir).unwrap();
16545 std::fs::write(mlx_dir.join("config.json"), b"{}").unwrap();
16546 std::fs::write(mlx_dir.join("model.safetensors"), b"fixture").unwrap();
16547
16548 gguf.id
16549 }
16550
16551 fn remote_stream_fixture_schema(
16552 id: &str,
16553 endpoint: String,
16554 protocol: schema::ApiProtocol,
16555 api_key_env: &str,
16556 ) -> ModelSchema {
16557 ModelSchema {
16558 id: id.into(),
16559 name: "gemini-test".into(),
16560 provider: "test".into(),
16561 family: "test".into(),
16562 version: "1".into(),
16563 capabilities: vec![ModelCapability::Generate],
16564 context_length: 128_000,
16565 max_output_tokens: Some(8_192),
16566 param_count: String::new(),
16567 quantization: None,
16568 performance: Default::default(),
16569 cost: Default::default(),
16570 source: ModelSource::RemoteApi {
16571 endpoint,
16572 api_key_env: api_key_env.into(),
16573 api_key_envs: vec![],
16574 api_version: None,
16575 protocol,
16576 },
16577 tags: vec!["test".into()],
16578 supported_params: vec![],
16579 public_benchmarks: vec![],
16580 trust_tier: TrustTier::Community,
16581 deprecated: false,
16582 available: true,
16583 weights_ready: true,
16584 }
16585 }
16586
16587 async fn assert_remote_model_identity_contract(
16588 protocol: schema::ApiProtocol,
16589 model_id: &str,
16590 provider_model: &str,
16591 api_key_env: &str,
16592 ) {
16593 use wiremock::matchers::{method, path};
16594 use wiremock::{Mock, MockServer, ResponseTemplate};
16595
16596 let server = MockServer::start().await;
16597 let (endpoint_path, response) = match protocol {
16598 schema::ApiProtocol::OpenAiCompat => (
16599 "/v1/chat/completions",
16600 serde_json::json!({
16601 "choices": [{
16602 "message": {"content": "openai exact"},
16603 "finish_reason": "stop"
16604 }],
16605 "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}
16606 }),
16607 ),
16608 schema::ApiProtocol::Anthropic => (
16609 "/v1/messages",
16610 serde_json::json!({
16611 "content": [{"type": "text", "text": "anthropic exact"}],
16612 "stop_reason": "end_turn",
16613 "usage": {"input_tokens": 2, "output_tokens": 2}
16614 }),
16615 ),
16616 _ => unreachable!("identity regression covers the newsroom's two remote providers"),
16617 };
16618 Mock::given(method("POST"))
16619 .and(path(endpoint_path))
16620 .respond_with(ResponseTemplate::new(200).set_body_json(response))
16621 .mount(&server)
16622 .await;
16623 unsafe { std::env::set_var(api_key_env, "fixture") };
16624
16625 let tmp = TempDir::new().unwrap();
16626 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
16627 let mut schema =
16628 remote_stream_fixture_schema(model_id, server.uri(), protocol, api_key_env);
16629 schema.name = provider_model.into();
16630 engine.register_model(schema);
16631
16632 let mut exact_request = GenerateRequest {
16633 prompt: "return the exact identity".into(),
16634 ..Default::default()
16635 };
16636 pin_exact_model_id(&mut exact_request, model_id.into()).unwrap();
16637 let exact = engine.generate_tracked(exact_request).await.unwrap();
16638 assert_eq!(exact.model_used, model_id);
16639 assert_eq!(
16640 exact.model_identity.requested_model_id.as_deref(),
16641 Some(model_id)
16642 );
16643 assert_eq!(exact.model_identity.resolved_model_id, model_id);
16644 let exact_envelope = serde_json::to_value(&exact).unwrap();
16645 assert_eq!(exact_envelope["model_used"], model_id);
16646 assert_eq!(exact_envelope["requested_model_id"], model_id);
16647 assert_eq!(exact_envelope["resolved_model_id"], model_id);
16648 assert!(exact_envelope["row_digest"].is_string());
16649 assert!(exact_envelope["catalog_revision"].is_string());
16650
16651 let loose = engine
16652 .generate_tracked(GenerateRequest {
16653 prompt: "keep legacy display-name routing".into(),
16654 model: Some(provider_model.into()),
16655 params: GenerateParams {
16656 strict_model: true,
16657 ..Default::default()
16658 },
16659 ..Default::default()
16660 })
16661 .await
16662 .unwrap();
16663 assert_eq!(loose.model_used, provider_model);
16664 assert_eq!(loose.model_identity.requested_model_id, None);
16665 assert_eq!(loose.model_identity.resolved_model_id, model_id);
16666
16667 let requests = server.received_requests().await.unwrap();
16668 assert_eq!(requests.len(), 2);
16669 for request in requests {
16670 let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
16671 assert_eq!(body["model"], provider_model);
16672 }
16673
16674 unsafe { std::env::remove_var(api_key_env) };
16675 }
16676
16677 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16683 #[tokio::test]
16684 async fn mlx_device_lock_is_singleton_and_serializes() {
16685 use std::sync::atomic::{AtomicUsize, Ordering};
16686 use std::sync::Arc;
16687 assert!(
16689 Arc::ptr_eq(
16690 &InferenceEngine::mlx_device_lock(),
16691 &InferenceEngine::mlx_device_lock()
16692 ),
16693 "device lock must be a process-wide singleton"
16694 );
16695 let inside = Arc::new(AtomicUsize::new(0));
16697 let peak = Arc::new(AtomicUsize::new(0));
16698 let mut handles = Vec::new();
16699 for _ in 0..8 {
16700 let inside = inside.clone();
16701 let peak = peak.clone();
16702 handles.push(tokio::spawn(async move {
16703 let _g = InferenceEngine::mlx_device_lock().lock_owned().await;
16704 let n = inside.fetch_add(1, Ordering::SeqCst) + 1;
16705 peak.fetch_max(n, Ordering::SeqCst);
16706 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
16707 inside.fetch_sub(1, Ordering::SeqCst);
16708 }));
16709 }
16710 for h in handles {
16711 h.await.unwrap();
16712 }
16713 assert_eq!(
16714 peak.load(Ordering::SeqCst),
16715 1,
16716 "at most one MLX device holder at a time"
16717 );
16718 }
16719
16720 #[test]
16725 fn auto_thinking_budget_gates_code_complex_and_capability() {
16726 assert_eq!(auto_thinking_budget(true, false, true), Some(24_000));
16728 assert_eq!(auto_thinking_budget(false, true, true), Some(8_000));
16730 assert_eq!(auto_thinking_budget(true, true, true), Some(24_000));
16732 assert_eq!(auto_thinking_budget(true, true, false), None);
16734 assert_eq!(auto_thinking_budget(false, false, true), None);
16736 }
16737
16738 #[test]
16739 fn is_explicit_code_intent_keys_on_caller_intent_not_keyword_classifier() {
16740 use crate::intent::{IntentHint, TaskHint};
16741 let code = IntentHint {
16743 task: Some(TaskHint::Code),
16744 ..Default::default()
16745 };
16746 assert!(is_explicit_code_intent(Some(&code)));
16747 assert!(!is_explicit_code_intent(None));
16752 let reasoning = IntentHint {
16754 task: Some(TaskHint::Reasoning),
16755 ..Default::default()
16756 };
16757 assert!(!is_explicit_code_intent(Some(&reasoning)));
16758 let unset = IntentHint {
16760 task: None,
16761 ..Default::default()
16762 };
16763 assert!(!is_explicit_code_intent(Some(&unset)));
16764 }
16765
16766 #[test]
16767 fn strict_model_suppresses_the_local_last_resort_append() {
16768 assert!(should_append_local_last_resort(false, false));
16770 assert!(!should_append_local_last_resort(false, true));
16773 assert!(!should_append_local_last_resort(true, false));
16776 assert!(!should_append_local_last_resort(true, true));
16777 }
16778
16779 #[test]
16780 fn last_resort_fallback_never_returns_an_unrunnable_apple_foundation() {
16781 let tmp = TempDir::new().unwrap();
16796 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
16797 if let Some(name) = engine.first_installed_local_model(false) {
16798 let runnable = engine
16799 .unified_registry()
16800 .find_by_name(&name)
16801 .map(|s| !s.is_foundation_models() || s.available)
16802 .unwrap_or(false);
16803 assert!(
16804 runnable,
16805 "last-resort fallback returned a non-runnable model: {name}"
16806 );
16807 }
16808 }
16809
16810 #[test]
16811 fn empty_tool_catalog_is_no_tools() {
16812 let mut req = GenerateRequest {
16819 prompt: "p".into(),
16820 ..Default::default()
16821 };
16822 assert!(!InferenceEngine::request_has_tools(&req));
16823 req.tools = Some(vec![]);
16824 assert!(!InferenceEngine::request_has_tools(&req));
16825 req.tools = Some(vec![serde_json::json!({
16826 "name": "t", "description": "d", "parameters": {"type": "object"}
16827 })]);
16828 assert!(InferenceEngine::request_has_tools(&req));
16829 }
16830
16831 #[test]
16832 fn top_k_keeps_only_k_highest() {
16833 let mut probs = vec![0.1, 0.4, 0.2, 0.25, 0.05];
16835 InferenceEngine::apply_top_k_top_p(&mut probs, 2, 1.0);
16836 assert!(probs[0] == 0.0 && probs[2] == 0.0 && probs[4] == 0.0);
16838 assert!(probs[1] > 0.0 && probs[3] > 0.0);
16839 let sum: f32 = probs.iter().sum();
16840 assert!((sum - 1.0).abs() < 1e-5, "renormalized to 1.0, got {sum}");
16841 }
16842
16843 #[test]
16844 fn top_k_zero_is_a_noop() {
16845 let mut probs = vec![0.1, 0.4, 0.2, 0.3];
16846 let before = probs.clone();
16847 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 1.0);
16848 assert_eq!(probs, before);
16849 }
16850
16851 #[test]
16852 fn top_p_nucleus_truncates_tail() {
16853 let mut probs = vec![0.6, 0.3, 0.07, 0.03];
16854 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 0.9);
16855 assert!(probs[2] == 0.0 && probs[3] == 0.0);
16857 assert!(probs[0] > 0.0 && probs[1] > 0.0);
16858 }
16859
16860 #[test]
16861 fn truncate_at_stop_excludes_stop_sequence() {
16862 let stops = vec!["<|end|>".to_string(), "STOP".to_string()];
16863 assert_eq!(
16864 tasks::generate::truncate_at_stop("hello world<|end|>extra", &stops),
16865 "hello world"
16866 );
16867 assert_eq!(
16869 tasks::generate::truncate_at_stop("aSTOPb<|end|>c", &stops),
16870 "a"
16871 );
16872 assert_eq!(
16874 tasks::generate::truncate_at_stop("clean output", &stops),
16875 "clean output"
16876 );
16877 assert_eq!(
16879 tasks::generate::truncate_at_stop("text", &["".to_string()]),
16880 "text"
16881 );
16882 }
16883
16884 #[test]
16885 fn no_backend_hint_fires_on_missing_backend_phrases() {
16886 for phrase in [
16888 "no credential for proprietary provider 'parslee'",
16889 "model not found",
16890 "no models available",
16891 "model declares ModelSource::Delegated but no inference runner is registered",
16892 ] {
16893 let hint = no_backend_recovery_hint(phrase)
16894 .unwrap_or_else(|| panic!("expected a hint for {phrase:?}"));
16895 assert!(hint.contains("car models pull"));
16896 assert!(hint.contains("car auth login"));
16899 assert!(hint.contains(phrase));
16901 }
16902 }
16903
16904 #[test]
16905 fn no_backend_hint_opens_with_the_exported_marker() {
16906 let hint = no_backend_recovery_hint(
16910 "model declares ModelSource::Delegated but no inference runner is registered",
16911 )
16912 .expect("a delegated model with no runner is a no-backend case");
16913 assert!(
16914 hint.starts_with(NO_BACKEND_RECOVERY_MARKER),
16915 "hint must open with the exported marker: {hint}"
16916 );
16917 assert!(hint.contains(NO_BACKEND_RECOVERY_MARKER), "{hint}");
16918 }
16919
16920 #[test]
16921 fn route_failure_context_keeps_the_no_backend_marker_behind_the_credential_summary() {
16922 let credential = parslee_signed_out_route_failure();
16926 let underlying = InferenceError::InferenceFailed(
16927 "model declares ModelSource::Delegated but no inference runner is registered".into(),
16928 );
16929 let augmented = apply_route_failure_context(underlying, Some(&credential));
16930 let InferenceError::InferenceFailed(message) = &augmented else {
16931 panic!("a no-backend exhaustion stays InferenceFailed: {augmented:?}");
16932 };
16933 assert!(
16934 message.starts_with(&credential.summary),
16935 "credential context is prepended: {message}"
16936 );
16937 assert!(
16938 message.contains(NO_BACKEND_RECOVERY_MARKER),
16939 "the marker survives the prepend: {message}"
16940 );
16941 assert!(
16942 !message.starts_with(NO_BACKEND_RECOVERY_MARKER),
16943 "the marker is mid-message here, which is why consumers use contains: {message}"
16944 );
16945 }
16946
16947 #[test]
16955 fn credential_failure_is_matchable_as_data() {
16956 let expired = InferenceError::CredentialUnavailable {
16957 provider: "parslee".into(),
16958 model: "parslee/reasoning".into(),
16959 reason: CredentialFailure::Expired {
16960 expires_at: 1_754_257_929,
16961 },
16962 detail: "the Parslee token expired at unix 1754257929 and could not be refreshed"
16963 .into(),
16964 };
16965 let InferenceError::CredentialUnavailable { reason, .. } = &expired else {
16966 panic!("expected CredentialUnavailable");
16967 };
16968 assert_eq!(
16969 *reason,
16970 CredentialFailure::Expired {
16971 expires_at: 1_754_257_929
16972 },
16973 "a consumer must be able to branch on the expiry without parsing prose"
16974 );
16975 assert_ne!(
16979 CredentialFailure::SignedOut,
16980 CredentialFailure::StoreUnreadable
16981 );
16982 assert_ne!(
16983 CredentialFailure::SignedOut,
16984 CredentialFailure::Expired { expires_at: 0 }
16985 );
16986 assert_ne!(
16987 CredentialFailure::StoreUnreadable,
16988 CredentialFailure::RaceRetryable
16989 );
16990 }
16991
16992 #[test]
17002 fn typed_credential_error_still_satisfies_the_substring_classifiers() {
17003 for reason in [
17004 CredentialFailure::Expired { expires_at: 1 },
17005 CredentialFailure::SignedOut,
17006 CredentialFailure::StoreUnreadable,
17007 CredentialFailure::RaceRetryable,
17008 CredentialFailure::EnvVarMissing {
17009 env_var: "OPENAI_API_KEY".into(),
17010 },
17011 ] {
17012 let rendered = InferenceError::CredentialUnavailable {
17013 provider: "parslee".into(),
17014 model: "parslee/reasoning".into(),
17015 reason: reason.clone(),
17016 detail: "detail text".into(),
17017 }
17018 .to_string();
17019 assert!(
17021 rendered
17022 .to_ascii_lowercase()
17023 .contains("no credential for proprietary"),
17024 "classifier substring lost for {reason:?}: {rendered}"
17025 );
17026 assert!(rendered.contains("parslee/reasoning"), "{rendered}");
17028 assert!(rendered.contains("detail text"), "{rendered}");
17030 }
17031 }
17032
17033 #[test]
17034 fn auth_expired_hint_fires_on_auth_rejection_but_not_transient() {
17035 for phrase in [
17037 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17038 "HTTP 403: forbidden",
17039 "invalid_grant: The refresh token is invalid or expired",
17040 "token expired",
17041 ] {
17042 let hint = auth_expired_recovery_hint(phrase)
17043 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
17044 assert!(hint.contains("car auth login"));
17045 assert!(hint.contains(phrase));
17046 }
17047 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
17049 assert!(auth_expired_recovery_hint("request timed out").is_none());
17050 }
17051
17052 #[test]
17070 fn auth_expired_hint_never_carries_the_no_backend_marker() {
17071 for phrase in [
17074 "Parslee org lookup failed: the account service could not be reached",
17075 "Authentication required",
17076 "HTTP 401 Unauthorized",
17077 "HTTP 403 Forbidden",
17078 "invalid_grant: the refresh token is invalid or expired",
17079 "token expired",
17080 ] {
17081 let hint = auth_expired_recovery_hint(phrase)
17082 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
17083 assert!(
17084 !hint.contains(NO_BACKEND_RECOVERY_MARKER),
17085 "the auth-expired hint now carries the no-backend marker, so \
17086 car infer would auto-pull for {phrase:?}: {hint}"
17087 );
17088 assert!(hint.contains("car models pull"), "{hint}");
17092 }
17093 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
17096 }
17097
17098 #[test]
17103 fn auth_rejection_classifier_matches_the_real_expired_token_error() {
17104 assert!(is_auth_rejection_message(
17105 "inference failed: Parslee org lookup failed: HTTP 401 Unauthorized: \
17106 Authentication required"
17107 ));
17108 for transient in [
17111 "API returned 503: service unavailable",
17112 "request timed out",
17113 "connection reset by peer",
17114 ] {
17115 assert!(
17116 !is_auth_rejection_message(transient),
17117 "transient wrongly classified as an auth rejection: {transient:?}"
17118 );
17119 }
17120 }
17121
17122 #[test]
17130 fn typed_errors_classify_by_their_structure_not_their_prose() {
17131 use FallbackReason as R;
17132 let cases: Vec<(InferenceError, R)> = vec![
17133 (
17136 InferenceError::ProviderAccount {
17137 provider: "openai".into(),
17138 status: 401,
17139 message: "provider rejected the API key — check the configured credential"
17140 .into(),
17141 },
17142 R::CredentialRejected,
17143 ),
17144 (
17146 InferenceError::ProviderAccount {
17147 provider: "openrouter".into(),
17148 status: 402,
17149 message: "OpenRouter account is out of credits".into(),
17150 },
17151 R::QuotaExhausted,
17153 ),
17154 (
17155 InferenceError::CredentialUnavailable {
17156 provider: "parslee".into(),
17157 model: "parslee/reasoning".into(),
17158 reason: CredentialFailure::Expired { expires_at: 0 },
17159 detail: "session expired".into(),
17160 },
17161 R::CredentialRejected,
17162 ),
17163 (
17164 InferenceError::CredentialUnavailable {
17165 provider: "openai".into(),
17166 model: "openai/gpt-5.6".into(),
17167 reason: CredentialFailure::EnvVarMissing {
17168 env_var: "OPENAI_API_KEY".into(),
17169 },
17170 detail: "not set".into(),
17171 },
17172 R::CredentialAbsent,
17173 ),
17174 (
17177 InferenceError::CredentialUnavailable {
17178 provider: "parslee".into(),
17179 model: "parslee/reasoning".into(),
17180 reason: CredentialFailure::StoreUnreadable,
17181 detail: "the bounded Keychain helper timed out".into(),
17182 },
17183 R::Failed,
17184 ),
17185 (
17186 InferenceError::Transient {
17187 status: Some(429),
17188 message: "slow down".into(),
17189 },
17190 R::RateLimited,
17191 ),
17192 (
17197 InferenceError::Transient {
17198 status: None,
17199 message: "connection reset by peer".into(),
17200 },
17201 R::Failed,
17202 ),
17203 (
17204 InferenceError::Transient {
17205 status: Some(503),
17206 message: "upstream down".into(),
17207 },
17208 R::Failed,
17209 ),
17210 ];
17211 for (err, want) in cases {
17212 assert_eq!(classify_fallback_reason(&err), want, "{err}");
17213 }
17214 }
17215
17216 #[test]
17224 fn a_quoted_status_in_a_provider_body_does_not_decide_the_bucket() {
17225 use FallbackReason as R;
17226 for (msg, want) in [
17227 (
17228 "API returned 400 Bad Request: your last request 401'd upstream and was unauthorized",
17229 R::Failed,
17230 ),
17231 ("API returned 400 Bad Request: timeout param invalid", R::Failed),
17232 ("API returned 429 Too Many Requests: slow down", R::RateLimited),
17233 ("API returned 401 Unauthorized: bad key", R::CredentialRejected),
17234 ] {
17235 assert_eq!(
17236 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
17237 want,
17238 "{msg}"
17239 );
17240 }
17241 }
17242
17243 #[test]
17247 fn a_parslee_org_lookup_failure_is_classified_by_its_status() {
17248 use FallbackReason as R;
17249 for (msg, want) in [
17250 (
17251 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17252 R::CredentialRejected,
17253 ),
17254 (
17255 "Parslee org lookup failed: HTTP 429 Too Many Requests: slow down",
17256 R::RateLimited,
17257 ),
17258 (
17259 "Parslee org lookup failed: HTTP 500 Internal Server Error: boom",
17260 R::Failed,
17261 ),
17262 ] {
17263 assert_eq!(
17264 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
17265 want,
17266 "{msg}"
17267 );
17268 }
17269 }
17270
17271 #[test]
17275 fn every_skipped_lane_is_recorded_in_order() {
17276 let mut hops = Vec::new();
17277 for (cand, err) in [
17278 (
17279 "lane-one",
17280 InferenceError::Transient {
17281 status: Some(429),
17282 message: "x".into(),
17283 },
17284 ),
17285 (
17288 "lane-two",
17289 InferenceError::Transient {
17290 status: None,
17291 message: "connection refused".into(),
17292 },
17293 ),
17294 (
17295 "lane-three",
17296 InferenceError::ProviderAccount {
17297 provider: "openai".into(),
17298 status: 401,
17299 message: "rejected".into(),
17300 },
17301 ),
17302 ] {
17303 record_fallback_from(&mut hops, cand, &err);
17304 }
17305 assert_eq!(
17306 hops.iter()
17307 .map(|h| h.candidate.as_str())
17308 .collect::<Vec<_>>(),
17309 ["lane-one", "lane-two", "lane-three"]
17310 );
17311 assert_eq!(
17312 hops.iter().map(|h| h.reason).collect::<Vec<_>>(),
17313 [
17314 FallbackReason::RateLimited,
17315 FallbackReason::Failed,
17316 FallbackReason::CredentialRejected
17317 ]
17318 );
17319 }
17320
17321 #[test]
17329 fn a_refused_api_key_is_journaled_but_does_not_claim_sign_in_fixes_it() {
17330 let refused = InferenceError::ProviderAccount {
17331 provider: "openai".into(),
17332 status: 401,
17333 message: "provider rejected the API key — check the configured credential".into(),
17334 };
17335 let mut hops = Vec::new();
17336 record_fallback_from(&mut hops, "openai/gpt-5.6", &refused);
17337 assert_eq!(hops[0].reason, FallbackReason::CredentialRejected);
17338
17339 let mut auth = None;
17341 record_auth_dead_lane(&mut auth, "openai/gpt-5.6", &refused.to_string());
17342 assert_eq!(auth, None, "car auth login does not fix a bad OpenAI key");
17343 }
17344
17345 #[test]
17346 fn auth_dead_lane_records_first_rejected_candidate_only() {
17347 let mut slot: Option<String> = None;
17349 record_auth_dead_lane(
17350 &mut slot,
17351 "parslee/reasoning",
17352 "API returned 503: unavailable",
17353 );
17354 record_auth_dead_lane(&mut slot, "openai/gpt-5.6", "request timed out");
17355 assert_eq!(slot, None);
17356
17357 record_auth_dead_lane(
17359 &mut slot,
17360 "parslee/reasoning",
17361 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17362 );
17363 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
17364
17365 record_auth_dead_lane(&mut slot, "anthropic/claude", "HTTP 403: forbidden");
17368 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
17369 }
17370
17371 #[test]
17372 fn configured_provider_with_expired_token_is_named_before_a_local_oom() {
17373 let expired = InferenceError::CredentialUnavailable {
17374 provider: "parslee".into(),
17375 model: "parslee/reasoning".into(),
17376 reason: CredentialFailure::Expired { expires_at: 42 },
17377 detail: "access token expired".into(),
17378 };
17379 let mut credential = None;
17380 record_route_credential_failure(&mut credential, "parslee/reasoning", &expired, false);
17381
17382 let error = apply_route_failure_context(
17383 InferenceError::InferenceFailed(
17384 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17385 .into(),
17386 ),
17387 credential.as_ref(),
17388 )
17389 .to_string();
17390
17391 let auth_pos = error
17392 .find("Parslee login expired")
17393 .expect("expired login must be named");
17394 let remedy_pos = error
17395 .find("car auth login")
17396 .expect("credential remedy must be named");
17397 let oom_pos = error
17398 .find("9059 MB")
17399 .expect("fallback error must remain as secondary detail");
17400 assert!(auth_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
17401 assert!(is_auth_failure_message(&error), "{error}");
17402 }
17403
17404 #[test]
17405 fn absent_login_is_named_before_a_local_oom() {
17406 let failure = parslee_signed_out_route_failure();
17407 let error = apply_route_failure_context(
17408 InferenceError::InferenceFailed("local fallback needs 9059 MB".into()),
17409 Some(&failure),
17410 )
17411 .to_string();
17412
17413 let absent_pos = error
17414 .find("Parslee login is absent")
17415 .expect("missing login must be named");
17416 let remedy_pos = error
17417 .find("car auth login")
17418 .expect("credential remedy must be named");
17419 let oom_pos = error
17420 .find("9059 MB")
17421 .expect("fallback OOM must remain as secondary detail");
17422 assert!(absent_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
17423 }
17424
17425 #[test]
17426 fn genuine_local_oom_is_not_reclassified_as_auth() {
17427 let oom = InferenceError::InferenceFailed(
17428 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17429 .into(),
17430 );
17431 let error = apply_route_failure_context(oom, None);
17432 assert!(matches!(error, InferenceError::InferenceFailed(ref message)
17433 if message.starts_with("This model needs about 9059 MB")));
17434 }
17435
17436 #[test]
17437 fn unconfigured_provider_is_not_surfaced_over_the_terminal_failure() {
17438 let mut credential = None;
17439 record_route_credential_failure(
17440 &mut credential,
17441 "openai/gpt-5.6",
17442 &InferenceError::CredentialUnavailable {
17443 provider: "openai".into(),
17444 model: "openai/gpt-5.6".into(),
17445 reason: CredentialFailure::EnvVarMissing {
17446 env_var: "OPENAI_API_KEY".into(),
17447 },
17448 detail: "set OPENAI_API_KEY".into(),
17449 },
17450 false,
17451 );
17452
17453 let terminal = InferenceError::InferenceFailed(
17454 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17455 .into(),
17456 );
17457 let error = apply_route_failure_context(terminal, credential.as_ref()).to_string();
17458 assert!(
17459 credential.is_none(),
17460 "an unconfigured fallback is ambient noise"
17461 );
17462 assert!(error.contains("9059 MB"), "{error}");
17463 assert!(!error.contains("OPENAI_API_KEY"), "{error}");
17464 assert!(!is_auth_failure_message(&error), "{error}");
17465 }
17466
17467 fn workspace_required() -> InferenceError {
17471 InferenceError::WorkspaceRequired {
17472 provider: "Parslee".into(),
17473 detail: "finish setting up at https://parslee.ai, then try again".into(),
17474 }
17475 }
17476
17477 #[test]
17486 fn workspace_required_reads_as_configuration_not_sign_in() {
17487 let rendered = workspace_required().to_string();
17488 for marker in AUTH_FAILURE_MESSAGE_MARKERS {
17489 assert!(
17490 !rendered
17491 .to_ascii_lowercase()
17492 .contains(&marker.to_ascii_lowercase()),
17493 "`{marker}` must not appear in the no-workspace text: {rendered}"
17494 );
17495 }
17496 assert!(!is_auth_failure_message(&rendered), "{rendered}");
17497 assert!(!is_auth_rejection_message(&rendered), "{rendered}");
17498 assert!(rendered.contains("https://parslee.ai"), "{rendered}");
17499 }
17500
17501 #[test]
17508 fn workspace_required_survives_a_mixed_chain() {
17509 let mut credential = None;
17510 record_route_credential_failure(
17511 &mut credential,
17512 "openai/gpt-5.6",
17513 &InferenceError::CredentialUnavailable {
17514 provider: "openai".into(),
17515 model: "openai/gpt-5.6".into(),
17516 reason: CredentialFailure::SignedOut,
17517 detail: "no account is signed in".into(),
17518 },
17519 true,
17520 );
17521 assert!(
17522 credential.is_some(),
17523 "the fixture must actually record a credential failure"
17524 );
17525
17526 let final_error = apply_route_failure_context(workspace_required(), credential.as_ref());
17527 assert!(
17528 matches!(final_error, InferenceError::WorkspaceRequired { .. }),
17529 "the type must survive the route context: {final_error}"
17530 );
17531 assert!(!is_auth_failure_message(&final_error.to_string()));
17532 assert!(
17533 error_ends_fallback_chain(&final_error),
17534 "no later candidate on that account can succeed"
17535 );
17536 }
17537
17538 #[test]
17545 fn repeated_workspace_required_leaves_model_health_untouched() {
17546 assert!(
17547 !error_counts_against_circuit_breaker(&workspace_required()),
17548 "the breaker must not count a workspace gap"
17549 );
17550
17551 let mut tracker = OutcomeTracker::new();
17552 for _ in 0..5 {
17553 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
17554 record_dispatch_failure(&mut tracker, &trace, &workspace_required());
17555 }
17556 let failed = tracker
17557 .profile("parslee/advisor")
17558 .map(|p| p.fail_count)
17559 .unwrap_or(0);
17560 assert_eq!(
17561 failed, 0,
17562 "five workspace refusals must not bench the model"
17563 );
17564
17565 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
17566 record_dispatch_failure(
17567 &mut tracker,
17568 &trace,
17569 &InferenceError::InferenceFailed("decoder fell over".into()),
17570 );
17571 assert_eq!(
17572 tracker
17573 .profile("parslee/advisor")
17574 .map(|p| p.fail_count)
17575 .unwrap_or(0),
17576 1,
17577 "a genuine failure must still count, or the test above proves nothing"
17578 );
17579 }
17580
17581 #[test]
17582 fn explicitly_requested_missing_credential_uses_the_shared_auth_table() {
17583 let mut credential = None;
17584 record_route_credential_failure(
17585 &mut credential,
17586 "openai/gpt-5.6",
17587 &InferenceError::CredentialUnavailable {
17588 provider: "openai".into(),
17589 model: "openai/gpt-5.6".into(),
17590 reason: CredentialFailure::EnvVarMissing {
17591 env_var: "OPENAI_API_KEY".into(),
17592 },
17593 detail: "set OPENAI_API_KEY".into(),
17594 },
17595 true,
17596 );
17597 let summary = credential
17598 .expect("an explicitly requested provider must surface its missing key")
17599 .summary;
17600 assert!(summary.contains(AUTH_ENV_MISSING_MARKER), "{summary}");
17601 assert!(is_auth_failure_message(&summary), "{summary}");
17602 }
17603
17604 #[test]
17605 fn latest_actionable_credential_failure_wins() {
17606 let mut credential = None;
17607 record_route_credential_failure(
17608 &mut credential,
17609 "parslee/reasoning",
17610 &InferenceError::ProviderAccount {
17611 provider: "parslee".into(),
17612 status: 401,
17613 message: "Unauthorized".into(),
17614 },
17615 false,
17616 );
17617 record_route_credential_failure(
17618 &mut credential,
17619 "anthropic/claude",
17620 &InferenceError::CredentialUnavailable {
17621 provider: "anthropic".into(),
17622 model: "anthropic/claude".into(),
17623 reason: CredentialFailure::Expired { expires_at: 43 },
17624 detail: "configured token expired".into(),
17625 },
17626 false,
17627 );
17628
17629 assert_eq!(
17630 credential
17631 .expect("latest actionable credential cause must be retained")
17632 .summary,
17633 "anthropic login expired for `anthropic/claude` — run `car auth login`"
17634 );
17635 }
17636
17637 #[test]
17638 fn store_unreadable_summary_uses_the_shared_auth_table() {
17639 let summary = route_credential_failure(
17640 "parslee/reasoning",
17641 &InferenceError::CredentialUnavailable {
17642 provider: "parslee".into(),
17643 model: "parslee/reasoning".into(),
17644 reason: CredentialFailure::StoreUnreadable,
17645 detail: "keychain helper timed out".into(),
17646 },
17647 false,
17648 )
17649 .expect("an unreadable configured credential store is actionable");
17650 assert!(summary.contains(AUTH_STORE_UNREADABLE_MARKER), "{summary}");
17651 assert!(is_auth_failure_message(&summary), "{summary}");
17652 }
17653
17654 fn chain_gate_fixture_schema(id: &str, provider: &str, source: ModelSource) -> ModelSchema {
17655 ModelSchema {
17656 id: id.into(),
17657 name: id.into(),
17658 provider: provider.into(),
17659 family: "test".into(),
17660 version: "1".into(),
17661 capabilities: vec![ModelCapability::Generate],
17662 context_length: 32_768,
17663 max_output_tokens: Some(4_096),
17664 param_count: String::new(),
17665 quantization: None,
17666 performance: Default::default(),
17667 cost: Default::default(),
17668 source,
17669 tags: vec![],
17670 supported_params: vec![],
17671 public_benchmarks: vec![],
17672 trust_tier: TrustTier::Community,
17673 deprecated: false,
17674 available: true,
17675 weights_ready: true,
17676 }
17677 }
17678
17679 #[test]
17680 fn signed_out_pre_seed_is_gated_on_a_parslee_route_in_the_chain() {
17681 let local = chain_gate_fixture_schema(
17682 "qwen/qwen3-4b:q4_k_m",
17683 "qwen",
17684 ModelSource::Mlx {
17685 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
17686 hf_weight_file: None,
17687 },
17688 );
17689 let ollama = chain_gate_fixture_schema(
17690 "ollama/llama3",
17691 "ollama",
17692 ModelSource::Ollama {
17693 model_tag: "llama3".into(),
17694 host: "http://localhost:11434".into(),
17695 },
17696 );
17697 let parslee = chain_gate_fixture_schema(
17698 "parslee/reasoning",
17699 "parslee",
17700 ModelSource::Proprietary {
17701 provider: "parslee".into(),
17702 endpoint: "https://api.parslee.ai".into(),
17703 auth: ProprietaryAuth::OAuth2Pkce {
17704 authority: "https://login.example".into(),
17705 client_id: "client".into(),
17706 scopes: vec![],
17707 },
17708 protocol: Default::default(),
17709 },
17710 );
17711 let cloud = chain_gate_fixture_schema(
17712 "openai/gpt-5.6",
17713 "openai",
17714 ModelSource::RemoteApi {
17715 endpoint: "https://api.openai.com".into(),
17716 api_key_env: "OPENAI_API_KEY".into(),
17717 api_key_envs: vec![],
17718 api_version: None,
17719 protocol: schema::ApiProtocol::OpenAiCompat,
17720 },
17721 );
17722 let schemas: std::collections::HashMap<&str, &ModelSchema> = [
17723 ("qwen/qwen3-4b:q4_k_m", &local),
17724 ("ollama/llama3", &ollama),
17725 ("parslee/reasoning", &parslee),
17726 ("openai/gpt-5.6", &cloud),
17727 ]
17728 .into_iter()
17729 .collect();
17730 let resolve = |m: &str| schemas.get(m).copied();
17731
17732 let local_only = vec![
17734 "qwen/qwen3-4b:q4_k_m".to_string(),
17735 "ollama/llama3".to_string(),
17736 ];
17737 assert!(!chain_includes_parslee_route(resolve, &local_only));
17738
17739 let with_parslee = vec![
17741 "parslee/reasoning".to_string(),
17742 "qwen/qwen3-4b:q4_k_m".to_string(),
17743 ];
17744 assert!(chain_includes_parslee_route(resolve, &with_parslee));
17745 let with_cloud = vec![
17746 "openai/gpt-5.6".to_string(),
17747 "qwen/qwen3-4b:q4_k_m".to_string(),
17748 ];
17749 assert!(!chain_includes_parslee_route(resolve, &with_cloud));
17753 let mut credential = chain_includes_parslee_route(resolve, &with_cloud)
17754 .then(parslee_signed_out_route_failure);
17755 let outage = InferenceError::Transient {
17756 status: Some(500),
17757 message: "OpenAI HTTP 500 Internal Server Error".into(),
17758 };
17759 record_route_credential_failure(&mut credential, &with_cloud[0], &outage, false);
17760 let terminal = apply_route_failure_context(outage, credential.as_ref());
17761 assert!(matches!(terminal, InferenceError::Transient { .. }));
17762 assert!(!is_auth_failure_message(&terminal.to_string()));
17763 let oom = InferenceError::InferenceFailed("local model out of memory".into());
17764 record_route_credential_failure(&mut credential, &with_cloud[1], &oom, false);
17765 let terminal = apply_route_failure_context(oom, credential.as_ref()).to_string();
17766 assert!(!terminal.contains("Parslee"), "{terminal}");
17767 assert!(!is_auth_failure_message(&terminal), "{terminal}");
17768
17769 record_route_credential_failure(
17771 &mut credential,
17772 &with_cloud[0],
17773 &InferenceError::ProviderAccount {
17774 provider: "openai".into(),
17775 status: 401,
17776 message: "invalid API key".into(),
17777 },
17778 false,
17779 );
17780 let terminal = apply_route_failure_context(
17781 InferenceError::InferenceFailed("local model out of memory".into()),
17782 credential.as_ref(),
17783 )
17784 .to_string();
17785 assert!(terminal.contains("openai"), "{terminal}");
17786 assert!(!terminal.contains("Parslee"), "{terminal}");
17787 assert!(is_auth_failure_message(&terminal), "{terminal}");
17788
17789 let unknown = vec!["missing/model".to_string()];
17791 assert!(!chain_includes_parslee_route(resolve, &unknown));
17792 }
17793
17794 #[test]
17798 fn local_only_terminal_failures_are_not_relabeled_as_auth() {
17799 let cases = [
17800 InferenceError::InferenceFailed(
17802 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17803 .into(),
17804 ),
17805 InferenceError::Transient {
17807 status: Some(500),
17808 message: "HTTP 500 Internal Server Error".into(),
17809 },
17810 InferenceError::InferenceFailed("llama runner process has terminated".into()),
17812 ];
17813 for case in cases {
17814 let error = apply_route_failure_context(case, None).to_string();
17815 assert!(!error.contains("Parslee login is absent"), "{error}");
17816 assert!(!is_auth_failure_message(&error), "{error}");
17817 }
17818
17819 let error = apply_route_failure_context(
17823 InferenceError::ModelNotFound("qwen/qwen3-4b:q4_k_m".into()),
17824 None,
17825 )
17826 .to_string();
17827 assert!(!error.contains("Parslee login is absent"), "{error}");
17828 assert!(error.contains("car models pull"), "{error}");
17829 }
17830
17831 #[test]
17834 fn credential_context_preserves_the_typed_terminal_variant() {
17835 let credential = parslee_signed_out_route_failure();
17836
17837 let transient = apply_route_failure_context(
17838 InferenceError::Transient {
17839 status: Some(500),
17840 message: "HTTP 500 Internal Server Error".into(),
17841 },
17842 Some(&credential),
17843 );
17844 match &transient {
17845 InferenceError::Transient { status, message } => {
17846 assert_eq!(*status, Some(500));
17847 assert!(message.starts_with("Parslee login is absent"), "{message}");
17848 assert!(message.contains("HTTP 500"), "{message}");
17849 }
17850 other => panic!("Transient must stay Transient, got {other:?}"),
17851 }
17852
17853 let account = apply_route_failure_context(
17854 InferenceError::ProviderAccount {
17855 provider: "openai".into(),
17856 status: 402,
17857 message: "insufficient credits".into(),
17858 },
17859 Some(&credential),
17860 );
17861 assert!(
17862 matches!(
17863 &account,
17864 InferenceError::ProviderAccount {
17865 provider,
17866 status: 402,
17867 ..
17868 } if provider == "openai"
17869 ),
17870 "ProviderAccount must stay ProviderAccount, got {account:?}"
17871 );
17872
17873 let unavailable = apply_route_failure_context(
17874 InferenceError::CredentialUnavailable {
17875 provider: "parslee".into(),
17876 model: "parslee/reasoning".into(),
17877 reason: CredentialFailure::Expired { expires_at: 42 },
17878 detail: "access token expired".into(),
17879 },
17880 Some(&credential),
17881 );
17882 match &unavailable {
17883 InferenceError::CredentialUnavailable { reason, detail, .. } => {
17884 assert_eq!(*reason, CredentialFailure::Expired { expires_at: 42 });
17885 assert!(detail.contains("Parslee login is absent"), "{detail}");
17886 }
17887 other => panic!("CredentialUnavailable must keep its reason data, got {other:?}"),
17888 }
17889 assert!(is_auth_failure_message(&unavailable.to_string()));
17890 }
17891
17892 #[test]
17895 fn non_parslee_rejection_summaries_match_the_shared_classifier() {
17896 let rejected = route_credential_failure(
17897 "openai/gpt-5.6",
17898 &InferenceError::ProviderAccount {
17899 provider: "openai".into(),
17900 status: 403,
17901 message: "key revoked".into(),
17902 },
17903 false,
17904 )
17905 .expect("a 403 from a configured provider is actionable");
17906 assert!(rejected.contains("credential was rejected"), "{rejected}");
17907 assert!(is_auth_failure_message(&rejected), "{rejected}");
17908
17909 let generic = route_credential_failure(
17910 "openai/gpt-5.6",
17911 &InferenceError::InferenceFailed("upstream said: token expired".into()),
17912 false,
17913 )
17914 .expect("an auth-rejection message from a non-Parslee route is actionable");
17915 assert!(generic.contains("repair its provider login"), "{generic}");
17916 assert!(is_auth_failure_message(&generic), "{generic}");
17917 }
17918
17919 #[test]
17924 fn the_recovery_hints_lead_with_parslee_sign_in() {
17925 for hint in [
17926 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
17927 .expect("a missing credential is a no-backend case"),
17928 auth_expired_recovery_hint(
17929 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17930 )
17931 .expect("a rejected session is an auth case"),
17932 ] {
17933 let login = hint
17934 .find("car auth login")
17935 .expect("sign-in must be offered");
17936 let pull = hint
17937 .find("car models pull")
17938 .expect("the local path must still be offered");
17939 assert!(login < pull, "sign-in must be read first:\n{hint}");
17940 }
17941 assert!(
17944 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
17945 .expect("hint")
17946 .starts_with(NO_BACKEND_RECOVERY_MARKER),
17947 "the marker opening is load-bearing"
17948 );
17949 }
17950
17951 #[test]
17954 fn fresh_install_exhaustion_keeps_the_models_pull_guidance() {
17955 let credential = parslee_signed_out_route_failure();
17956 let error = apply_route_failure_context(
17957 InferenceError::InferenceFailed("no models available for generate".into()),
17958 Some(&credential),
17959 )
17960 .to_string();
17961 assert!(error.contains("Parslee login is absent"), "{error}");
17962 assert!(
17963 error.contains(&format!("car models pull {LOCAL_TOOL_MODEL_PULL_ID}")),
17964 "{error}"
17965 );
17966 assert!(error.contains("car auth login"), "{error}");
17967 }
17968
17969 #[test]
17972 fn auth_fallback_from_round_trips_and_defaults_to_none() {
17973 let mut result: InferenceResult = serde_json::from_value(serde_json::json!({
17974 "text": "hi",
17975 "tool_calls": [],
17976 "trace_id": "t",
17977 "model_used": "openai/gpt-5.6",
17978 "latency_ms": 1,
17979 }))
17980 .expect("a payload without the field still deserializes");
17981 assert_eq!(result.auth_fallback_from, None);
17982 let json = serde_json::to_value(&result).unwrap();
17983 assert!(json.get("auth_fallback_from").is_none());
17984
17985 result.auth_fallback_from = Some("parslee/reasoning".to_string());
17986 let json = serde_json::to_value(&result).unwrap();
17987 assert_eq!(json["auth_fallback_from"], "parslee/reasoning");
17988 let back: InferenceResult = serde_json::from_value(json).unwrap();
17989 assert_eq!(
17990 back.auth_fallback_from.as_deref(),
17991 Some("parslee/reasoning")
17992 );
17993 }
17994
17995 #[test]
18000 fn local_last_resort_turn_carries_model_id_and_flag() {
18001 let ordinary: InferenceResult = serde_json::from_value(serde_json::json!({
18002 "text": "hi",
18003 "tool_calls": [],
18004 "trace_id": "ordinary",
18005 "model_used": "anthropic/claude-haiku-4-5:latest",
18006 "latency_ms": 1,
18007 }))
18008 .expect("older payloads default the marker");
18009 assert!(!ordinary.local_last_resort);
18010
18011 let fallback: InferenceResult = serde_json::from_value(serde_json::json!({
18012 "text": "offline answer",
18013 "tool_calls": [],
18014 "trace_id": "fallback",
18015 "model_used": "Qwen3 4B MLX",
18016 "resolved_model_id": "mlx/qwen3-4b:4bit",
18017 "latency_ms": 1,
18018 "local_last_resort": true,
18019 }))
18020 .expect("fallback attribution payload");
18021 assert_eq!(fallback.served_model_id(), "mlx/qwen3-4b:4bit");
18022 assert!(fallback.local_last_resort);
18023
18024 assert!(!is_local_last_resort(None, "mlx/qwen3-4b:4bit"));
18025 assert!(report_local_last_resort_served(
18026 Some("mlx/qwen3-4b:4bit"),
18027 "mlx/qwen3-4b:4bit",
18028 "mlx/qwen3-4b:4bit"
18029 ));
18030 assert!(!is_local_last_resort(
18031 Some("mlx/qwen3-4b:4bit"),
18032 "anthropic/claude-haiku-4-5:latest"
18033 ));
18034 }
18035
18036 #[test]
18037 fn no_backend_hint_passes_through_transient_errors() {
18038 for phrase in [
18041 "API returned 401 Unauthorized",
18042 "API returned 429 Too Many Requests",
18043 "API returned 500 Internal Server Error",
18044 "connection refused",
18045 "request timed out",
18046 "parse response: unexpected end of input",
18047 ] {
18048 assert!(
18049 no_backend_recovery_hint(phrase).is_none(),
18050 "transient error wrongly classified as no-backend: {phrase:?}"
18051 );
18052 }
18053 }
18054
18055 static ENV_MUTEX: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
18058
18059 #[cfg(unix)]
18060 #[tokio::test]
18061 async fn exact_codex_subscription_row_dispatches_without_an_openai_key() {
18062 use std::os::unix::fs::PermissionsExt;
18063
18064 let _environment = ENV_MUTEX.lock().await;
18065 let _restore = RestoredEnvironment::capture(&["CAR_CODEX_BIN", "OPENAI_API_KEY"]);
18066 let tmp = TempDir::new().unwrap();
18067 let fixture = tmp.path().join("codex-fixture.sh");
18068 std::fs::write(
18069 &fixture,
18070 r#"#!/bin/sh
18071if [ -n "${OPENAI_API_KEY-}" ]; then
18072 echo 'OPENAI_API_KEY leaked' >&2
18073 exit 91
18074fi
18075cat >/dev/null
18076printf '%s\n' '{"type":"turn.started"}'
18077printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"fixture newsroom answer"}}'
18078printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":17,"output_tokens":5}}'
18079"#,
18080 )
18081 .unwrap();
18082 std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o700)).unwrap();
18083 unsafe {
18084 std::env::set_var("CAR_CODEX_BIN", &fixture);
18085 std::env::set_var("OPENAI_API_KEY", "must-not-reach-codex");
18086 }
18087
18088 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
18089 let mut request = GenerateRequest {
18090 prompt: "write a brief".into(),
18091 params: GenerateParams {
18092 max_tokens: 1056,
18093 strict_model: true,
18094 ..Default::default()
18095 },
18096 ..Default::default()
18097 };
18098 pin_exact_model_id(&mut request, "openai/gpt-5.6-sol:high".into()).unwrap();
18099 let result = engine.generate_tracked(request).await.unwrap();
18100
18101 assert_eq!(result.model_used, "openai/gpt-5.6-sol:high");
18102 assert_eq!(result.text, "fixture newsroom answer");
18103 assert_eq!(result.usage.as_ref().unwrap().total_tokens, 22);
18104 assert_eq!(result.stop_reason, None, "Codex reports no finish reason");
18105 assert!(result.tool_calls.is_empty());
18106 }
18107
18108 #[tokio::test]
18109 async fn isolated_test_child_removes_parent_auth_lock_override() {
18110 const CHILD_SENTINEL: &str = "CAR_AUTH_LOCK_SANITIZER_CHILD";
18111 if std::env::var_os(CHILD_SENTINEL).is_some() {
18112 assert!(
18113 std::env::var_os("CAR_AUTH_LOCK_PATH").is_none(),
18114 "an isolated credential test must not inherit the invoking process's auth lock"
18115 );
18116 return;
18117 }
18118
18119 let _environment = ENV_MUTEX.lock().await;
18120 let _restore = RestoredEnvironment::capture(&["CAR_AUTH_LOCK_PATH"]);
18121 unsafe {
18122 std::env::set_var(
18123 "CAR_AUTH_LOCK_PATH",
18124 "/sentinel/must-not-reach-isolated-child.lock",
18125 );
18126 }
18127 assert!(!crate::run_in_isolated_test_process(
18128 "tests::isolated_test_child_removes_parent_auth_lock_override",
18129 CHILD_SENTINEL,
18130 ));
18131 }
18132
18133 fn test_config(models_dir: PathBuf) -> InferenceConfig {
18134 let state_root = models_dir
18138 .parent()
18139 .map(Path::to_path_buf)
18140 .unwrap_or_else(|| models_dir.clone());
18141 InferenceConfig {
18142 models_dir,
18143 state_root,
18144 device: None,
18145 generation_model: "Qwen3-0.6B".into(),
18146 preferred_generation_model: None,
18147 embedding_model: "Qwen3-Embedding-0.6B".into(),
18148 preferred_embedding_model: None,
18149 classification_model: "Qwen3-0.6B".into(),
18150 preferred_classification_model: None,
18151 }
18152 }
18153
18154 fn metal_mac_for_fit(ram_gb: u64) -> HardwareInfo {
18155 HardwareInfo {
18156 os: "macos".into(),
18157 arch: "aarch64".into(),
18158 cpu_cores: 8,
18159 total_ram_mb: ram_gb * 1024,
18160 gpu_backend: crate::hardware::GpuBackend::Metal,
18161 gpu_memory_mb: None,
18162 gpu_devices: vec![],
18163 recommended_model: String::new(),
18164 recommended_context: 8_192,
18165 max_model_mb: 0,
18166 }
18167 }
18168
18169 #[test]
18176 fn unified_rows_carry_fit_per_machine_and_keep_deprecated_rows() {
18177 if !crate::run_in_isolated_test_process(
18178 "tests::unified_rows_carry_fit_per_machine_and_keep_deprecated_rows",
18179 "CAR_UNIFIED_ROWS_FIT_TEST_CHILD",
18180 ) {
18181 return;
18182 }
18183
18184 let _credential_scope = crate::openrouter::test_credential_scope();
18191 crate::openrouter::set_test_credential(Some("unified-rows-fit-test-key"));
18192 let secret_activity_before = car_secrets::secret_store_activity();
18193 let root = tempfile::tempdir().unwrap();
18194 let config = test_config(root.path().join("weights"));
18195 let mut engine = InferenceEngine::new(config);
18196 let mut retired = engine
18197 .list_schemas()
18198 .into_iter()
18199 .find(|schema| schema.id == "mlx/qwen3-4b:4bit")
18200 .expect("builtin 4B MLX row");
18201 retired.id = "test/retired-4b:4bit".into();
18202 retired.name = "Retired 4B".into();
18203 retired.deprecated = true;
18204 engine.register_model(retired);
18205 let policy = resource_policy::ResourcePolicy::everyday();
18206
18207 let at_8 = engine.list_models_unified_for(&metal_mac_for_fit(8), &policy);
18208 let at_32 = engine.list_models_unified_for(&metal_mac_for_fit(32), &policy);
18209 let ids = |rows: &[ModelInfo]| rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
18210 assert_eq!(
18211 ids(&at_8),
18212 ids(&at_32),
18213 "the machine never removes or reorders a row"
18214 );
18215 let row = |rows: &[ModelInfo], id: &str| {
18216 rows.iter()
18217 .find(|row| row.id == id)
18218 .unwrap_or_else(|| panic!("{id} missing"))
18219 .clone()
18220 };
18221
18222 let eight_b_small = row(&at_8, "mlx/qwen3-8b:4bit");
18223 let eight_b_large = row(&at_32, "mlx/qwen3-8b:4bit");
18224 assert_eq!(eight_b_small.fit, ModelFitStatus::TooBig);
18225 assert_eq!(eight_b_large.fit, ModelFitStatus::Fits);
18226 assert!(eight_b_small.platform_compatible && eight_b_large.platform_compatible);
18227 assert_eq!(
18228 eight_b_small.estimated_peak_mb, eight_b_large.estimated_peak_mb,
18229 "the estimate is the model's; only the budget differs"
18230 );
18231 assert!(eight_b_small.estimated_peak_mb.is_some_and(|mb| mb > 4_800));
18232 assert_eq!(eight_b_small.family.as_deref(), Some("qwen3"));
18233 assert!(eight_b_small.version.is_some());
18234
18235 let retired = row(&at_8, "test/retired-4b:4bit");
18236 assert!(retired.deprecated, "deprecated rows stay listed, flagged");
18237 assert!(!eight_b_small.deprecated);
18238
18239 let mut saw_remote = false;
18240 for remote in at_8.iter().filter(|row| !row.is_local) {
18241 saw_remote = true;
18242 assert_eq!(remote.fit, ModelFitStatus::Fits, "{}", remote.id);
18243 assert!(remote.platform_compatible, "{}", remote.id);
18244 assert_eq!(remote.estimated_peak_mb, None, "{}", remote.id);
18245 assert_eq!(
18246 remote.family, None,
18247 "{}: no upstream identifier here",
18248 remote.id
18249 );
18250 assert_eq!(remote.version, None, "{}", remote.id);
18251 }
18252 assert!(saw_remote);
18253 assert_eq!(
18254 car_secrets::secret_store_activity(),
18255 secret_activity_before,
18256 "catalog snapshots must use the injected credential without consulting host secrets"
18257 );
18258
18259 for (small, large) in at_8.iter().zip(&at_32) {
18262 let strip = |row: &ModelInfo| {
18263 let mut value = serde_json::to_value(row).unwrap();
18264 let object = value.as_object_mut().unwrap();
18265 for key in ["fit", "estimated_peak_mb", "platform_compatible"] {
18266 object.remove(key);
18267 }
18268 value
18269 };
18270 assert_eq!(strip(small), strip(large), "{}", small.id);
18271 }
18272 }
18273
18274 #[tokio::test]
18275 async fn explicit_model_blocked_by_zero_budget_does_not_substitute_or_download() {
18276 let root = tempfile::tempdir().unwrap();
18277 let models_dir = root.path().join("weights");
18278 let config = test_config(models_dir.clone());
18279 let repository =
18280 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
18281 crate::resource_policy::ResourcePolicyRepository::save(
18282 &repository,
18283 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
18284 )
18285 .unwrap();
18286 let engine = InferenceEngine::new(config);
18287
18288 let error = engine
18289 .generate_tracked(GenerateRequest {
18290 prompt: "hello".into(),
18291 model: Some("mlx/qwen3-4b:4bit".into()),
18292 ..Default::default()
18293 })
18294 .await
18295 .unwrap_err();
18296
18297 assert!(matches!(
18298 error,
18299 InferenceError::LocalResourceBlocked {
18300 preflight: crate::resource_policy::LocalLoadPreflight {
18301 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
18302 ..
18303 },
18304 ..
18305 }
18306 ));
18307 assert!(
18308 !models_dir.exists() || std::fs::read_dir(models_dir).unwrap().next().is_none(),
18309 "admission must happen before download/load"
18310 );
18311 }
18312
18313 #[tokio::test]
18314 async fn external_vllm_mlx_bypasses_local_admission_while_managed_artifact_does_not() {
18315 let schema = crate::vllm_mlx::to_model_schema(
18316 &crate::vllm_mlx::DiscoveredModel {
18317 id: "mlx-community/Qwen3-4B-4bit".into(),
18318 owned_by: None,
18319 },
18320 "http://localhost:8000",
18321 );
18322 assert!(!InferenceEngine::requires_local_admission(&schema));
18323
18324 let root = tempfile::tempdir().unwrap();
18325 let config = test_config(root.path().join("weights"));
18326 let repository =
18327 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
18328 crate::resource_policy::ResourcePolicyRepository::save(
18329 &repository,
18330 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
18331 )
18332 .unwrap();
18333 let engine = InferenceEngine::new(config);
18334 let (external, reservation) = engine
18335 .vllm_live_schema(schema.clone(), None, 0)
18336 .await
18337 .unwrap();
18338 assert!(matches!(
18339 external.source,
18340 ModelSource::VllmMlx { ref endpoint, .. } if endpoint == "http://localhost:8000"
18341 ));
18342 assert!(reservation.is_none());
18343
18344 let mut managed = schema;
18345 managed.source = ModelSource::ManagedVllmMlx {
18346 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
18347 hf_weight_file: None,
18348 };
18349 assert!(InferenceEngine::requires_local_admission(&managed));
18350 let error = engine
18351 .reserve_local_request(&managed, 0)
18352 .expect_err("zero-GB policy must reject before vllm runtime/download/spawn");
18353 assert!(matches!(
18354 error,
18355 InferenceError::LocalResourceBlocked {
18356 preflight: crate::resource_policy::LocalLoadPreflight {
18357 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
18358 ..
18359 },
18360 ..
18361 }
18362 ));
18363 }
18364
18365 #[cfg(unix)]
18366 async fn assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(
18367 streaming: bool,
18368 cancel_first: bool,
18369 ) {
18370 struct FixedProbe;
18371 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
18372 fn available_memory_mb(
18373 &self,
18374 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
18375 Ok(Some(24_000))
18376 }
18377 }
18378
18379 let Some(python) = crate::vllm_runtime::test_python_interpreter() else {
18380 panic!("a real Python interpreter is required for the managed-vllm dispatch fixture");
18381 };
18382 let root = tempfile::tempdir().unwrap();
18383 let script = root.path().join("fake-vllm-mlx");
18384 let spawned = root.path().join("spawned");
18385 let release = root.path().join("release");
18386 std::fs::write(
18387 &script,
18388 format!(
18389 "#!{}\n\
18390 import http.server, os, sys, time\n\
18391 import socket as _s; _s.getfqdn = lambda n='': n\n\
18392 marker = {:?}\n\
18393 release = {:?}\n\
18394 open(marker, 'w').close()\n\
18395 while not os.path.exists(release): time.sleep(0.01)\n\
18396 port = int(sys.argv[sys.argv.index('--port') + 1])\n\
18397 class H(http.server.BaseHTTPRequestHandler):\n\
18398 \x20 def do_GET(self):\n\
18399 \x20 self.send_response(200); self.end_headers(); self.wfile.write(b'ok')\n\
18400 \x20 def do_POST(self):\n\
18401 \x20 length = int(self.headers.get('content-length', '0'))\n\
18402 \x20 request = self.rfile.read(length).replace(b' ', b'')\n\
18403 \x20 if b'\"stream\":true' in request:\n\
18404 \x20 body = b'data: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"round13-stream\"}},\"finish_reason\":null}}]}}\\n\\ndata: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"stop\"}}]}}\\n\\ndata: [DONE]\\n\\n'\n\
18405 \x20 content_type = 'text/event-stream'\n\
18406 \x20 else:\n\
18407 \x20 body = b'{{\"id\":\"fixture\",\"object\":\"chat.completion\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"message\":{{\"role\":\"assistant\",\"content\":\"round13-ok\"}},\"finish_reason\":\"stop\"}}],\"usage\":{{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}}}'\n\
18408 \x20 content_type = 'application/json'\n\
18409 \x20 self.send_response(200); self.send_header('content-type', content_type); self.send_header('content-length', str(len(body))); self.end_headers(); self.wfile.write(body)\n\
18410 \x20 def log_message(self, *args): pass\n\
18411 http.server.HTTPServer(('127.0.0.1', port), H).serve_forever()\n",
18412 python.display(),
18413 spawned.display().to_string(),
18414 release.display().to_string(),
18415 ),
18416 )
18417 .unwrap();
18418 use std::os::unix::fs::PermissionsExt;
18419 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
18420
18421 let models_dir = root.path().join("models");
18422 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
18423 let mut schema = crate::registry::builtin_catalog()
18424 .into_iter()
18425 .find(ModelSchema::is_car_managed_vllm_mlx)
18426 .expect("managed vllm fixture schema");
18427 schema.id = "vllm-mlx/round12-singleflight".into();
18428 schema.name = "round12-singleflight".into();
18429 schema.cost.size_mb = Some(1);
18430 schema.cost.ram_mb = Some(1);
18431 schema.source = ModelSource::ManagedVllmMlx {
18432 hf_repo: "fixture/round12-singleflight".into(),
18433 hf_weight_file: None,
18434 };
18435 let model_dir = models_dir.join(&schema.name);
18436 std::fs::create_dir_all(&model_dir).unwrap();
18437 std::fs::write(model_dir.join("config.json"), b"{}").unwrap();
18438 std::fs::write(model_dir.join("model.safetensors"), b"fixture").unwrap();
18439 engine
18440 .unified_registry_mut()
18441 .register_project_model(schema.clone());
18442
18443 let coordinator = Arc::new(
18444 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
18445 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
18450 crate::hardware::HardwareInfo::detect(),
18451 Arc::new(FixedProbe),
18452 ),
18453 );
18454 engine.local_admission = coordinator.clone();
18455 engine.vllm_pool = Arc::new(crate::vllm_pool::VllmServerPool::with_test_runtime(
18456 std::time::Duration::from_secs(300),
18457 coordinator,
18458 script,
18459 ));
18460 let engine = Arc::new(engine);
18461 let model_id = schema.id.clone();
18462 let mut request = GenerateRequest {
18463 prompt: "hello from round 13".into(),
18464 model: Some(model_id.clone()),
18465 ..Default::default()
18466 };
18467 request.params.strict_model = true;
18472 let (estimated_input, _, _) = engine.estimated_tokens(&request, Some(&model_id));
18473 let expected_footprint = estimated_input.saturating_add(request.params.max_tokens);
18474 let expected_estimate = engine
18475 .local_model_preflight(&model_id, expected_footprint)
18476 .unwrap()
18477 .estimate;
18478 let expected_request_overhead_mb = expected_estimate
18479 .runtime_overhead_mb
18480 .saturating_add(expected_estimate.context_overhead_mb)
18481 .saturating_add(expected_estimate.transient_margin_mb);
18482
18483 async fn execute(
18484 engine: Arc<InferenceEngine>,
18485 request: GenerateRequest,
18486 streaming: bool,
18487 direct_vllm: bool,
18488 ) -> Result<String, InferenceError> {
18489 if direct_vllm {
18490 let model_id = request.model.as_deref().expect("explicit fixture model");
18491 let schema = engine
18492 .unified_registry()
18493 .get(model_id)
18494 .cloned()
18495 .expect("fixture schema");
18496 let (schema, _) = engine.vllm_live_schema(schema, None, 0).await?;
18497 return match schema.source {
18498 ModelSource::VllmMlx { endpoint, .. } => Ok(endpoint),
18499 source => Err(InferenceError::InferenceFailed(format!(
18500 "fixture did not resolve to a vllm endpoint: {source:?}"
18501 ))),
18502 };
18503 }
18504 if !streaming {
18505 return engine
18506 .generate_tracked(request)
18507 .await
18508 .map(|result| result.text);
18509 }
18510 let mut tracked = engine.generate_tracked_stream(request).await?;
18511 let mut accumulator = crate::stream::StreamAccumulator::default();
18512 while let Some(event) = tracked.events.recv().await {
18513 let done = matches!(event, crate::stream::StreamEvent::Done { .. });
18514 accumulator.push(&event);
18515 if done {
18516 break;
18517 }
18518 }
18519 Ok(accumulator.finish().0)
18520 }
18521
18522 let first_engine = engine.clone();
18523 let first_request = request.clone();
18524 let mut first = tokio::spawn(async move {
18525 execute(first_engine, first_request, streaming, cancel_first).await
18526 });
18527 tokio::time::timeout(std::time::Duration::from_secs(5), async {
18528 while !spawned.exists() {
18529 if first.is_finished() {
18530 let result = (&mut first).await;
18531 panic!("first dispatch ended before spawn: {result:?}");
18532 }
18533 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
18534 }
18535 })
18536 .await
18537 .expect("first request must spawn before publication");
18538 if !cancel_first {
18539 let during_startup = engine.local_model_preflight(&model_id, 0).unwrap();
18540 assert_eq!(
18541 during_startup.active_reservations_mb, expected_request_overhead_mb,
18542 "the cold-to-pending transfer must retain all request context/KV overhead"
18543 );
18544 assert_eq!(
18545 during_startup.resident_model_mb, expected_estimate.weights_mb,
18546 "the pending allocation must replace the exact cold-weight charge"
18547 );
18548 assert_eq!(
18549 during_startup
18550 .active_reservations_mb
18551 .saturating_add(during_startup.resident_model_mb),
18552 expected_estimate.estimated_peak_mb,
18553 "active request overhead plus pending weights must preserve the full admitted footprint"
18554 );
18555 }
18556
18557 let second_engine = engine.clone();
18558 let mut second =
18559 tokio::spawn(
18560 async move { execute(second_engine, request, streaming, cancel_first).await },
18561 );
18562 assert!(
18563 tokio::time::timeout(std::time::Duration::from_millis(100), &mut second)
18564 .await
18565 .is_err(),
18566 "request 2 must wait for request 1 to publish instead of failing on startup state"
18567 );
18568
18569 let first_result = if cancel_first {
18570 std::fs::remove_file(&spawned).unwrap();
18571 first.abort();
18572 let _ = first.await;
18573 tokio::time::timeout(std::time::Duration::from_secs(5), async {
18574 while !spawned.exists() {
18575 if second.is_finished() {
18576 let result = (&mut second).await;
18577 panic!("waiter ended before replacement spawn: {result:?}");
18578 }
18579 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
18580 }
18581 })
18582 .await
18583 .expect("waiter must continue after cancelled owner teardown");
18584 None
18585 } else {
18586 std::fs::write(&release, b"release").unwrap();
18587 Some(
18588 tokio::time::timeout(std::time::Duration::from_secs(5), first)
18589 .await
18590 .expect("first dispatch completes")
18591 .unwrap()
18592 .unwrap(),
18593 )
18594 };
18595 std::fs::write(&release, b"release").unwrap();
18596 let second_result = tokio::time::timeout(std::time::Duration::from_secs(5), second)
18597 .await
18598 .expect("second dispatch completes")
18599 .unwrap()
18600 .unwrap();
18601 let expected = if cancel_first {
18602 None
18603 } else if streaming {
18604 Some("round13-stream")
18605 } else {
18606 Some("round13-ok")
18607 };
18608 if let (Some(first_result), Some(expected)) = (first_result, expected) {
18609 assert_eq!(first_result, expected);
18610 }
18611 if let Some(expected) = expected {
18612 assert_eq!(second_result, expected);
18613 } else {
18614 assert!(second_result.starts_with("http://127.0.0.1:"));
18615 }
18616 assert_eq!(engine.vllm_pool.len().await, 1);
18617 assert!(engine
18618 .vllm_pool
18619 .release_model_if_present(&model_id)
18620 .await
18621 .unwrap());
18622 }
18623
18624 #[cfg(unix)]
18625 #[tokio::test]
18626 async fn concurrent_managed_vllm_generate_waits_before_outer_reservation() {
18627 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, false).await;
18628 }
18629
18630 #[cfg(unix)]
18631 #[tokio::test]
18632 async fn concurrent_managed_vllm_stream_waits_before_outer_reservation() {
18633 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(true, false).await;
18634 }
18635
18636 #[cfg(unix)]
18637 #[tokio::test]
18638 async fn managed_vllm_waiter_continues_after_startup_owner_cancellation() {
18639 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, true).await;
18640 }
18641
18642 #[tokio::test]
18643 async fn same_state_root_engines_share_runtime_components() {
18644 const CHILD_ENV: &str = "CAR_TWO_ENGINE_RUNTIME_TEST_CHILD";
18645 if std::env::var_os(CHILD_ENV).is_some() {
18646 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
18647 return;
18648 }
18649 let root = tempfile::tempdir().unwrap();
18650 let first = InferenceEngine::new(test_config(root.path().join("weights")));
18651 let second = InferenceEngine::new(test_config(root.path().join("weights")));
18652
18653 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
18654 assert!(Arc::ptr_eq(&first.vllm_pool, &second.vllm_pool));
18655 assert!(Arc::ptr_eq(
18656 &first.resource_policy_generation,
18657 &second.resource_policy_generation
18658 ));
18659 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
18660 {
18661 assert!(Arc::ptr_eq(&first.text_backends, &second.text_backends));
18662 assert!(Arc::ptr_eq(&first.local_backends, &second.local_backends));
18663 }
18664 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18665 {
18666 assert!(Arc::ptr_eq(&first.backend, &second.backend));
18667 assert!(Arc::ptr_eq(
18668 &first.embedding_backend,
18669 &second.embedding_backend
18670 ));
18671 }
18672
18673 let loads = Arc::new(std::sync::atomic::AtomicU64::new(0));
18674 let first_cache = first._runtime_scope.load_probe.clone();
18675 let second_cache = second._runtime_scope.load_probe.clone();
18676 let mut threads = Vec::new();
18677 for cache in [first_cache, second_cache] {
18678 let loads = loads.clone();
18679 threads.push(std::thread::spawn(move || {
18680 cache
18681 .get_or_load::<()>("same/model", 1, || {
18682 loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
18683 std::thread::sleep(std::time::Duration::from_millis(20));
18684 Ok(())
18685 })
18686 .unwrap()
18687 }));
18688 }
18689 let handles = threads
18690 .into_iter()
18691 .map(|thread| thread.join().unwrap())
18692 .collect::<Vec<_>>();
18693 assert_eq!(loads.load(std::sync::atomic::Ordering::SeqCst), 1);
18694 assert!(Arc::ptr_eq(&handles[0], &handles[1]));
18695
18696 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
18697 .arg("--exact")
18698 .arg("tests::same_state_root_engines_share_runtime_components")
18699 .env(CHILD_ENV, "1")
18700 .stdin(std::process::Stdio::null())
18701 .stdout(std::process::Stdio::null())
18702 .stderr(std::process::Stdio::null())
18703 .kill_on_drop(true)
18704 .spawn()
18705 .unwrap();
18706 first
18707 .vllm_pool
18708 .insert_test_process("vllm-mlx/two-engine", child)
18709 .await;
18710 first.local_admission.mark_resident_allocation(
18711 "vllm-mlx/two-engine",
18712 &resource_policy::vllm_process_allocation_id("vllm-mlx/two-engine"),
18713 1,
18714 );
18715 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
18716 drop(first);
18717 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
18718 assert!(second.local_admission.is_resident("vllm-mlx/two-engine"));
18719 assert!(second.vllm_pool.evict_model("vllm-mlx/two-engine").await);
18720 assert!(!second.local_admission.is_resident("vllm-mlx/two-engine"));
18721 }
18722
18723 #[tokio::test]
18724 async fn last_engine_drop_reaps_vllm_before_new_runtime_admission() {
18725 const CHILD_ENV: &str = "CAR_LAST_ENGINE_VLLM_DROP_TEST_CHILD";
18726 if std::env::var_os(CHILD_ENV).is_some() {
18727 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
18728 return;
18729 }
18730 let root = tempfile::tempdir().unwrap();
18731 let config = test_config(root.path().join("weights"));
18732 let engine = InferenceEngine::new(config.clone());
18733 let coordinator = engine.local_admission.clone();
18734 let schema = engine
18735 .unified_registry()
18736 .all()
18737 .find(|schema| schema.is_vllm_mlx())
18738 .cloned()
18739 .expect("supervised vllm schema");
18740 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
18741 .arg("--exact")
18742 .arg("tests::last_engine_drop_reaps_vllm_before_new_runtime_admission")
18743 .env(CHILD_ENV, "1")
18744 .stdin(std::process::Stdio::null())
18745 .stdout(std::process::Stdio::null())
18746 .stderr(std::process::Stdio::null())
18747 .kill_on_drop(true)
18748 .spawn()
18749 .unwrap();
18750 engine
18751 .vllm_pool
18752 .insert_test_process(&schema.id, child)
18753 .await;
18754 coordinator.mark_resident_allocation(
18755 &schema.id,
18756 &resource_policy::vllm_process_allocation_id(&schema.id),
18757 1,
18758 );
18759
18760 drop(engine);
18761
18762 tokio::time::timeout(std::time::Duration::from_secs(2), async {
18763 while coordinator.teardown_pending(&schema.id) {
18764 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
18765 }
18766 })
18767 .await
18768 .expect("last runtime drop must eventually confirm supervised child exit");
18769 assert!(!coordinator.is_resident(&schema.id));
18770 let next = InferenceEngine::new(config);
18771 assert!(Arc::ptr_eq(&coordinator, &next.local_admission));
18772 let preflight = next.local_model_preflight(&schema.id, 0).unwrap();
18773 assert_eq!(preflight.resident_model_mb, 0);
18774 assert!(preflight.estimated_incremental_mb >= preflight.estimate.weights_mb);
18775 }
18776
18777 #[cfg(unix)]
18778 #[test]
18779 fn symlinked_state_roots_share_exact_coordinator_and_runtime() {
18780 use std::os::unix::fs::symlink;
18781
18782 let fixture = tempfile::tempdir().unwrap();
18783 let real = fixture.path().join("real-state");
18784 std::fs::create_dir(&real).unwrap();
18785 let alias = fixture.path().join("state-alias");
18786 symlink(&real, &alias).unwrap();
18787
18788 let first = InferenceEngine::new(test_config(real.join("weights")));
18789 let second = InferenceEngine::new(test_config(alias.join("weights")));
18790 assert!(Arc::ptr_eq(&first.local_admission, &second.local_admission));
18791 assert!(Arc::ptr_eq(&first._runtime_scope, &second._runtime_scope));
18792 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
18793 }
18794
18795 #[tokio::test]
18796 async fn stream_reservation_lives_until_returned_receiver_is_released() {
18797 let root = tempfile::tempdir().unwrap();
18798 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18799 let schema = crate::registry::builtin_catalog()
18800 .into_iter()
18801 .find(|schema| schema.is_local() && !schema.is_vllm_mlx())
18802 .expect("local model schema");
18803 engine.local_admission.mark_resident(&schema.id, 1);
18804 let reservation = engine.reserve_local_request(&schema, 64).unwrap();
18805 assert_eq!(engine.local_admission.active_request_count(&schema.id), 1);
18806 let (source_tx, source_rx) = tokio::sync::mpsc::channel(1);
18807 let returned =
18808 InferenceEngine::hold_optional_reservation_for_stream(source_rx, Some(reservation));
18809
18810 drop(returned);
18811 source_tx
18812 .send(stream::StreamEvent::TextDelta("release".into()))
18813 .await
18814 .unwrap();
18815 for _ in 0..20 {
18816 if engine.local_admission.active_request_count(&schema.id) == 0 {
18817 break;
18818 }
18819 tokio::task::yield_now().await;
18820 }
18821 assert_eq!(engine.local_admission.active_request_count(&schema.id), 0);
18822
18823 let compact_source = include_str!("lib.rs")
18824 .split_whitespace()
18825 .collect::<String>();
18826 let remote_handoff = [
18827 "Self::hold_optional_reservation_for_stream",
18828 "(receiver,",
18829 "candidate_reservation.take(),",
18830 ");",
18831 ]
18832 .concat();
18833 assert!(compact_source.contains(&remote_handoff));
18834 }
18835
18836 #[tokio::test]
18837 async fn residual_voice_allocation_blocks_removal_with_typed_error() {
18838 let root = tempfile::tempdir().unwrap();
18839 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18840 engine
18841 .local_admission
18842 .mark_resident("voice/removal-fixture", 1);
18843
18844 let error = match engine
18845 .prepare_local_model_removal("voice/removal-fixture")
18846 .await
18847 {
18848 Err(error) => error,
18849 Ok(_) => panic!("voice allocation must be released by its owner first"),
18850 };
18851 assert!(matches!(
18852 error,
18853 crate::resource_policy::ModelMaintenanceError::ResidualResidency {
18854 model_id,
18855 allocation_ids,
18856 } if model_id == "voice/removal-fixture"
18857 && allocation_ids == vec!["voice/removal-fixture"]
18858 ));
18859 }
18860
18861 #[tokio::test]
18862 async fn catalog_voice_id_blocks_removal_of_live_provider_alias() {
18863 let root = tempfile::tempdir().unwrap();
18864 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18865 engine.local_admission.register_model_aliases(
18866 "voice/parakeet-tdt-0.6b",
18867 ["mlx/parakeet-tdt-0.6b-v3:default"],
18868 );
18869 let mut reservation = engine
18870 .local_admission
18871 .reserve_measured_host_allocation(
18872 "voice/parakeet-tdt-0.6b",
18873 "voice/parakeet-tdt-0.6b#voice-allocation-0",
18874 1024 * 1024,
18875 0,
18876 )
18877 .unwrap();
18878 reservation.publish_resident_weights(1024 * 1024);
18879 drop(reservation);
18880
18881 let result = engine
18882 .prepare_local_model_removal("mlx/parakeet-tdt-0.6b-v3:default")
18883 .await;
18884 assert!(matches!(
18885 result,
18886 Err(crate::resource_policy::ModelMaintenanceError::ResidualResidency { .. })
18887 ));
18888 }
18889
18890 struct RefusingRemovalOffload;
18891
18892 #[async_trait::async_trait]
18893 impl crate::offload::LocalGenerationOffload for RefusingRemovalOffload {
18894 async fn generate(
18895 &self,
18896 _request: GenerateRequest,
18897 ) -> Result<InferenceResult, InferenceError> {
18898 unreachable!("removal test does not generate")
18899 }
18900
18901 async fn stream(
18902 &self,
18903 _request: GenerateRequest,
18904 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
18905 unreachable!("removal test does not stream")
18906 }
18907
18908 async fn resident_models(&self) -> Vec<String> {
18909 vec!["worker/refuses-release".into()]
18910 }
18911
18912 async fn release_model(&self, _model_id: &str) -> Result<bool, InferenceError> {
18913 Ok(false)
18914 }
18915 }
18916
18917 #[tokio::test]
18918 async fn worker_release_boolean_is_required_for_model_removal() {
18919 let _offload_guard = crate::offload::test_offload_lock().lock().await;
18920 crate::offload::set_local_offload(Some(Arc::new(RefusingRemovalOffload)));
18921 let root = tempfile::tempdir().unwrap();
18922 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18923 let result = engine
18924 .prepare_local_model_removal("worker/refuses-release")
18925 .await;
18926 crate::offload::set_local_offload(None);
18927
18928 assert!(matches!(
18929 result,
18930 Err(crate::resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
18931 model_id
18932 )) if model_id == "worker/refuses-release"
18933 ));
18934 }
18935
18936 #[test]
18937 fn transient_mlx_vlm_allocation_never_becomes_resident() {
18938 struct FixedProbe;
18939 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
18940 fn available_memory_mb(
18941 &self,
18942 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
18943 Ok(Some(24_000))
18944 }
18945 }
18946
18947 let root = tempfile::tempdir().unwrap();
18948 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
18949 engine.local_admission = Arc::new(
18950 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
18951 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
18952 crate::hardware::HardwareInfo {
18953 total_ram_mb: 32 * 1024,
18954 ..crate::hardware::HardwareInfo::detect()
18955 },
18956 Arc::new(FixedProbe),
18957 ),
18958 );
18959 let mut schema = crate::registry::builtin_catalog()
18960 .into_iter()
18961 .find(|schema| schema.tags.iter().any(|tag| tag == "mlx-vlm-cli"))
18962 .expect("one-shot mlx-vlm schema");
18963 schema.param_count = "1M".into();
18964 schema.quantization = Some(Quantization::parse("Q4"));
18965 let mut reservation = engine.reserve_local_request(&schema, 64).unwrap();
18966 InferenceEngine::reconcile_transient_local_allocation(&mut reservation, 1024 * 1024)
18967 .unwrap();
18968 drop(reservation);
18969 assert!(!engine.local_admission.is_resident(&schema.id));
18970 }
18971
18972 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18973 #[test]
18974 fn non_apple_backend_map_keeps_two_local_model_identities_exact() {
18975 let mut backends = std::collections::HashMap::<String, u32>::new();
18976 backends.insert("local/model-a".into(), 1);
18977 backends.insert("local/model-b".into(), 2);
18978 assert_eq!(backends.get("local/model-a"), Some(&1));
18979 assert_eq!(backends.get("local/model-b"), Some(&2));
18980 assert_eq!(backends.remove("local/model-a"), Some(1));
18981 assert!(!backends.contains_key("local/model-a"));
18982 assert_eq!(backends.get("local/model-b"), Some(&2));
18983 }
18984
18985 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18986 #[tokio::test]
18987 async fn non_apple_remote_classification_at_zero_gb_routes_without_local_allocation() {
18988 use wiremock::matchers::{method, path};
18989 use wiremock::{Mock, MockServer, ResponseTemplate};
18990
18991 let _credential_scope = crate::openrouter::test_credential_scope();
18992 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
18993 let server = MockServer::start().await;
18994 Mock::given(method("POST"))
18995 .and(path("/v1/chat/completions"))
18996 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
18997 "choices": [{
18998 "message": {"role": "assistant", "content": "positive"},
18999 "finish_reason": "stop"
19000 }],
19001 "usage": {"prompt_tokens": 16, "completion_tokens": 1}
19002 })))
19003 .mount(&server)
19004 .await;
19005
19006 let fixture = tempfile::tempdir().unwrap();
19007 let config = test_config(fixture.path().join("weights"));
19008 let repository =
19009 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19010 crate::resource_policy::ResourcePolicyRepository::save(
19011 &repository,
19012 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19013 )
19014 .unwrap();
19015 let mut engine = InferenceEngine::new(config);
19016 let mut remote = crate::openrouter::curated_schemas()
19017 .into_iter()
19018 .next()
19019 .expect("OpenRouter schema");
19020 remote.id = "openrouter/test/classifier".into();
19021 remote.name = remote.id.clone();
19022 if let ModelSource::RemoteApi { endpoint, .. } = &mut remote.source {
19023 *endpoint = server.uri();
19024 }
19025 let remote_id = remote.id.clone();
19026 engine.unified_registry_mut().register_project_model(remote);
19027
19028 let result = engine
19029 .classify(ClassifyRequest {
19030 work_context: None,
19031 text: "a good outcome".into(),
19032 labels: vec!["positive".into(), "negative".into()],
19033 model: Some(remote_id),
19034 })
19035 .await
19036 .unwrap();
19037 assert_eq!(
19038 result.first().map(|item| item.label.as_str()),
19039 Some("positive")
19040 );
19041 assert_eq!(engine.local_admission.resident_model_mb(), 0);
19042 assert!(
19043 !engine.config.models_dir.exists()
19044 || std::fs::read_dir(&engine.config.models_dir)
19045 .unwrap()
19046 .next()
19047 .is_none(),
19048 "remote classification must not download local weights"
19049 );
19050 }
19051
19052 #[test]
19053 fn adaptive_speech_skips_blocked_local_but_explicit_speech_fails() {
19054 let fixture = TempDir::new().unwrap();
19055 let config = InferenceConfig {
19056 models_dir: fixture.path().join("weights"),
19057 state_root: fixture.path().join("state"),
19058 ..Default::default()
19059 };
19060 let repository =
19061 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19062 crate::resource_policy::ResourcePolicyRepository::save(
19063 &repository,
19064 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19065 )
19066 .unwrap();
19067 let engine = InferenceEngine::new(config);
19068 let local = engine
19069 .unified_registry()
19070 .all()
19071 .find(|schema| {
19072 schema.is_local() && schema.has_capability(ModelCapability::SpeechToText)
19073 })
19074 .cloned()
19075 .expect("built-in local STT model");
19076
19077 assert!(matches!(
19078 engine.admit_speech_candidate(&local, false),
19079 SpeechCandidateAdmission::SkipBlocked(_)
19080 ));
19081 assert!(matches!(
19082 engine.admit_speech_candidate(&local, true),
19083 SpeechCandidateAdmission::FailBlocked(InferenceError::LocalResourceBlocked { .. })
19084 ));
19085
19086 let mut os_owned = local;
19087 os_owned.id = "windows/speech-synthesis:test".into();
19088 os_owned.source = ModelSource::WindowsSpeech {};
19089 assert!(matches!(
19090 engine.admit_speech_candidate(&os_owned, true),
19091 SpeechCandidateAdmission::Proceed(None)
19092 ));
19093 }
19094
19095 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19096 async fn cancelled_native_blocking_work_keeps_its_exact_charge_until_completion() {
19097 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
19098 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
19099 crate::hardware::HardwareInfo {
19100 os: "test".into(),
19101 arch: "test".into(),
19102 cpu_cores: 8,
19103 total_ram_mb: 32 * 1024,
19104 gpu_backend: crate::hardware::GpuBackend::Cpu,
19105 gpu_memory_mb: None,
19106 gpu_devices: Vec::new(),
19107 recommended_model: "fixture".into(),
19108 recommended_context: 4096,
19109 max_model_mb: 32 * 1024,
19110 },
19111 ));
19112 let reservation = coordinator
19113 .reserve_measured_host("detached-native-a", 512 * 1024 * 1024, 0)
19114 .unwrap();
19115 let lease = reservation.detached_lease();
19116 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
19117 let release =
19118 std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
19119 let release_worker = release.clone();
19120 let operation = tokio::spawn(async move {
19121 let _request_reservation = reservation;
19122 run_admitted_blocking(lease, move || {
19123 let _ = started_tx.send(());
19124 let (lock, ready) = &*release_worker;
19125 let mut released = lock
19126 .lock()
19127 .unwrap_or_else(std::sync::PoisonError::into_inner);
19128 while !*released {
19129 released = ready
19130 .wait(released)
19131 .unwrap_or_else(std::sync::PoisonError::into_inner);
19132 }
19133 })
19134 .await
19135 });
19136 started_rx.await.unwrap();
19137 operation.abort();
19138 let _ = operation.await;
19139
19140 let blocked = coordinator.reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0);
19141 assert!(
19142 blocked.is_err(),
19143 "cancelling the await must not advertise memory still owned by spawn_blocking"
19144 );
19145
19146 let (lock, ready) = &*release;
19147 *lock
19148 .lock()
19149 .unwrap_or_else(std::sync::PoisonError::into_inner) = true;
19150 ready.notify_one();
19151 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19152 loop {
19153 if coordinator
19154 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
19155 .is_ok()
19156 {
19157 break;
19158 }
19159 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19160 }
19161 })
19162 .await
19163 .expect("the detached charge must clear after native work really exits");
19164 }
19165
19166 #[test]
19167 fn nonstream_native_text_generation_uses_cancellable_blocking_boundary() {
19168 let source = include_str!("lib.rs");
19169 let mlx = source
19170 .split("async fn generate_mlx(")
19171 .nth(1)
19172 .and_then(|tail| tail.split("async fn generate_local(").next())
19173 .expect("generate_mlx source");
19174 let local = source
19175 .split("async fn generate_local(")
19176 .nth(1)
19177 .and_then(|tail| tail.split("fn apply_top_k_top_p").next())
19178 .expect("generate_local source");
19179
19180 assert!(
19181 mlx.contains("run_admitted_blocking"),
19182 "native MLX decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
19183 );
19184 assert!(
19185 local.contains("run_admitted_blocking"),
19186 "polymorphic local decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
19187 );
19188 }
19189
19190 #[cfg(unix)]
19191 #[tokio::test]
19192 async fn cancelled_speech_subprocess_is_killed_and_reaped_before_charge_clears() {
19193 use std::os::unix::fs::PermissionsExt;
19194
19195 let fixture = TempDir::new().unwrap();
19196 let runtime = SpeechRuntime::new(fixture.path().join("speech-runtime"));
19197 std::fs::create_dir_all(runtime.stt_program.parent().unwrap()).unwrap();
19198 std::fs::write(
19199 &runtime.stt_program,
19200 b"#!/bin/sh\necho started > \"$1\"\nsleep 1\necho continued > \"$2\"\n",
19201 )
19202 .unwrap();
19203 let mut permissions = std::fs::metadata(&runtime.stt_program)
19204 .unwrap()
19205 .permissions();
19206 permissions.set_mode(0o755);
19207 std::fs::set_permissions(&runtime.stt_program, permissions).unwrap();
19208
19209 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
19210 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
19211 crate::hardware::HardwareInfo {
19212 total_ram_mb: 32 * 1024,
19213 ..crate::hardware::HardwareInfo::detect()
19214 },
19215 ));
19216 let reservation = coordinator
19217 .reserve_measured_host("mlx-audio-a", 512 * 1024 * 1024, 0)
19218 .unwrap();
19219 let lease = reservation.detached_lease();
19220 let started = fixture.path().join("started");
19221 let continued = fixture.path().join("continued");
19222 let args = vec![
19223 started.display().to_string(),
19224 continued.display().to_string(),
19225 ];
19226 let command = tokio::spawn(async move {
19227 let _request_reservation = reservation;
19228 run_mlx_audio_command(&runtime, "stt.generate", &args, lease).await
19229 });
19230 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19231 while !started.exists() {
19232 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19233 }
19234 })
19235 .await
19236 .expect("speech fixture child must start");
19237 command.abort();
19238 let _ = command.await;
19239
19240 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19241 loop {
19242 if coordinator
19243 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
19244 .is_ok()
19245 {
19246 break;
19247 }
19248 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19249 }
19250 })
19251 .await
19252 .expect("charge clears only after the cancelled child is reaped");
19253 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
19254 assert!(
19255 !continued.exists(),
19256 "the cancelled local speech process must not keep allocating in the background"
19257 );
19258 }
19259
19260 #[test]
19261 fn local_model_eviction_surface_includes_every_in_process_cache() {
19262 let source = include_str!("lib.rs");
19263 for cache in [
19264 "self.text_backends.evict_if_idle(model_id)",
19265 "self.local_backends.evict_if_idle(model_id)",
19266 "self.flux_cache.evict_if_idle(model_id)",
19267 "self.ltx_cache.evict_if_idle(model_id)",
19268 "self.kokoro_cache.evict_if_idle(model_id)",
19269 "self.local_backends.evict_idle()",
19270 ] {
19271 assert!(source.contains(cache), "missing eviction seam: {cache}");
19272 }
19273 }
19274
19275 #[tokio::test]
19281 async fn catalog_refresh_uses_authority_hints_without_secret_reads() {
19282 if !crate::run_in_isolated_test_process(
19283 "tests::catalog_refresh_uses_authority_hints_without_secret_reads",
19284 "CAR_CATALOG_ZERO_SECRET_READ_CHILD",
19285 ) {
19286 return;
19287 }
19288 let _environment = ENV_MUTEX.lock().await;
19289 let fixture = TempDir::new().unwrap();
19290 let credential_envs = [
19291 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19292 car_auth::PARSLEE_API_BASE_KEY,
19293 crate::openrouter::API_KEY_ENV,
19294 "OPENAI_API_KEY",
19295 "ANTHROPIC_API_KEY",
19296 "GOOGLE_API_KEY",
19297 "ELEVENLABS_API_KEY",
19298 ];
19299 let mut restored_names = vec![car_home::ENV_VAR, "CAR_SECRETS_FILE_DIR"];
19300 restored_names.extend(credential_envs);
19301 let _restore = RestoredEnvironment::capture(&restored_names);
19302 unsafe {
19303 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
19304 std::env::set_var(
19305 "CAR_SECRETS_FILE_DIR",
19306 fixture.path().join("isolated-secrets"),
19307 );
19308 for name in credential_envs {
19309 std::env::remove_var(name);
19310 }
19311 }
19312
19313 let before = car_secrets::secret_store_activity();
19314 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
19315
19316 let listed = engine.list_models_unified();
19318 let schemas = engine.list_schemas();
19319 for provider in ["parslee", "openai", "anthropic", "google", "elevenlabs"] {
19320 assert!(
19321 schemas.iter().any(|schema| schema.provider == provider),
19322 "real builtin catalog lost the credential-bearing {provider} rows"
19323 );
19324 }
19325 assert!(
19326 schemas.iter().any(|schema| matches!(
19327 schema.source,
19328 ModelSource::RemoteApi {
19329 protocol: ApiProtocol::OpenRouter,
19330 ..
19331 }
19332 )),
19333 "real builtin catalog lost the reviewed OpenRouter rows"
19334 );
19335 assert_eq!(listed.len(), schemas.len());
19336
19337 let schema_refs: Vec<&ModelSchema> = schemas.iter().collect();
19339 let _setup_plan = crate::recommend(
19340 &schema_refs,
19341 &HardwareInfo::detect(),
19342 UseCase::default(),
19343 QualityTier::default(),
19344 Privacy::OnDevice,
19345 );
19346
19347 let _speech = engine.speech_health();
19349 let _concierge = engine.concierge_status(false).await;
19350 let _health = engine.model_health().await;
19351
19352 let after = car_secrets::secret_store_activity();
19353 assert_eq!(
19354 after, before,
19355 "passive builtin catalog surfaces performed secret-store operations"
19356 );
19357 }
19358
19359 #[tokio::test]
19360 async fn list_models_unified_and_model_health_are_zero_secret_store_probes() {
19361 if !crate::run_in_isolated_test_process(
19362 "tests::list_models_unified_and_model_health_are_zero_secret_store_probes",
19363 "CAR_DIRECT_MODEL_SURFACES_ZERO_SECRET_CHILD",
19364 ) {
19365 return;
19366 }
19367 let _environment = ENV_MUTEX.lock().await;
19368 let fixture = TempDir::new().unwrap();
19369 let _restore = RestoredEnvironment::capture(&[
19370 car_home::ENV_VAR,
19371 "CAR_SECRETS_FILE_DIR",
19372 crate::openrouter::API_KEY_ENV,
19373 ]);
19374 unsafe {
19375 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
19376 std::env::set_var(
19377 "CAR_SECRETS_FILE_DIR",
19378 fixture.path().join("isolated-secrets"),
19379 );
19380 std::env::remove_var(crate::openrouter::API_KEY_ENV);
19381 }
19382 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
19383 let before = car_secrets::secret_store_activity();
19384
19385 let rows = engine.list_models_unified();
19386 let health = engine.model_health().await;
19387
19388 assert!(!rows.is_empty(), "the unified catalog fixture must exist");
19389 assert!(health.total_models > 0, "the health fixture must exist");
19390 let after = car_secrets::secret_store_activity();
19391 assert_eq!(after.status_attempts, before.status_attempts);
19392 assert_eq!(after.get_attempts, before.get_attempts);
19393
19394 assert!(
19404 rows.iter().all(|row| row.credential_required.is_none()),
19405 "a passive catalog read must not populate `credential_required` — its \
19406 availability verdict cannot see the secret store, so any credential it \
19407 named might already be present: {:?}",
19408 rows.iter()
19409 .filter(|row| row.credential_required.is_some())
19410 .map(|row| (&row.id, &row.credential_required))
19411 .collect::<Vec<_>>()
19412 );
19413 }
19414
19415 #[test]
19425 fn car_home_moves_the_catalog_and_discovery_caches_but_never_the_weights() {
19426 let _environment = crate::openrouter::test_environment_scope();
19427 let prior = std::env::var_os(car_home::ENV_VAR);
19428
19429 unsafe { std::env::remove_var(car_home::ENV_VAR) };
19430 let shared = InferenceConfig::default();
19431 let default_catalog = crate::catalog::cache_path(&shared.state_root);
19432 let default_discovery = crate::discovery::cache_path(&shared.state_models_dir());
19433
19434 let alt = Path::new("/tmp/car-home-inference-cache-test");
19435 unsafe { std::env::set_var(car_home::ENV_VAR, alt) };
19436 let relocated = InferenceConfig::default();
19437 let catalog = crate::catalog::cache_path(&relocated.state_root);
19438 let discovery = crate::discovery::cache_path(&relocated.state_models_dir());
19439
19440 match prior {
19441 Some(value) => unsafe { std::env::set_var(car_home::ENV_VAR, value) },
19442 None => unsafe { std::env::remove_var(car_home::ENV_VAR) },
19443 }
19444
19445 assert_eq!(catalog, alt.join(crate::catalog::CATALOG_CACHE_FILE));
19446 assert_eq!(
19447 discovery,
19448 alt.join("models")
19449 .join(crate::discovery::DISCOVERED_MODELS_FILE)
19450 );
19451 assert_ne!(
19452 catalog, default_catalog,
19453 "the catalog cache must not resolve back into the shared root",
19454 );
19455 assert_ne!(
19456 discovery, default_discovery,
19457 "the discovery cache must not resolve back into the shared root",
19458 );
19459
19460 assert_eq!(
19461 relocated.models_dir, shared.models_dir,
19462 "the weights cache is machine-global and must not follow CAR_HOME",
19463 );
19464 assert!(
19465 !relocated.models_dir.starts_with(alt),
19466 "the weights cache must not be dragged under the override",
19467 );
19468 }
19469
19470 #[derive(Clone, Copy)]
19471 enum CacheRoutingSurface {
19472 Generate,
19473 Stream,
19474 }
19475
19476 async fn invoke_cache_routed_openrouter(
19482 surface: CacheRoutingSurface,
19483 cache_read_estimate: usize,
19484 cache_write_estimate: usize,
19485 ) -> String {
19486 use wiremock::matchers::{method, path};
19487 use wiremock::{Mock, MockServer, ResponseTemplate};
19488
19489 let _credential_scope = crate::openrouter::test_credential_scope();
19490 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
19491 let server = MockServer::start().await;
19492 let response = match surface {
19493 CacheRoutingSurface::Generate => ResponseTemplate::new(200).set_body_json(
19494 serde_json::json!({
19495 "choices": [{
19496 "message": {"role": "assistant", "content": "cache-route-ok"},
19497 "finish_reason": "stop"
19498 }],
19499 "usage": {"prompt_tokens": 40_000, "completion_tokens": 8}
19500 }),
19501 ),
19502 CacheRoutingSurface::Stream => ResponseTemplate::new(200).set_body_raw(
19503 concat!(
19504 "data: {\"choices\":[{\"delta\":{\"content\":\"cache-route-ok\"},\"finish_reason\":\"stop\"}]}\n\n",
19505 "data: [DONE]\n\n"
19506 ),
19507 "text/event-stream",
19508 ),
19509 };
19510 Mock::given(method("POST"))
19511 .and(path("/v1/chat/completions"))
19512 .respond_with(response)
19513 .mount(&server)
19514 .await;
19515
19516 let tmp = TempDir::new().unwrap();
19517 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19518 let mut uncached_cheap = crate::openrouter::curated_schemas()
19519 .into_iter()
19520 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
19521 .unwrap();
19522 uncached_cheap.id = "openrouter/test/uncached-cheap".into();
19523 uncached_cheap.name = uncached_cheap.id.clone();
19524 uncached_cheap.cost = CostModel {
19525 input_per_mtok: Some(1.0),
19526 output_per_mtok: Some(1.0),
19527 cache_read_input_per_mtok: Some(100.0),
19528 cache_write_input_per_mtok: Some(100.0),
19529 ..Default::default()
19530 };
19531 if let ModelSource::RemoteApi { endpoint, .. } = &mut uncached_cheap.source {
19532 *endpoint = server.uri();
19533 }
19534
19535 let mut cached_cheap = uncached_cheap.clone();
19536 cached_cheap.id = "openrouter/test/cached-cheap".into();
19537 cached_cheap.name = cached_cheap.id.clone();
19538 cached_cheap.cost = CostModel {
19539 input_per_mtok: Some(80.0),
19540 output_per_mtok: Some(1.0),
19541 cache_read_input_per_mtok: Some(0.001),
19542 cache_write_input_per_mtok: Some(0.001),
19543 ..Default::default()
19544 };
19545
19546 let uncached_id = uncached_cheap.id.clone();
19547 let cached_id = cached_cheap.id.clone();
19548 engine
19549 .unified_registry_mut()
19550 .register_project_model(uncached_cheap);
19551 engine
19552 .unified_registry_mut()
19553 .register_project_model(cached_cheap);
19554 let exclude_models = engine
19555 .list_schemas()
19556 .into_iter()
19557 .map(|schema| schema.id)
19558 .filter(|id| id != &uncached_id && id != &cached_id)
19559 .collect();
19560
19561 let mut params = GenerateParams {
19565 max_tokens: 8,
19566 ..Default::default()
19567 };
19568 assert_eq!(params.estimated_cache_read_input_tokens, 0);
19569 assert_eq!(params.estimated_cache_write_input_tokens, 0);
19570 params.estimated_cache_read_input_tokens = cache_read_estimate;
19571 params.estimated_cache_write_input_tokens = cache_write_estimate;
19572 let req = GenerateRequest {
19573 prompt: "x".repeat(160_000),
19574 params,
19575 cache_control: true,
19576 intent: Some(IntentHint {
19577 prefer_quality: true,
19578 exclude_models,
19579 ..Default::default()
19580 }),
19581 ..Default::default()
19582 };
19583
19584 match surface {
19585 CacheRoutingSurface::Generate => {
19586 engine
19587 .generate_tracked(req)
19588 .await
19589 .expect("mocked OpenRouter generation should succeed")
19590 .model_used
19591 }
19592 CacheRoutingSurface::Stream => {
19593 let mut handle = engine
19594 .generate_tracked_stream(req)
19595 .await
19596 .expect("mocked OpenRouter stream should start");
19597 let selected = handle.model_used.clone();
19598 while handle.events.recv().await.is_some() {}
19599 selected
19600 }
19601 }
19602 }
19603
19604 #[tokio::test(flavor = "current_thread")]
19605 async fn tracked_generate_uses_explicit_cache_estimate_and_defaults_to_zero() {
19606 let without_estimate =
19607 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 0, 0).await;
19608 let with_estimate =
19609 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 40_000, 0).await;
19610 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
19611 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
19612 }
19613
19614 #[tokio::test(flavor = "current_thread")]
19615 async fn tracked_stream_uses_explicit_cache_estimate_and_defaults_to_zero() {
19616 let without_estimate =
19617 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 0).await;
19618 let with_estimate =
19619 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 40_000).await;
19620 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
19621 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
19622 }
19623
19624 #[tokio::test(flavor = "current_thread")]
19625 async fn authenticated_openrouter_registry_stays_static_and_rejects_unknown_ids() {
19626 let _credential_scope = crate::openrouter::test_credential_scope();
19627 crate::openrouter::set_test_credential(Some("static-key"));
19628 let tmp = TempDir::new().unwrap();
19629 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19630 let personal: Vec<_> = engine
19631 .list_schemas()
19632 .into_iter()
19633 .filter(|schema| schema.id.starts_with("openrouter/"))
19634 .collect();
19635 assert_eq!(personal.len(), crate::openrouter::curated_model_count());
19636 assert!(personal.iter().all(ModelSchema::available_now));
19637 assert!(personal
19638 .iter()
19639 .all(|schema| schema.trust_tier == TrustTier::Curated));
19640 assert!(personal
19641 .iter()
19642 .all(|schema| !schema.tags.iter().any(|tag| tag == "dynamic")));
19643
19644 for unknown in [
19645 "openrouter/vendor/brand-new-model",
19646 "openrouter/openai/gpt-5.4-typo",
19647 ] {
19648 assert!(engine
19649 .list_schemas()
19650 .iter()
19651 .all(|schema| schema.id != unknown));
19652 assert_eq!(engine.model_context_window(unknown), 0);
19653 let error = engine
19654 .generate_tracked(GenerateRequest {
19655 prompt: "must fail before transport".into(),
19656 model: Some(unknown.into()),
19657 params: GenerateParams {
19658 strict_model: true,
19659 ..Default::default()
19660 },
19661 ..Default::default()
19662 })
19663 .await
19664 .expect_err("unregistered personal OpenRouter ids must not reach inference");
19665 assert!(
19666 matches!(&error, InferenceError::ModelNotFound(id) if id == unknown),
19667 "{unknown}: {error}"
19668 );
19669 let stream_error = match engine
19670 .generate_tracked_stream(GenerateRequest {
19671 prompt: "must fail before stream transport".into(),
19672 model: Some(unknown.into()),
19673 ..Default::default()
19674 })
19675 .await
19676 {
19677 Ok(_) => panic!("unregistered ids must also fail before streaming"),
19678 Err(error) => error,
19679 };
19680 assert!(
19681 matches!(&stream_error, InferenceError::ModelNotFound(id) if id == unknown),
19682 "{unknown}: {stream_error}"
19683 );
19684 }
19685 }
19686
19687 #[tokio::test(flavor = "current_thread")]
19688 async fn static_openrouter_rows_participate_in_adaptive_routing_only_with_a_key() {
19689 let (_credential_scope, _provider_env) =
19690 crate::openrouter::test_credential_and_environment_scope_async().await;
19691 let tmp = TempDir::new().unwrap();
19692 crate::openrouter::set_test_credential(Some("static-key"));
19693 unsafe {
19694 std::env::set_var("CAR_STATIC_ROUTING_PEER_KEY", "peer-key");
19695 }
19696 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19697 let mut peer = remote_stream_fixture_schema(
19698 "test/openai-routing-peer",
19699 "http://127.0.0.1:9".into(),
19700 schema::ApiProtocol::OpenAiCompat,
19701 "CAR_STATIC_ROUTING_PEER_KEY",
19702 );
19703 peer.provider = "openai".into();
19704 peer.trust_tier = TrustTier::Curated;
19705 engine.unified_registry_mut().register_project_model(peer);
19706 let reviewed: std::collections::HashSet<_> = engine
19707 .list_schemas()
19708 .into_iter()
19709 .filter(|schema| schema.id.starts_with("openrouter/"))
19710 .map(|schema| schema.id)
19711 .collect();
19712 assert_eq!(reviewed.len(), crate::openrouter::curated_model_count());
19713
19714 let with_key = engine
19715 .route_adaptive_with_intent(
19716 "Answer this simple question cheaply.",
19717 Some(IntentHint::default()),
19718 )
19719 .await;
19720 let openrouter_candidates: Vec<_> = std::iter::once(with_key.model_id.as_str())
19721 .chain(
19722 with_key
19723 .candidates
19724 .iter()
19725 .map(|candidate| candidate.model_id.as_str()),
19726 )
19727 .chain(with_key.fallbacks.iter().map(String::as_str))
19728 .filter(|id| id.starts_with("openrouter/"))
19729 .collect();
19730 assert!(
19731 !openrouter_candidates.is_empty(),
19732 "keyed adaptive decision must include a reviewed OpenRouter row: {with_key:?}"
19733 );
19734 assert!(openrouter_candidates
19735 .iter()
19736 .all(|id| reviewed.contains(*id)));
19737 assert!(
19738 std::iter::once(with_key.model_id.as_str())
19739 .chain(with_key.fallbacks.iter().map(String::as_str),)
19740 .any(|id| !id.starts_with("openrouter/")),
19741 "fallback chain must retain cross-provider alternatives: {with_key:?}"
19742 );
19743
19744 crate::openrouter::set_test_credential(None);
19745 let without_key = engine
19746 .route_adaptive_with_intent(
19747 "Answer this simple question cheaply.",
19748 Some(IntentHint::default()),
19749 )
19750 .await;
19751 assert!(!std::iter::once(without_key.model_id.as_str())
19752 .chain(
19753 without_key
19754 .candidates
19755 .iter()
19756 .map(|candidate| candidate.model_id.as_str()),
19757 )
19758 .chain(without_key.fallbacks.iter().map(String::as_str))
19759 .any(|id| id.starts_with("openrouter/")));
19760 unsafe {
19761 std::env::remove_var("CAR_STATIC_ROUTING_PEER_KEY");
19762 }
19763 }
19764
19765 #[tokio::test(flavor = "current_thread")]
19766 async fn v2_parslee_auth_drives_managed_registration_routing_lane_and_logout() {
19767 if !crate::run_in_isolated_test_process(
19768 "tests::v2_parslee_auth_drives_managed_registration_routing_lane_and_logout",
19769 "CAR_V2_ROUTING_AUTH_CHILD",
19770 ) {
19771 return;
19772 }
19773 let tmp = TempDir::new().unwrap();
19774 let (_credential_scope, _provider_env) =
19775 crate::openrouter::test_credential_and_environment_scope_async().await;
19776 let _restore = RestoredEnvironment::capture(&[
19777 "CAR_SECRETS_FILE_DIR",
19778 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19779 ]);
19780 let _home = crate::openrouter::StateRootScope::new();
19787 crate::openrouter::clear_gateway_unconfigured();
19788 crate::parslee_credential::clear_credential_rejected();
19789 let secrets_dir = tmp.path().join("secrets");
19790 unsafe {
19791 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
19792 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
19793 }
19794 crate::openrouter::set_test_credential(Some("personal-openrouter-key"));
19795
19796 let store = car_secrets::SecretStore::new();
19797 let state_ref =
19798 car_secrets::SecretRef::with_default_service(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
19799 store
19800 .publish(
19801 &state_ref,
19802 &serde_json::json!({
19803 "schema": 2,
19804 "revision": 7,
19805 "generation": 3,
19806 "active": {
19807 "account_id": "account-v2",
19808 "access_token": "v2-access",
19809 "refresh_token": "v2-refresh",
19810 "expires_at": 9_999_999_999_u64,
19811 "api_base": "https://api.parslee.ai"
19812 },
19813 "accounts": [{
19814 "account_id": "account-v2",
19815 "access_token": "v2-access",
19816 "refresh_token": "v2-refresh",
19817 "expires_at": 9_999_999_999_u64,
19818 "api_base": "https://api.parslee.ai"
19819 }],
19820 "tombstone": false
19821 })
19822 .to_string(),
19823 )
19824 .unwrap();
19825 assert_eq!(car_auth::access_token().as_deref(), Some("v2-access"));
19826 assert!(
19827 !store
19828 .status(&car_secrets::SecretRef::with_default_service(
19829 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19830 ))
19831 .unwrap()
19832 .exists,
19833 "the regression must exercise V2 state without the import-only legacy slot"
19834 );
19835
19836 let managed_ids = [
19837 "parslee/openrouter/frontier-general",
19838 "parslee/openrouter/balanced-general",
19839 ];
19840 let personal_id = "openrouter/deepseek/deepseek-v3.2";
19841 let rogue_id = "community/custom-oauth";
19842 let schemas = crate::openrouter::curated_schemas();
19843 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("registry-models"));
19844 for id in managed_ids.into_iter().chain(std::iter::once(personal_id)) {
19845 registry.register_project_model(
19846 schemas
19847 .iter()
19848 .find(|schema| schema.id == id)
19849 .unwrap_or_else(|| panic!("missing curated schema {id}"))
19850 .clone(),
19851 );
19852 }
19853 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
19854 for id in managed_ids {
19855 assert!(
19856 registry.get(id).unwrap().available_now(),
19857 "{id} must be available immediately when registered from V2 auth"
19858 );
19859 }
19860 assert!(registry.get(personal_id).unwrap().available_now());
19861 let mut rogue = schemas
19862 .iter()
19863 .find(|schema| schema.id == managed_ids[0])
19864 .unwrap()
19865 .clone();
19866 rogue.id = rogue_id.into();
19867 rogue.provider = "community".into();
19868 if let ModelSource::Proprietary {
19869 provider, endpoint, ..
19870 } = &mut rogue.source
19871 {
19872 *provider = "community".into();
19873 *endpoint = "https://untrusted.example".into();
19874 } else {
19875 panic!("managed fixture must remain proprietary");
19876 }
19877 registry.register(rogue);
19878 assert!(
19879 !registry.get(rogue_id).unwrap().available_now(),
19880 "a non-Parslee OAuth schema must not inherit Parslee V2 availability"
19881 );
19882
19883 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
19884 for id in managed_ids {
19885 assert!(
19886 registry.get(id).unwrap().available_now(),
19887 "{id} must stay available in a refreshed V2-auth snapshot"
19888 );
19889 }
19890 assert!(
19891 !registry.get(rogue_id).unwrap().available_now(),
19892 "refresh must keep non-Parslee OAuth schemas unavailable"
19893 );
19894
19895 let router = AdaptiveRouter::new(
19896 crate::hardware::HardwareInfo::detect(),
19897 RoutingConfig {
19898 prefer_local: false,
19899 prior_strength: 1_000_000.0,
19900 quality_first_cold_start: false,
19901 ..RoutingConfig::default()
19902 },
19903 );
19904 let tracker = OutcomeTracker::new();
19905 let intent = IntentHint {
19906 task: Some(crate::intent::TaskHint::Chat),
19907 exclude_models: vec![personal_id.into()],
19908 ..Default::default()
19909 };
19910 let decision = router.route_with(crate::adaptive_router::RouteRequest {
19911 intent: Some(&intent),
19912 ..crate::adaptive_router::RouteRequest::new(
19913 "Explain this architecture.",
19914 ®istry,
19915 &tracker,
19916 )
19917 });
19918 assert!(
19919 managed_ids.contains(&decision.model_id.as_str()),
19920 "managed V2-auth alias must be selectable: {decision:?}"
19921 );
19922 assert!(
19923 decision
19924 .candidates
19925 .iter()
19926 .any(|candidate| managed_ids.contains(&candidate.model_id.as_str())),
19927 "managed V2-auth alias must appear in adaptive candidates: {decision:?}"
19928 );
19929 assert!(
19930 decision
19931 .fallbacks
19932 .iter()
19933 .any(|id| managed_ids.contains(&id.as_str())),
19934 "managed V2-auth alias must appear in adaptive fallbacks: {decision:?}"
19935 );
19936 assert!(
19937 !std::iter::once(decision.model_id.as_str())
19938 .chain(
19939 decision
19940 .candidates
19941 .iter()
19942 .map(|candidate| candidate.model_id.as_str())
19943 )
19944 .chain(decision.fallbacks.iter().map(String::as_str))
19945 .any(|id| id == rogue_id),
19946 "a non-Parslee OAuth schema must never enter adaptive selection, candidates, or fallbacks: {decision:?}"
19947 );
19948
19949 let engine = InferenceEngine::new(test_config(tmp.path().join("engine-models")));
19950 let managed_lane_id = managed_ids[0];
19951 engine.lane_defaults_cache.write().unwrap().set(
19952 None,
19953 crate::intent::UseCase::Assistant,
19954 managed_lane_id.into(),
19955 1,
19956 );
19957 let request = GenerateRequest {
19958 prompt: "lane default".into(),
19959 intent: Some(IntentHint {
19960 task: Some(crate::intent::TaskHint::Chat),
19961 ..Default::default()
19962 }),
19963 ..Default::default()
19964 };
19965 assert_eq!(
19966 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
19967 Some(managed_lane_id.to_string()),
19968 "a managed lane default must resolve from authoritative V2 auth"
19969 );
19970
19971 car_auth::logout()
19972 .await
19973 .expect("production logout must publish a signed-out tombstone");
19974 let persisted_logout: serde_json::Value =
19975 serde_json::from_str(&store.get(&state_ref).unwrap()).unwrap();
19976 assert_eq!(persisted_logout["tombstone"], true);
19977 assert_eq!(persisted_logout["accounts"], serde_json::json!([]));
19978 assert!(persisted_logout.get("active").is_none());
19979 assert_eq!(car_auth::access_token(), None);
19980
19981 registry.refresh_routing_availability(None, true);
19982 for id in managed_ids {
19983 assert!(
19984 !registry.get(id).unwrap().available_now(),
19985 "{id} must disappear from routing after the signed-out tombstone"
19986 );
19987 }
19988 assert!(
19989 registry.get(personal_id).unwrap().available_now(),
19990 "personal rows must continue to follow their independent personal-key seam"
19991 );
19992
19993 let signed_out = engine.routing_registry_snapshot().await;
19994 assert!(
19995 signed_out
19996 .list()
19997 .into_iter()
19998 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
19999 .all(|schema| !schema.available_now()),
20000 "the next engine snapshot must exclude every managed alias after logout"
20001 );
20002 assert!(
20003 signed_out.get(personal_id).unwrap().available_now(),
20004 "logout must not disable a still-keyed personal OpenRouter row"
20005 );
20006 assert_eq!(
20007 engine.lane_pin_for(&request, &signed_out),
20008 None,
20009 "a signed-out managed lane default must stop resolving"
20010 );
20011
20012 crate::openrouter::set_test_credential(None);
20013 registry.refresh_availability();
20014 assert!(
20015 !registry.get(personal_id).unwrap().available_now(),
20016 "personal row availability must still turn off with the personal-key seam"
20017 );
20018 }
20019
20020 #[tokio::test(flavor = "current_thread")]
20021 async fn legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates() {
20022 if !crate::run_in_isolated_test_process(
20023 "tests::legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates",
20024 "CAR_LEGACY_V2_MIGRATION_CHILD",
20025 ) {
20026 return;
20027 }
20028 let tmp = TempDir::new().unwrap();
20029 let _provider_env = crate::openrouter::test_environment_scope_async().await;
20030 let _restore = RestoredEnvironment::capture(&[
20031 "CAR_SECRETS_FILE_DIR",
20032 car_home::ENV_VAR,
20033 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20034 ]);
20035 let _home = crate::openrouter::StateRootScope::new();
20050 crate::openrouter::clear_gateway_unconfigured();
20051 crate::parslee_credential::clear_credential_rejected();
20052 unsafe {
20053 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
20054 std::env::set_var(car_home::ENV_VAR, tmp.path().join("car-home"));
20055 }
20056
20057 let secret = |key| car_secrets::SecretRef::with_default_service(key);
20058 let seed_legacy = |store: &car_secrets::SecretStore| {
20059 store
20060 .put(
20061 &secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY),
20062 "legacy-access",
20063 )
20064 .unwrap();
20065 store
20066 .put(
20067 &secret(car_secrets::PARSLEE_ACTIVE_ACCOUNT_ID_KEY),
20068 "legacy-account",
20069 )
20070 .unwrap();
20071 store.put(
20072 &secret(car_secrets::PARSLEE_ACCOUNTS_KEY),
20073 r#"{"active":"legacy-account","accounts":[{"id":"legacy-account","email":"legacy@example.test"}]}"#,
20074 )
20075 .unwrap();
20076 };
20077 let state_ref = secret(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
20078 let managed = crate::openrouter::curated_schemas()
20079 .into_iter()
20080 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
20081 .unwrap();
20082
20083 unsafe {
20084 std::env::set_var("CAR_SECRETS_FILE_DIR", tmp.path().join("tombstone-secrets"));
20085 }
20086 let tombstoned_store = car_secrets::SecretStore::new();
20087 seed_legacy(&tombstoned_store);
20088 tombstoned_store
20089 .publish(
20090 &state_ref,
20091 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
20092 )
20093 .unwrap();
20094 let mut tombstoned_registry =
20095 UnifiedRegistry::new_empty(tmp.path().join("tombstoned-models"));
20096 tombstoned_registry.register_project_model(managed.clone());
20097 assert!(
20098 !tombstoned_registry
20099 .get(&managed.id)
20100 .unwrap()
20101 .available_now(),
20102 "passive catalog registration must remain disabled without a configured hint"
20103 );
20104 let tombstoned = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
20105 .await
20106 .unwrap();
20107 assert!(tombstoned.is_none(), "the V2 tombstone is authoritative");
20108 tombstoned_registry.refresh_routing_availability(None, true);
20109 assert!(!tombstoned_registry
20110 .get(&managed.id)
20111 .unwrap()
20112 .available_now());
20113
20114 unsafe {
20115 std::env::set_var(
20116 "CAR_SECRETS_FILE_DIR",
20117 tmp.path().join("legacy-only-secrets"),
20118 );
20119 }
20120 let legacy_store = car_secrets::SecretStore::new();
20121 seed_legacy(&legacy_store);
20122 assert!(!legacy_store.status(&state_ref).unwrap().exists);
20123 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("legacy-models"));
20124 registry.register_project_model(managed.clone());
20125 assert!(
20126 !registry.get(&managed.id).unwrap().available_now(),
20127 "passive registration must not inspect attributable legacy slots"
20128 );
20129 registry.refresh_availability();
20130 assert!(
20131 !registry.get(&managed.id).unwrap().available_now(),
20132 "passive refresh must remain secret-store free until request-time migration"
20133 );
20134 let resolved = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
20135 .await
20136 .unwrap()
20137 .expect("request-time auth reconciliation must migrate attributable legacy state");
20138 registry.refresh_routing_availability(Some(&resolved.api_base), false);
20139 assert!(registry.get(&managed.id).unwrap().available_now());
20140 assert!(
20141 legacy_store.status(&state_ref).unwrap().exists,
20142 "request-time auth reconciliation must publish the migrated V2 record"
20143 );
20144 assert!(
20145 !legacy_store
20146 .status(&secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY))
20147 .unwrap()
20148 .exists,
20149 "successful V2 migration must clean the legacy access slot"
20150 );
20151 }
20152
20153 #[tokio::test]
20154 async fn reviewed_openrouter_lane_default_tracks_live_credential_availability() {
20155 let _credential_scope = crate::openrouter::test_credential_scope();
20156 crate::openrouter::set_test_credential(Some("static-key"));
20157 let tmp = TempDir::new().unwrap();
20158 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20159 let reviewed_id = "openrouter/deepseek/deepseek-v3.2";
20160 engine.lane_defaults_cache.write().unwrap().set(
20161 None,
20162 crate::intent::UseCase::Assistant,
20163 reviewed_id.into(),
20164 1,
20165 );
20166 let request = GenerateRequest {
20167 prompt: "lane default".into(),
20168 intent: Some(IntentHint {
20169 task: Some(crate::intent::TaskHint::Chat),
20170 ..Default::default()
20171 }),
20172 ..Default::default()
20173 };
20174 assert_eq!(
20175 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
20176 Some(reviewed_id.to_string()),
20177 "a reviewed keyed row must be eligible as a lane default"
20178 );
20179 crate::openrouter::set_test_credential(None);
20180 assert_eq!(
20181 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
20182 None,
20183 "the same static lane default must stop being eligible immediately after key removal"
20184 );
20185 }
20186
20187 #[tokio::test(flavor = "current_thread")]
20188 async fn personal_openrouter_baseline_remains_visible_but_disabled_without_a_credential() {
20189 let _credential_scope = crate::openrouter::test_credential_scope();
20190 crate::openrouter::set_test_credential(None);
20191 let tmp = TempDir::new().unwrap();
20192 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20193
20194 let personal: Vec<_> = engine
20195 .list_schemas()
20196 .into_iter()
20197 .filter(|schema| schema.id.starts_with("openrouter/"))
20198 .collect();
20199 assert_eq!(
20200 personal.len(),
20201 crate::openrouter::curated_model_count(),
20202 "the vetted personal rows stay discoverable as a disabled baseline"
20203 );
20204 assert!(
20205 personal.iter().all(|schema| !schema.available_now()),
20206 "no-key baseline rows must never become routing candidates"
20207 );
20208
20209 let error = engine
20210 .generate_tracked(GenerateRequest {
20211 prompt: "hello".into(),
20212 model: Some("openrouter/openai/gpt-5.4".into()),
20213 params: GenerateParams {
20214 strict_model: true,
20215 ..Default::default()
20216 },
20217 ..Default::default()
20218 })
20219 .await
20220 .expect_err("explicitly selecting a disabled baseline row must be actionable");
20221 let message = error.to_string();
20222 assert!(
20223 message.contains("car keys set openrouter") && message.contains("CarHost"),
20224 "disabled personal row must explain how to connect OpenRouter: {message}"
20225 );
20226 }
20227
20228 #[tokio::test]
20229 async fn reviewed_openrouter_metadata_stays_static_across_credential_changes() {
20230 let _credential_scope = crate::openrouter::test_credential_scope();
20231 crate::openrouter::set_test_credential(Some("static-key"));
20232
20233 let tmp = TempDir::new().unwrap();
20234 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20235 let live = engine.routing_registry_snapshot().await;
20236 let vetted = live.get("openrouter/openai/gpt-5.4").unwrap();
20237 assert!(vetted.available_now());
20238 assert_eq!(vetted.context_length, 1_050_000);
20239 assert_eq!(vetted.max_output_tokens, Some(128_000));
20240 assert_eq!(vetted.cost.input_per_mtok, Some(2.5));
20241 assert_eq!(vetted.cost.output_per_mtok, Some(15.0));
20242 assert!(
20243 !vetted.cost.pricing_tiers.is_empty(),
20244 "reviewed high-context pricing tiers must remain in the static row"
20245 );
20246 assert_eq!(vetted.trust_tier, TrustTier::Curated);
20247 assert!(vetted.tags.iter().any(|tag| tag == "frontier"));
20248 assert!(vetted.has_capability(ModelCapability::Code));
20249 assert!(vetted
20250 .supported_params
20251 .contains(&schema::GenerateParam::ExtendedThinking));
20252
20253 assert!(live.get("openrouter/vendor/unreviewed-model").is_none());
20254
20255 crate::openrouter::set_test_credential(None);
20256 let reverted = engine.routing_registry_snapshot().await;
20257 assert!(reverted.get("openrouter/vendor/unreviewed-model").is_none());
20258 let baseline = reverted.get("openrouter/openai/gpt-5.4").unwrap();
20259 assert!(
20260 !baseline.available_now(),
20261 "credential removal must disable the reviewed row without removing it"
20262 );
20263 }
20264
20265 #[test]
20284 fn model_pulled_at_runtime_is_ready_without_a_daemon_restart() {
20285 let tmp = TempDir::new().unwrap();
20286 let models_dir = tmp.path().join("models");
20287 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20288 engine.register_model(ModelSchema {
20289 id: "mlx/pulled-later".into(),
20290 name: "PulledLater".into(),
20291 provider: "local".into(),
20292 family: "qwen3".into(),
20293 version: "test".into(),
20294 capabilities: vec![ModelCapability::Generate, ModelCapability::Code],
20295 context_length: 4096,
20296 max_output_tokens: None,
20297 param_count: String::new(),
20298 quantization: None,
20299 performance: schema::PerformanceEnvelope::default(),
20300 cost: schema::CostModel::default(),
20301 source: ModelSource::Mlx {
20302 hf_repo: "example/pulled-later".into(),
20303 hf_weight_file: None,
20304 },
20305 tags: vec![],
20306 supported_params: vec![],
20307 public_benchmarks: vec![],
20308 trust_tier: TrustTier::Curated,
20309 deprecated: false,
20310 available: true,
20311 weights_ready: false,
20312 });
20313
20314 let engine = &engine;
20316 let is_ready = || {
20317 engine
20318 .list_schemas()
20319 .into_iter()
20320 .find(|s| s.id == "mlx/pulled-later")
20321 .expect("registered model must be listed")
20322 .weights_ready
20323 };
20324
20325 assert!(!is_ready(), "precondition: no weights on disk yet");
20326
20327 let dir = models_dir.join("PulledLater");
20330 std::fs::create_dir_all(&dir).unwrap();
20331 std::fs::write(dir.join("config.json"), "{}").unwrap();
20332 std::fs::write(dir.join("model.safetensors"), b"weights").unwrap();
20333
20334 assert!(
20335 is_ready(),
20336 "a model pulled at runtime must be ready without restarting the daemon"
20337 );
20338
20339 std::fs::remove_file(dir.join("model.safetensors")).unwrap();
20342 assert!(
20343 !is_ready(),
20344 "readiness must track the disk in both directions, not latch"
20345 );
20346 }
20347
20348 #[tokio::test(flavor = "current_thread")]
20349 async fn missing_openrouter_key_is_typed_without_changing_its_string_contract() {
20350 let (_credential_scope, _provider_environment) =
20354 crate::openrouter::test_credential_and_environment_scope_async().await;
20355 let home = crate::openrouter::StateRootScope::new();
20356 let _restore = RestoredEnvironment::capture(&["CAR_SECRETS_FILE_DIR"]);
20357 unsafe {
20358 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20359 }
20360
20361 crate::openrouter::set_test_credential(None);
20365
20366 let models_dir = home.path().join("models");
20367 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20368 let schema = crate::openrouter::curated_schemas()
20369 .into_iter()
20370 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
20371 .expect("personal OpenRouter fixture");
20372 let model_id = schema.id.clone();
20373 engine.unified_registry_mut().register_project_model(schema);
20374
20375 let error = engine
20376 .generate_tracked(GenerateRequest {
20377 prompt: "must stop before provider dispatch".into(),
20378 model: Some(model_id.clone()),
20379 params: GenerateParams {
20380 strict_model: true,
20381 ..Default::default()
20382 },
20383 ..Default::default()
20384 })
20385 .await
20386 .expect_err("an OpenRouter route without a key must fail before dispatch");
20387
20388 let tracker = engine.outcome_tracker();
20389 assert!(
20390 tracker.read().await.profile(&model_id).is_none(),
20391 "a missing OpenRouter key must not create model health"
20392 );
20393 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20394 receipts.extend(tracker.write().await.drain_ledger());
20395 let receipts = receipts
20396 .iter()
20397 .filter(|receipt| receipt.model_id == model_id)
20398 .collect::<Vec<_>>();
20399 assert_eq!(receipts.len(), 1);
20400 assert_eq!(receipts[0].success, None);
20401 assert_eq!(receipts[0].quality, None);
20402 assert!(
20403 engine
20404 .adaptive_router
20405 .circuit_breakers
20406 .lock()
20407 .unwrap()
20408 .state(&model_id)
20409 .is_none(),
20410 "a missing OpenRouter key must not create breaker state"
20411 );
20412
20413 let message = "OpenRouter requires a key — run `car keys set openrouter` or connect your OpenRouter account in CarHost";
20414 assert_eq!(
20415 error.to_string(),
20416 format!("inference failed: {message}"),
20417 "guard: the pre-existing Display contract must stay byte-identical"
20418 );
20419 assert!(
20420 !is_auth_failure_message(&error.to_string()),
20421 "guard: an OpenRouter key is not repaired by Parslee sign-in"
20422 );
20423 assert_eq!(
20424 classify_fallback_reason(&error),
20425 classify_fallback_reason(&InferenceError::InferenceFailed(message.into())),
20426 "retyping must preserve prior fallback classification"
20427 );
20428 match error {
20429 InferenceError::ProviderKeyMissing {
20430 provider,
20431 model,
20432 env_vars,
20433 message: actual_message,
20434 } => {
20435 assert_eq!(provider, "openrouter");
20436 assert_eq!(model, model_id);
20437 assert_eq!(env_vars, vec![crate::openrouter::API_KEY_ENV]);
20438 assert_eq!(actual_message, message);
20439 }
20440 other => panic!("expected ProviderKeyMissing, got {other:?}"),
20441 }
20442 }
20443
20444 #[tokio::test(flavor = "current_thread")]
20445 async fn missing_generic_remote_key_is_typed_without_changing_its_string_contract() {
20446 const KEY_ENV: &str = "CAR_TEST_A47_GENERIC_PROVIDER_KEY_DO_NOT_SET";
20447 const EXTRA_KEY_ENV_1: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_1_DO_NOT_SET";
20448 const EXTRA_KEY_ENV_2: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_2_DO_NOT_SET";
20449 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
20450 let home = crate::openrouter::StateRootScope::new();
20451 let _restore = RestoredEnvironment::capture(&[
20452 "CAR_SECRETS_FILE_DIR",
20453 KEY_ENV,
20454 EXTRA_KEY_ENV_1,
20455 EXTRA_KEY_ENV_2,
20456 ]);
20457 unsafe {
20458 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20459 std::env::remove_var(KEY_ENV);
20460 std::env::remove_var(EXTRA_KEY_ENV_1);
20461 std::env::remove_var(EXTRA_KEY_ENV_2);
20462 }
20463
20464 let endpoint = "http://127.0.0.1:9/a47-missing-key";
20465 let model_id = "test/a47-generic-missing-key";
20466 let models_dir = home.path().join("models");
20467 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20468 let mut schema = remote_stream_fixture_schema(
20469 model_id,
20470 endpoint.into(),
20471 schema::ApiProtocol::OpenAiCompat,
20472 KEY_ENV,
20473 );
20474 let ModelSource::RemoteApi { api_key_envs, .. } = &mut schema.source else {
20475 panic!("generic remote fixture must use RemoteApi");
20476 };
20477 *api_key_envs = vec![EXTRA_KEY_ENV_1.into(), EXTRA_KEY_ENV_2.into()];
20478 engine.unified_registry_mut().register_project_model(schema);
20479
20480 let error = engine
20481 .generate_tracked(GenerateRequest {
20482 prompt: "must stop before provider dispatch".into(),
20483 model: Some(model_id.into()),
20484 params: GenerateParams {
20485 strict_model: true,
20486 ..Default::default()
20487 },
20488 ..Default::default()
20489 })
20490 .await
20491 .expect_err("a generic remote route without a key must fail before dispatch");
20492
20493 let tracker = engine.outcome_tracker();
20494 assert!(
20495 tracker.read().await.profile(model_id).is_none(),
20496 "a missing generic provider key must not create model health"
20497 );
20498 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20499 receipts.extend(tracker.write().await.drain_ledger());
20500 let receipts = receipts
20501 .iter()
20502 .filter(|receipt| receipt.model_id == model_id)
20503 .collect::<Vec<_>>();
20504 assert_eq!(receipts.len(), 1);
20505 assert_eq!(receipts[0].success, None);
20506 assert_eq!(receipts[0].quality, None);
20507 assert!(
20508 engine
20509 .adaptive_router
20510 .circuit_breakers
20511 .lock()
20512 .unwrap()
20513 .state(model_id)
20514 .is_none(),
20515 "a missing generic provider key must not create breaker state"
20516 );
20517
20518 let message = format!(
20519 "no API keys available for endpoint {endpoint} (checked env vars: [\"{KEY_ENV}\", \"{EXTRA_KEY_ENV_1}\", \"{EXTRA_KEY_ENV_2}\"])"
20520 );
20521 assert_eq!(
20522 error.to_string(),
20523 format!("inference failed: {message}"),
20524 "guard: the pre-existing Display contract must stay byte-identical"
20525 );
20526 assert!(
20527 !is_auth_failure_message(&error.to_string()),
20528 "guard: a generic provider key is not repaired by Parslee sign-in"
20529 );
20530 assert_eq!(
20531 classify_fallback_reason(&error),
20532 classify_fallback_reason(&InferenceError::InferenceFailed(message.clone())),
20533 "retyping must preserve prior fallback classification"
20534 );
20535 match error {
20536 InferenceError::ProviderKeyMissing {
20537 provider,
20538 model,
20539 env_vars,
20540 message: actual_message,
20541 } => {
20542 assert_eq!(provider, "test");
20543 assert_eq!(model, model_id);
20544 assert_eq!(env_vars, vec![KEY_ENV, EXTRA_KEY_ENV_1, EXTRA_KEY_ENV_2]);
20545 assert_eq!(actual_message, message);
20546 }
20547 other => panic!("expected ProviderKeyMissing, got {other:?}"),
20548 }
20549 }
20550
20551 #[test]
20555 fn credential_unavailable_outcomes_do_not_degrade_models_or_open_breakers() {
20556 for reason in [
20557 CredentialFailure::Expired { expires_at: 42 },
20558 CredentialFailure::SignedOut,
20559 CredentialFailure::StoreUnreadable,
20560 CredentialFailure::EnvVarMissing {
20561 env_var: "TEST_PROVIDER_KEY".into(),
20562 },
20563 CredentialFailure::RaceRetryable,
20564 ] {
20565 let model_id = "parslee/test-credential-outcome";
20566 let error = InferenceError::CredentialUnavailable {
20567 provider: "parslee".into(),
20568 model: model_id.into(),
20569 reason: reason.clone(),
20570 detail: "credential resolution stopped before dispatch".into(),
20571 };
20572 let mut tracker = OutcomeTracker::new();
20573 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
20574 record_dispatch_failure(&mut tracker, &trace, &error);
20575
20576 assert!(
20577 tracker.profile(model_id).is_none(),
20578 "{reason:?} changed the model profile: {:?}",
20579 tracker.profile(model_id)
20580 );
20581 let receipts = tracker.drain_ledger();
20582 assert_eq!(receipts.len(), 1, "{reason:?}");
20583 assert_eq!(receipts[0].success, None, "{reason:?}");
20584 assert_eq!(receipts[0].quality, None, "{reason:?}");
20585
20586 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
20587 if error_counts_against_circuit_breaker(&error) {
20588 breakers.record_failure(model_id);
20589 }
20590 assert!(
20591 breakers.state(model_id).is_none(),
20592 "{reason:?} created per-model breaker state"
20593 );
20594 }
20595 }
20596
20597 #[tokio::test(flavor = "current_thread")]
20598 async fn signed_out_managed_attempts_leave_health_and_breaker_untouched() {
20599 if !crate::run_in_isolated_test_process(
20600 "tests::signed_out_managed_attempts_leave_health_and_breaker_untouched",
20601 "CAR_SIGNED_OUT_MODEL_HEALTH_CHILD",
20602 ) {
20603 return;
20604 }
20605
20606 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
20607 let home = crate::openrouter::StateRootScope::new();
20608 let _restore = RestoredEnvironment::capture(&[
20609 "CAR_SECRETS_FILE_DIR",
20610 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20611 car_auth::PARSLEE_API_BASE_KEY,
20612 ]);
20613 unsafe {
20614 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20615 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
20616 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
20617 }
20618 crate::openrouter::clear_gateway_unconfigured();
20619 crate::parslee_credential::clear_credential_rejected();
20620
20621 let store = car_secrets::SecretStore::new();
20622 store
20623 .publish(
20624 &car_secrets::SecretRef::with_default_service(
20625 car_secrets::PARSLEE_AUTH_STATE_V2_KEY,
20626 ),
20627 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
20628 )
20629 .unwrap();
20630
20631 let models_dir = home.path().join("models");
20632 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20633 let schema = crate::openrouter::curated_schemas()
20634 .into_iter()
20635 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
20636 .expect("managed alias fixture");
20637 let model_id = schema.id.clone();
20638 engine.unified_registry_mut().register_project_model(schema);
20639
20640 for _ in 0..3 {
20644 let error = engine
20645 .generate_tracked(GenerateRequest {
20646 prompt: "must stop before provider dispatch".into(),
20647 model: Some(model_id.clone()),
20648 params: GenerateParams {
20649 strict_model: true,
20650 ..Default::default()
20651 },
20652 ..Default::default()
20653 })
20654 .await
20655 .expect_err("signed-out managed inference must be rejected");
20656 assert!(
20657 matches!(
20658 &error,
20659 InferenceError::CredentialUnavailable {
20660 reason: CredentialFailure::SignedOut,
20661 ..
20662 }
20663 ),
20664 "unexpected signed-out error: {error:?}"
20665 );
20666 }
20667
20668 let tracker = engine.outcome_tracker();
20669 assert!(
20670 tracker.read().await.profile(&model_id).is_none(),
20671 "signed-out attempts must not create a model-health profile"
20672 );
20673 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20677 receipts.extend(tracker.write().await.drain_ledger());
20678 let signed_out_receipts = receipts
20679 .iter()
20680 .filter(|entry| entry.model_id == model_id)
20681 .collect::<Vec<_>>();
20682 assert_eq!(signed_out_receipts.len(), 3);
20683 assert!(signed_out_receipts
20684 .iter()
20685 .all(|entry| entry.success.is_none() && entry.quality.is_none()));
20686 assert!(
20687 engine
20688 .adaptive_router
20689 .circuit_breakers
20690 .lock()
20691 .unwrap()
20692 .state(&model_id)
20693 .is_none(),
20694 "signed-out attempts must not create per-model breaker state"
20695 );
20696 }
20697
20698 fn admission_refusals() -> Vec<InferenceError> {
20702 use resource_policy::{LocalLoadVerdict, ModelMemoryEstimate, ModelResourceEvidence};
20703 [
20704 LocalLoadVerdict::LiveMemoryUnknown,
20705 LocalLoadVerdict::DisabledByPolicy,
20706 LocalLoadVerdict::ExceedsConfiguredCeiling,
20707 LocalLoadVerdict::InsufficientLiveMemory,
20708 LocalLoadVerdict::ModelMaintenance,
20709 LocalLoadVerdict::PendingTeardown,
20710 ]
20711 .into_iter()
20712 .map(|verdict| {
20713 InferenceError::from(resource_policy::LocalAdmissionError {
20714 preflight: resource_policy::LocalLoadPreflight {
20715 model_id: "vllm-mlx/big:4bit".into(),
20716 estimate: ModelMemoryEstimate {
20717 weights_mb: 16_000,
20718 runtime_overhead_mb: 1_000,
20719 context_overhead_mb: 2_000,
20720 transient_margin_mb: 1_000,
20721 estimated_peak_mb: 20_000,
20722 evidence: ModelResourceEvidence::CatalogExact,
20723 },
20724 configured_ceiling_mb: 26_214,
20725 resident_model_mb: 0,
20726 active_reservations_mb: 0,
20727 estimated_incremental_mb: 20_000,
20728 accelerator_total_mb: None,
20729 accelerator_resident_mb: None,
20730 accelerator_incremental_mb: None,
20731 live_available_mb: Some(4_000),
20732 emergency_reserve_mb: 6_553,
20733 verdict,
20734 },
20735 })
20736 })
20737 .collect()
20738 }
20739
20740 #[test]
20744 fn local_admission_refusal_is_not_a_model_failure() {
20745 let model_id = "vllm-mlx/big:4bit";
20746 for refusal in admission_refusals() {
20747 assert!(matches!(
20748 refusal,
20749 InferenceError::LocalResourceBlocked { .. }
20750 ));
20751 let mut tracker = OutcomeTracker::new();
20752 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
20753 record_dispatch_failure(&mut tracker, &trace, &refusal);
20754 assert!(
20755 tracker.profile(model_id).is_none(),
20756 "admission refusal changed the model profile: {refusal}"
20757 );
20758 let receipt = tracker.drain_ledger().pop().expect("receipt kept");
20759 assert_eq!(receipt.success, None, "{refusal}");
20760 assert_eq!(receipt.quality, None, "{refusal}");
20761 assert!(!error_counts_against_circuit_breaker(&refusal), "{refusal}");
20762 }
20763 }
20764
20765 #[test]
20766 fn content_refusal_closes_its_parenthetical_in_every_shape() {
20767 let render = |kind: Option<&str>, code: Option<&str>| {
20768 InferenceError::ContentRefused {
20769 provider: "p".into(),
20770 kind: kind.map(Into::into),
20771 code: code.map(Into::into),
20772 message: "m".into(),
20773 }
20774 .to_string()
20775 };
20776 assert_eq!(
20777 render(Some("k"), Some("c")),
20778 "p refused this request on content grounds (type=k, code=c): m"
20779 );
20780 assert_eq!(
20781 render(Some("k"), None),
20782 "p refused this request on content grounds (type=k): m"
20783 );
20784 assert_eq!(
20785 render(None, Some("c")),
20786 "p refused this request on content grounds (code=c): m"
20787 );
20788 assert_eq!(
20789 render(None, None),
20790 "p refused this request on content grounds: m"
20791 );
20792 }
20793
20794 fn legacy_unattributed_errors() -> Vec<String> {
20799 let mut legacy: Vec<String> = admission_refusals()
20800 .iter()
20801 .map(ToString::to_string)
20802 .collect();
20803 legacy.push(
20804 InferenceError::ProviderKeyMissing {
20805 provider: "anthropic".into(),
20806 model: "m".into(),
20807 env_vars: vec!["ANTHROPIC_API_KEY".into()],
20808 message: "no API keys available for endpoint https://api.anthropic.com (checked env vars: [\"ANTHROPIC_API_KEY\"])".into(),
20809 }
20810 .to_string(),
20811 );
20812 legacy.push(
20813 InferenceError::InferenceFailed(
20814 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required".into(),
20815 )
20816 .to_string(),
20817 );
20818 legacy.push(
20819 InferenceError::InferenceFailed(
20820 "FoundationModels declined the request (guardrailViolation): unsafe".into(),
20821 )
20822 .to_string(),
20823 );
20824 legacy.push(
20825 InferenceError::InferenceFailed(
20826 InferenceError::UnsupportedMode {
20827 mode: "apple-foundation-models",
20828 backend: "remote",
20829 reason: "no",
20830 }
20831 .to_string(),
20832 )
20833 .to_string(),
20834 );
20835 legacy
20836 }
20837
20838 fn write_legacy_ledger(
20842 ledger: &std::path::Path,
20843 real: &str,
20844 ghost: &str,
20845 ) -> Vec<outcome::ModelProfile> {
20846 let mut writer = OutcomeTracker::new();
20847 for error in legacy_unattributed_errors() {
20848 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
20849 writer.record_failure(&t, &error);
20850 }
20851 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
20852 writer.record_failure(&t, GENUINE_FAILURE);
20853 let t = writer.record_start(ghost, InferenceTask::Generate, "explicit model");
20854 writer.record_failure(
20855 &t,
20856 &InferenceError::InferenceFailed(
20857 InferenceError::ModelNotFound(ghost.into()).to_string(),
20858 )
20859 .to_string(),
20860 );
20861 outcome::append_ledger_entries(ledger, &writer.drain_ledger()).unwrap();
20862 writer.export_profiles()
20863 }
20864
20865 fn reconcile_for_test(
20868 ledger: &std::path::Path,
20869 tracker: &mut OutcomeTracker,
20870 is_registered: &dyn Fn(&str) -> bool,
20871 ) -> std::io::Result<outcome::LedgerReconciliation> {
20872 let plan = outcome::plan_reconcile(
20873 ledger,
20874 tracker,
20875 is_registered,
20876 &std::collections::HashMap::new(),
20877 );
20878 if let Some(entries) = &plan.ledger {
20879 outcome::write_reconciled_ledger(ledger, entries)?;
20880 }
20881 if let Some(profiles) = plan.profiles {
20882 tracker.commit_reconciled(profiles);
20883 }
20884 Ok(plan.report)
20885 }
20886
20887 const GENUINE_FAILURE: &str =
20888 "inference failed: forward: \"[metal::malloc] Attempting to allocate 41846087808 bytes\"";
20889
20890 fn profile_json(tracker: &OutcomeTracker, id: &str) -> serde_json::Value {
20891 let mut v = serde_json::to_value(tracker.profile(id).unwrap()).unwrap();
20892 v.as_object_mut().unwrap().remove("updated_at");
20893 v
20894 }
20895
20896 #[test]
20901 fn reconcile_takes_back_misattributed_failures_and_their_quality_evidence() {
20902 let dir = tempfile::tempdir().unwrap();
20903 let ledger = dir.path().join("outcome_ledger.jsonl");
20904 let (real, ghost) = ("mlx/real:4bit", "nonexistent-model-id-xyz");
20905 let profiles = write_legacy_ledger(&ledger, real, ghost);
20906 let legacy = legacy_unattributed_errors().len();
20907
20908 let mut tracker = OutcomeTracker::new();
20909 tracker.import_profiles(profiles);
20910 let before = tracker.profile(real).unwrap().clone();
20911 assert_eq!(before.fail_count, legacy as u64 + 1);
20912 assert!(
20913 before.quality_observations > 1,
20914 "the legacy rows moved the EMA"
20915 );
20916 assert!(
20917 before.ema_quality < 0.4 - 1e-9,
20918 "buried below one failure's worth"
20919 );
20920
20921 let is_registered = |id: &str| id == real;
20922 let report = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
20923 assert_eq!(report.reclassified, legacy);
20924 assert_eq!(report.unregistered, 1);
20925 assert_eq!(report.profiles_rebuilt, 1);
20926 assert_eq!(report.profiles_removed, 1);
20927
20928 let p = tracker.profile(real).unwrap();
20929 assert_eq!((p.fail_count, p.total_calls, p.success_count), (1, 1, 0));
20930 assert_eq!(p.quality_observations, 1);
20931 assert!(
20932 (p.ema_quality - 0.4).abs() < 1e-12,
20933 "0.5 stepped once toward 0: {}",
20934 p.ema_quality
20935 );
20936 let ts = &p.task_stats["generate"];
20937 assert_eq!((ts.failures, ts.calls, ts.quality_observations), (1, 0, 1));
20939 assert!(tracker.profile(ghost).is_none());
20940
20941 let rows = outcome::read_ledger(&ledger, 0);
20942 assert_eq!(rows.len(), legacy + 2, "nothing is deleted");
20943 for row in &rows {
20944 match row.error.as_deref() {
20945 Some(GENUINE_FAILURE) => {
20946 assert_eq!(row.success, Some(false));
20947 assert_eq!(row.attribution, None);
20948 }
20949 _ if row.model_id == ghost => {
20950 assert_eq!(row.success, None);
20951 assert_eq!(
20952 row.attribution,
20953 Some(outcome::UnattributedReason::Unregistered)
20954 );
20955 }
20956 Some(error) => {
20957 use outcome::UnattributedReason as R;
20958 assert_eq!(row.success, None);
20959 assert_eq!(row.quality, None);
20960 let expected = if error.contains("memory")
20962 || error.contains("allocation")
20963 || error.contains("maintained")
20964 || error.contains("previous local model")
20965 {
20966 R::ResourceBlocked
20967 } else if error.contains("API keys") || error.contains("org lookup") {
20968 R::Account
20969 } else if error.contains("declined") {
20970 R::ContentRefused
20971 } else {
20972 R::Capability
20973 };
20974 assert_eq!(row.attribution, Some(expected), "{error}");
20975 }
20976 None => panic!("every legacy row carries its error"),
20977 }
20978 }
20979 #[cfg(unix)]
20980 {
20981 use std::os::unix::fs::PermissionsExt;
20982 let mode = std::fs::metadata(&ledger).unwrap().permissions().mode() & 0o777;
20983 assert_eq!(mode, 0o600, "a rewritten ledger stays owner-only");
20984 }
20985
20986 let rebuilt = profile_json(&tracker, real);
20988 let again = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
20989 assert_eq!(
20990 again.reclassified + again.unregistered + again.profiles_removed,
20991 0
20992 );
20993 assert_eq!(profile_json(&tracker, real), rebuilt);
20994 }
20995
20996 #[test]
21002 fn reconcile_repairs_a_ledger_the_first_migration_already_rewrote() {
21003 let dir = tempfile::tempdir().unwrap();
21004 let ledger = dir.path().join("outcome_ledger.jsonl");
21005 let real = "mlx/real:4bit";
21006 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
21007
21008 let mut clean = OutcomeTracker::new();
21010 clean.import_profiles(profiles.clone());
21011 let clean_ledger = dir.path().join("clean.jsonl");
21012 std::fs::copy(&ledger, &clean_ledger).unwrap();
21013 reconcile_for_test(&clean_ledger, &mut clean, &|id| id == real).unwrap();
21014 let expected = profile_json(&clean, real);
21015
21016 let rows: Vec<_> = outcome::read_ledger(&ledger, 0)
21019 .into_iter()
21020 .map(|mut r| {
21021 if r.model_id == real && r.error.as_deref() != Some(GENUINE_FAILURE) {
21022 r.success = None;
21023 r.quality = None;
21024 }
21025 r
21026 })
21027 .collect();
21028 std::fs::remove_file(&ledger).unwrap();
21029 outcome::append_ledger_entries(&ledger, &rows).unwrap();
21030 let mut v2 = profiles;
21031 for p in &mut v2 {
21032 if p.model_id == real {
21033 p.fail_count = 1;
21034 p.total_calls = 1;
21035 p.task_stats.get_mut("generate").unwrap().failures = 1;
21036 }
21037 }
21038 let mut tracker = OutcomeTracker::new();
21039 tracker.import_profiles(v2);
21040 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21041 assert_eq!(profile_json(&tracker, real), expected);
21042 }
21043
21044 #[test]
21049 fn reconcile_clamps_a_broken_profile_and_keeps_ids_that_ran() {
21050 let dir = tempfile::tempdir().unwrap();
21051 let ledger = dir.path().join("outcome_ledger.jsonl");
21052 let (model, renamed) = ("parslee/reasoning", "openai/gpt-5.4");
21053 let mut writer = OutcomeTracker::new();
21054 for id in [model, renamed] {
21055 for _ in 0..4 {
21056 let t = writer.record_start(id, InferenceTask::Generate, "r");
21057 writer.record_failure(
21058 &t,
21059 "inference failed: Parslee inference returned no content",
21060 );
21061 }
21062 }
21063 let mut rows = writer.drain_ledger();
21064 let mut ok = rows[0].clone();
21065 ok.model_id = renamed.into();
21066 ok.success = Some(true);
21067 ok.error = None;
21068 rows.push(ok);
21069 outcome::append_ledger_entries(&ledger, &rows).unwrap();
21070
21071 let mut profiles = writer.export_profiles();
21072 for p in &mut profiles {
21073 if p.model_id == model {
21074 p.success_count = 1522;
21075 p.total_calls = 1519;
21076 p.fail_count = 0;
21077 p.task_stats.get_mut("generate").unwrap().failures = 0;
21078 }
21079 }
21080 let mut tracker = OutcomeTracker::new();
21081 tracker.import_profiles(profiles);
21082 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == model).unwrap();
21083 assert_eq!(report.unregistered, 0, "an id that ran is not unregistered");
21084 let p = tracker.profile(model).unwrap();
21085 assert_eq!(
21086 (p.success_count, p.fail_count, p.total_calls),
21087 (1522, 0, 1522)
21088 );
21089 assert_eq!(tracker.profile(renamed).unwrap().fail_count, 4);
21090 assert!(outcome::read_ledger(&ledger, 0)
21091 .iter()
21092 .filter(|r| r.model_id == renamed && r.success != Some(true))
21093 .all(|r| r.success == Some(false)));
21094 }
21095
21096 #[test]
21102 fn reconcile_keeps_successes_the_ledger_never_saw() {
21103 let dir = tempfile::tempdir().unwrap();
21104 let ledger = dir.path().join("outcome_ledger.jsonl");
21105 let model = "mlx/qwen3-30b-a3b:4bit";
21106 let mut writer = OutcomeTracker::new();
21107 for _ in 0..20 {
21108 let t = writer.record_start(model, InferenceTask::Generate, "r");
21109 writer.record_complete(&t, 100, 10, 20);
21110 }
21111 for refusal in admission_refusals().iter().take(3) {
21112 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
21113 writer.record_failure(&t, &refusal.to_string());
21114 }
21115 let written = writer.drain_ledger();
21117 assert!(written.iter().all(|r| r.success == Some(false)));
21118 outcome::append_ledger_entries(&ledger, &written).unwrap();
21119 let mut tracker = OutcomeTracker::new();
21120 tracker.import_profiles(writer.export_profiles());
21121 let before = tracker.profile(model).unwrap().clone();
21122 assert_eq!((before.success_count, before.fail_count), (20, 3));
21123
21124 reconcile_for_test(&ledger, &mut tracker, &|_| true).unwrap();
21125 let p = tracker.profile(model).unwrap();
21126 assert_eq!((p.success_count, p.fail_count), (20, 0));
21127 assert_eq!(p.total_calls, before.total_calls - 3);
21128 assert_eq!(p.quality_observations, 0);
21129 assert_eq!(p.ema_quality, 0.5);
21130 }
21131
21132 #[test]
21136 fn reconcile_replays_a_seeded_profile_from_its_benchmark_prior() {
21137 let dir = tempfile::tempdir().unwrap();
21138 let ledger = dir.path().join("outcome_ledger.jsonl");
21139 let model = "mlx/qwen3-4b:4bit";
21140 let mut seeded = outcome::ModelProfile::new(model.into());
21141 seeded.ema_quality = 0.7;
21142 seeded.prior_sample_size = 3;
21143 let mut writer = OutcomeTracker::new();
21144 writer.import_profiles(vec![seeded]);
21145 for refusal in admission_refusals().iter().take(2) {
21146 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
21147 writer.record_failure(&t, &refusal.to_string());
21148 }
21149 outcome::append_ledger_entries(&ledger, &writer.drain_ledger()).unwrap();
21150 let mut tracker = OutcomeTracker::new();
21151 tracker.import_profiles(writer.export_profiles());
21152 let before = tracker.profile(model).unwrap().clone();
21153 assert_eq!(before.quality_observations, 2);
21154 assert!(before.ema_quality < 0.7 * 0.8 * 0.8 + 1e-9);
21155
21156 let seeds = std::collections::HashMap::from([(
21157 model.to_string(),
21158 outcome::QualitySeed {
21159 overall: Some(0.7),
21160 tasks: std::collections::HashMap::new(),
21161 },
21162 )]);
21163 let plan = outcome::plan_reconcile(&ledger, &tracker, &|_| true, &seeds);
21164 assert_eq!(plan.report.profiles_rebuilt, 1);
21165 tracker.commit_reconciled(plan.profiles.expect("a corrected profile set"));
21166 let p = tracker.profile(model).unwrap();
21167 assert_eq!(p.quality_observations, 0);
21168 assert!(
21169 (p.ema_quality - 0.7).abs() < 1e-12,
21170 "back to the seed: {}",
21171 p.ema_quality
21172 );
21173 assert_eq!(p.prior_sample_size, 3, "the prior's weight is the prior's");
21174 }
21175
21176 #[test]
21180 fn reconcile_is_quiet_in_steady_state() {
21181 let dir = tempfile::tempdir().unwrap();
21182 let ledger = dir.path().join("outcome_ledger.jsonl");
21183 let real = "mlx/real:4bit";
21184 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
21185 let mut tracker = OutcomeTracker::new();
21186 tracker.import_profiles(profiles);
21187 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21188
21189 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21191 tracker.record_complete(&t, 120, 10, 20);
21192 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21193 tracker.record_complete(&t, 80, 10, 0);
21194 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21195 tracker.record_failure(&t, GENUINE_FAILURE);
21196 outcome::append_ledger_entries(&ledger, &tracker.drain_ledger()).unwrap();
21197 let settled = profile_json(&tracker, real);
21198
21199 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21200 assert_eq!(report, outcome::LedgerReconciliation::default());
21201 assert_eq!(profile_json(&tracker, real), settled);
21202
21203 outcome::prune_ledger(&ledger, 2).unwrap();
21206 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21207 assert_eq!(report, outcome::LedgerReconciliation::default());
21208 assert_eq!(profile_json(&tracker, real), settled);
21209 }
21210
21211 #[test]
21215 fn neutral_profile_is_not_backfilled_on_load() {
21216 let dir = tempfile::tempdir().unwrap();
21217 let path = dir.path().join("profiles.json");
21218 let mut writer = OutcomeTracker::new();
21219 let t = writer.record_start("m", InferenceTask::Generate, "r");
21220 writer.record_failure(&t, "daemon read timeout on infer after 30s");
21221 let p = writer.profile("m").unwrap();
21222 assert_eq!((p.fail_count, p.quality_observations), (1, 0));
21223 assert_eq!(p.ema_quality, 0.5);
21224 writer.save_to_file(&path).unwrap();
21225 let mut reader = OutcomeTracker::new();
21226 reader.load_from_file(&path).unwrap();
21227 assert_eq!(reader.profile("m").unwrap().quality_observations, 0);
21228 }
21229
21230 #[tokio::test]
21234 async fn reconcile_refuses_when_profiles_could_not_be_read() {
21235 let root = tempfile::tempdir().unwrap();
21236 let state = root.path().join("state");
21237 let models = state.join("models");
21238 std::fs::create_dir_all(&models).unwrap();
21239 let ledger = models.join("outcome_ledger.jsonl");
21240 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
21241 std::fs::write(models.join("outcome_profiles.json"), b"{ not json").unwrap();
21242 let before = std::fs::read(&ledger).unwrap();
21243 let engine = InferenceEngine::new(InferenceConfig {
21244 state_root: state.clone(),
21245 models_dir: root.path().join("weights"),
21246 ..InferenceConfig::default()
21247 });
21248 let report = engine.reconcile_outcome_ledger().await.unwrap();
21249 assert_eq!(report, outcome::LedgerReconciliation::default());
21250 assert_eq!(std::fs::read(&ledger).unwrap(), before, "ledger untouched");
21251 assert_eq!(
21252 std::fs::read(models.join("outcome_profiles.json")).unwrap(),
21253 b"{ not json",
21254 "profiles untouched"
21255 );
21256 }
21257
21258 #[cfg(unix)]
21261 #[tokio::test]
21262 async fn retire_plan_includes_the_projection_and_engine_refusals() {
21263 let _serial = RETIRE_LOCKS.lock().await;
21264 let root = tempfile::tempdir().unwrap();
21265 let root_path = root.path().canonicalize().unwrap();
21266 let hub = root_path.join("hub");
21267 let models = root_path.join("weights");
21268 let engine = InferenceEngine::new(InferenceConfig {
21269 state_root: root_path.join("state"),
21270 models_dir: models.clone(),
21271 ..InferenceConfig::default()
21272 });
21273 let schema = engine
21274 .unified_registry()
21275 .all()
21276 .find(|m| m.id == plain_mlx_id(&engine))
21277 .unwrap()
21278 .clone();
21279 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
21280 unreachable!()
21281 };
21282 let repo = hf_cache::repo_dir_in(&hub, hf_repo);
21283 std::fs::create_dir_all(repo.join("snapshots/s")).unwrap();
21284 std::fs::create_dir_all(repo.join("blobs")).unwrap();
21285 std::fs::write(repo.join("blobs/w"), vec![0u8; 64]).unwrap();
21286 std::os::unix::fs::symlink(repo.join("blobs/w"), repo.join("snapshots/s/w")).unwrap();
21287 std::fs::create_dir_all(&models).unwrap();
21288 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
21289
21290 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21291 assert!(!plan.is_refused(), "{plan:?}");
21292 assert_eq!(
21293 plan.freed_bytes, 64,
21294 "the link frees nothing; the repo frees its blob"
21295 );
21296 assert_eq!(
21297 plan.projections,
21298 vec![retire::Projection {
21299 path: models.join(&schema.name),
21300 receipt_for: None,
21301 }]
21302 );
21303 assert!(matches!(
21304 &plan.deletions[..],
21305 [retire::HubDeletion::Repo { .. }]
21306 ));
21307
21308 let guard = crate::download::acquire_model_lock(&schema.id).await;
21309 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21310 assert!(plan.refusals.contains(&retire::RetireRefusal::Downloading));
21311 drop(guard);
21312
21313 engine.lane_defaults_cache.write().unwrap().set(
21314 None,
21315 crate::intent::UseCase::Coding,
21316 schema.id.clone(),
21317 1,
21318 );
21319 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21320 assert!(plan
21321 .refusals
21322 .iter()
21323 .any(|r| matches!(r, retire::RetireRefusal::LaneDefault { .. })));
21324 }
21325
21326 #[cfg(unix)]
21329 #[tokio::test]
21330 async fn retire_plan_contains_projections_and_sees_sibling_downloads() {
21331 let _serial = RETIRE_LOCKS.lock().await;
21332 let root = tempfile::tempdir().unwrap();
21333 let root_path = root.path().canonicalize().unwrap();
21334 let hub = root_path.join("hub");
21335 std::fs::create_dir_all(&hub).unwrap();
21336 let mut engine = InferenceEngine::new(InferenceConfig {
21337 state_root: root_path.join("state"),
21338 models_dir: root_path.join("weights"),
21339 ..InferenceConfig::default()
21340 });
21341 let mut escaping = engine
21342 .unified_registry()
21343 .get("mlx/qwen3-4b:4bit")
21344 .unwrap()
21345 .clone();
21346 escaping.id = "user/escape".into();
21347 escaping.name = "../../outside".into();
21348 std::fs::create_dir_all(root_path.join("outside")).unwrap();
21349 engine
21350 .unified_registry_mut()
21351 .register_project_model(escaping);
21352 let plan = engine.retire_plan_in("user/escape", &hub).unwrap();
21353 assert!(plan.projections.is_empty(), "{plan:?}");
21354 assert!(plan
21355 .refusals
21356 .iter()
21357 .any(|r| matches!(r, retire::RetireRefusal::UnsafePath { .. })));
21358
21359 let guard = crate::download::acquire_model_lock("vllm-mlx/gemma-4-12b-it").await;
21361 let plan = engine
21362 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
21363 .unwrap();
21364 assert!(
21365 plan.refusals.contains(&retire::RetireRefusal::Downloading),
21366 "{plan:?}"
21367 );
21368 assert_eq!(
21370 plan.also_retires,
21371 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
21372 );
21373 drop(guard);
21374 let plan = engine
21375 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
21376 .unwrap();
21377 assert!(!plan.refusals.contains(&retire::RetireRefusal::Downloading));
21378 }
21379
21380 #[cfg(unix)]
21383 #[test]
21384 fn discovered_rows_are_retired_alone_and_their_files_are_kept() {
21385 let root = tempfile::tempdir().unwrap();
21386 let root_path = root.path().canonicalize().unwrap();
21387 let hub = root_path.join("hub");
21388 let models = root_path.join("weights");
21389 std::fs::create_dir_all(&hub).unwrap();
21390 let mut engine = InferenceEngine::new(InferenceConfig {
21391 state_root: root_path.join("state"),
21392 models_dir: models.clone(),
21393 ..InferenceConfig::default()
21394 });
21395 let base = engine
21396 .unified_registry()
21397 .get("mlx/qwen3-4b:4bit")
21398 .unwrap()
21399 .clone();
21400 for (id, name) in [("disk/one", "One-MLX"), ("disk/two", "Two-MLX")] {
21401 let mut row = base.clone();
21402 row.id = id.into();
21403 row.name = name.into();
21404 row.source = ModelSource::Mlx {
21405 hf_repo: String::new(),
21406 hf_weight_file: None,
21407 };
21408 engine.unified_registry_mut().register_project_model(row);
21409 std::fs::create_dir_all(models.join(name)).unwrap();
21410 std::fs::write(models.join(name).join("model.safetensors"), b"user").unwrap();
21411 }
21412 let plan = engine.retire_plan_in("disk/one", &hub).unwrap();
21413 assert!(plan.also_retires.is_empty(), "{plan:?}");
21414 assert!(plan.projections.is_empty(), "{plan:?}");
21415 assert_eq!(plan.user_content, vec![models.join("One-MLX")]);
21416 assert_eq!(plan.freed_bytes, 0);
21417 }
21418
21419 static RETIRE_LOCKS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
21423
21424 #[cfg(unix)]
21428 fn plain_mlx_id(engine: &InferenceEngine) -> String {
21429 let protected = engine.protected_models(&engine.unified_registry(), true);
21430 [
21431 "mlx/qwen3-1.7b:3bit",
21432 "mlx/qwen3-0.6b:6bit",
21433 "mlx/qwen3-30b-a3b:4bit",
21434 ]
21435 .into_iter()
21436 .find(|id| !protected.contains_key(*id))
21437 .unwrap()
21438 .to_string()
21439 }
21440
21441 #[cfg(unix)]
21443 fn hub_repo(hub: &Path, repo: &str, files: &[(&str, usize)]) -> PathBuf {
21444 let dir = hf_cache::repo_dir_in(hub, repo);
21445 std::fs::create_dir_all(dir.join("snapshots/s")).unwrap();
21446 std::fs::create_dir_all(dir.join("blobs")).unwrap();
21447 for (i, (path, size)) in files.iter().enumerate() {
21448 let blob = dir.join("blobs").join(format!("b{i}"));
21449 std::fs::write(&blob, vec![0u8; *size]).unwrap();
21450 let entry = dir.join("snapshots/s").join(path);
21451 std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
21452 std::os::unix::fs::symlink(&blob, &entry).unwrap();
21453 }
21454 dir
21455 }
21456
21457 fn isolated_engine(root: &Path) -> (InferenceEngine, PathBuf, PathBuf) {
21458 let hub = root.join("hub");
21459 let models = root.join("weights");
21460 std::fs::create_dir_all(&hub).unwrap();
21461 std::fs::create_dir_all(&models).unwrap();
21462 let engine = InferenceEngine::new(InferenceConfig {
21463 state_root: root.join("state"),
21464 models_dir: models.clone(),
21465 ..InferenceConfig::default()
21466 });
21467 (engine, hub, models)
21468 }
21469
21470 #[tokio::test]
21476 async fn a_swapped_signed_catalog_is_one_generation_for_every_reader() {
21477 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21478 let root = tempfile::tempdir().unwrap();
21479 let (mut engine, _hub, _models) = isolated_engine(root.path());
21480 let builtin = crate::registry::builtin_catalog()
21481 .into_iter()
21482 .find(|m| !m.is_local())
21483 .expect("a remote builtin row");
21484 let mut added = builtin.clone();
21485 added.id = "signed/swapped-in".into();
21486 added.name = "swapped-in".into();
21487 added.tags.retain(|t| t != "builtin");
21488 let mut redefined = builtin.clone();
21489 redefined.context_length = builtin.context_length + 1;
21490 let mut kept = added.clone();
21492 kept.id = "project/registered-in-memory".into();
21493 kept.name = "registered-in-memory".into();
21494 assert!(engine
21495 .unified_registry_mut()
21496 .register_project_model(kept.clone()));
21497
21498 let before = engine.unified_registry();
21499 let (first, _key) = crate::catalog::signed_test_catalog(
21500 crate::catalog::CatalogDoc {
21501 revoked: Vec::new(),
21502 version: 2,
21503 models: vec![added.clone(), redefined],
21504 },
21505 7,
21506 );
21507 assert_eq!(engine.swap_signed_catalog(first), 2);
21509 assert!(engine.list_schemas().iter().any(|m| m.id == added.id));
21510 assert!(engine.unified_registry().get(&added.id).is_some());
21511 assert_eq!(
21512 engine.model_schema("swapped-in").map(|m| m.id),
21513 Some(added.id.clone())
21514 );
21515 assert_eq!(
21516 engine
21517 .unified_registry()
21518 .get(&builtin.id)
21519 .unwrap()
21520 .context_length,
21521 builtin.context_length,
21522 "a signed row never redefines a compiled builtin's identity"
21523 );
21524 assert!(
21525 before.get(&added.id).is_none(),
21526 "an older generation is unchanged"
21527 );
21528 assert!(engine.unified_registry().get(&kept.id).is_some());
21529
21530 let (second, _key) = crate::catalog::signed_test_catalog(
21531 crate::catalog::CatalogDoc {
21532 revoked: Vec::new(),
21533 version: 3,
21534 models: vec![],
21535 },
21536 7,
21537 );
21538 assert_eq!(engine.swap_signed_catalog(second), 0);
21539 assert!(
21540 engine.unified_registry().get(&added.id).is_none(),
21541 "withdrawn"
21542 );
21543 assert!(engine.model_schema("swapped-in").is_none());
21544 assert!(engine.unified_registry().get(&builtin.id).is_some());
21545 assert!(engine.unified_registry().get(&kept.id).is_some());
21546 }
21547
21548 #[tokio::test]
21551 async fn a_withdrawn_row_a_lane_defaults_to_is_kept_deprecated() {
21552 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21553 let root = tempfile::tempdir().unwrap();
21554 let (engine, _hub, _models) = isolated_engine(root.path());
21555 let mut row = crate::registry::builtin_catalog()
21556 .into_iter()
21557 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
21558 .unwrap();
21559 row.id = "signed/lane-default".into();
21560 row.name = "lane-default".into();
21561 let catalog = |models, version| {
21562 crate::catalog::signed_test_catalog(
21563 crate::catalog::CatalogDoc {
21564 revoked: Vec::new(),
21565 version,
21566 models,
21567 },
21568 13,
21569 )
21570 .0
21571 };
21572 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
21573 engine
21574 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
21575 .unwrap();
21576 engine.swap_signed_catalog(catalog(vec![], 3));
21577 let kept = engine
21578 .unified_registry()
21579 .get(&row.id)
21580 .cloned()
21581 .expect("kept");
21582 assert!(kept.deprecated);
21583 }
21584
21585 #[tokio::test]
21588 async fn a_row_withdrawn_mid_pull_is_kept_and_recorded() {
21589 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21590 let root = tempfile::tempdir().unwrap();
21591 let (engine, _hub, _models) = isolated_engine(root.path());
21592 let mut row = crate::registry::builtin_catalog()
21593 .into_iter()
21594 .find(|m| !m.is_local())
21595 .unwrap();
21596 row.id = "signed/mid-pull".into();
21597 row.name = "mid-pull".into();
21598 let catalog = |models, version| {
21599 crate::catalog::signed_test_catalog(
21600 crate::catalog::CatalogDoc {
21601 revoked: Vec::new(),
21602 version,
21603 models,
21604 },
21605 17,
21606 )
21607 .0
21608 };
21609 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
21610 let retained = catalog::retained_path(&root.path().join("state"));
21611 engine
21612 .with_pinned_registry(async {
21613 engine.swap_signed_catalog(catalog(vec![], 3));
21614 assert!(engine.current_registry().get(&row.id).is_none());
21615 assert!(!retained.exists(), "nothing was kept yet");
21616 engine.keep_pulled_row_if_withdrawn("mid-pull");
21617 })
21618 .await;
21619 let kept = engine
21620 .unified_registry()
21621 .get(&row.id)
21622 .cloned()
21623 .expect("kept");
21624 assert!(kept.deprecated);
21625 let recorded = std::fs::read_to_string(&retained).unwrap();
21626 assert!(recorded.contains("signed/mid-pull"), "{recorded}");
21627 }
21628
21629 #[tokio::test]
21633 async fn the_scheduled_refresh_loads_only_a_newer_verified_catalog() {
21634 use std::io::{Read, Write};
21635 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21636 let root = tempfile::tempdir().unwrap();
21637 let (engine, _hub, _models) = isolated_engine(root.path());
21638 let mut row = crate::registry::builtin_catalog()
21639 .into_iter()
21640 .find(|m| !m.is_local())
21641 .unwrap();
21642 row.id = "signed/served".into();
21643 row.name = "served".into();
21644 let (published, key) = crate::catalog::signed_test_catalog(
21645 crate::catalog::CatalogDoc {
21646 revoked: Vec::new(),
21647 version: 5,
21648 models: vec![row],
21649 },
21650 31,
21651 );
21652 let serving = std::sync::Arc::new(std::sync::Mutex::new(published.envelope()));
21653 let served = serving.clone();
21654 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
21655 let url = format!("http://{}/catalog.json", listener.local_addr().unwrap());
21656 std::thread::spawn(move || {
21657 for stream in listener.incoming() {
21658 let Ok(mut stream) = stream else { return };
21659 let mut request = [0u8; 2048];
21660 let n = stream.read(&mut request).unwrap_or(0);
21661 let request = String::from_utf8_lossy(&request[..n]);
21662 let envelope = served.lock().unwrap().clone();
21663 let body = if request.starts_with("GET /catalog.json.sig ") {
21664 envelope.signature.clone()
21665 } else {
21666 envelope.signed_body.clone()
21667 };
21668 let _ = write!(
21669 stream,
21670 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
21671 body.len(),
21672 body
21673 );
21674 }
21675 });
21676 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(Some(1)));
21677 assert!(engine.unified_registry().get("signed/served").is_some());
21678 assert_eq!(
21679 engine.refresh_catalog_from(&url, &key).await,
21680 Ok(None),
21681 "nothing newer is not an error"
21682 );
21683 let status_path = catalog::status_path(&engine.config.state_root);
21685 let status = catalog::load_status(&status_path);
21686 assert_eq!(status.source.as_deref(), Some(url.as_str()));
21687 assert_eq!(status.verified_version, Some(5));
21688 assert!(status.last_verified_at.is_some() && status.failing_since.is_none());
21689 let (_, other_key) = crate::catalog::signed_test_catalog(
21690 crate::catalog::CatalogDoc {
21691 revoked: Vec::new(),
21692 version: 1,
21693 models: vec![],
21694 },
21695 32,
21696 );
21697 assert!(engine.refresh_catalog_from(&url, &other_key).await.is_err());
21698 let status = catalog::load_status(&status_path);
21701 assert!(status.failing_since.is_some(), "{status:?}");
21702 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Rejected));
21703 assert!(matches!(
21704 engine.catalog_freshness(),
21705 catalog::CatalogFreshness::Current { .. }
21706 ));
21707
21708 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
21711 assert!(catalog::load_status(&status_path).failing_since.is_none());
21712 let (older, _) = crate::catalog::signed_test_catalog(
21713 crate::catalog::CatalogDoc {
21714 revoked: Vec::new(),
21715 version: 4,
21716 models: vec![],
21717 },
21718 31,
21719 );
21720 *serving.lock().unwrap() = older.envelope();
21721 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
21722 let status = catalog::load_status(&status_path);
21723 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Older));
21724 assert!(
21725 status
21726 .last_error
21727 .as_deref()
21728 .is_some_and(|e| e.contains("older than the cached v5")),
21729 "{status:?}"
21730 );
21731
21732 let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
21734 let gone = format!("http://{}/catalog.json", closed.local_addr().unwrap());
21735 drop(closed);
21736 assert!(engine.refresh_catalog_from(&gone, &key).await.is_err());
21737 assert_eq!(
21738 catalog::load_status(&status_path).last_failure,
21739 Some(catalog::CheckFailure::Unreachable)
21740 );
21741 }
21742
21743 #[cfg(unix)]
21747 #[tokio::test]
21748 async fn a_revoked_installed_model_is_unlinked_and_not_rediscovered() {
21749 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21750 let root = tempfile::tempdir().unwrap();
21751 let root_path = root.path().canonicalize().unwrap();
21752 let (engine, _hub, models) = isolated_engine(&root_path);
21753 let mut row = crate::registry::builtin_catalog()
21754 .into_iter()
21755 .find(|m| matches!(m.source, ModelSource::Local { .. }))
21756 .expect("a downloadable local row");
21757 row.id = "signed/revoked-local".into();
21758 row.name = "signed-revoked-local".into();
21759 row.tags.retain(|t| t != "builtin");
21760 let state = root_path.join("state");
21761 let path = catalog::cache_path(&state);
21762 let doc = |models, revoked, version| catalog::CatalogDoc {
21763 version,
21764 models,
21765 revoked,
21766 };
21767 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 61);
21768 engine
21769 .adopt_verified_catalog(&path, &v2, &key)
21770 .await
21771 .unwrap();
21772
21773 let shared = root_path.join("hub-snapshot");
21775 std::fs::create_dir_all(&shared).unwrap();
21776 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
21777 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
21778 std::os::unix::fs::symlink(&shared, models.join(&row.name)).unwrap();
21779 engine.adopt_model_into_car(&row.id).await.unwrap();
21780
21781 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 61);
21782 engine
21783 .adopt_verified_catalog(&path, &v3, &key)
21784 .await
21785 .unwrap();
21786 assert!(engine.unified_registry().get(&row.id).is_none());
21787 assert!(
21788 std::fs::symlink_metadata(models.join(&row.name)).is_err(),
21789 "CAR's linkage to revoked weights is gone"
21790 );
21791 assert!(
21792 shared.join("model.gguf").exists(),
21793 "the bytes are left to the orphan rules"
21794 );
21795 let restarted =
21796 crate::registry::UnifiedRegistry::new_with_state_root(state, models.clone());
21797 assert!(
21798 restarted.list().iter().all(|m| m.name != row.name),
21799 "nothing on disk brings it back"
21800 );
21801 }
21802
21803 #[cfg(unix)]
21809 #[tokio::test]
21810 async fn revoked_models_without_receipts_never_come_back_and_keep_their_bytes() {
21811 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21812 let root = tempfile::tempdir().unwrap();
21813 let root_path = root.path().canonicalize().unwrap();
21814 let (engine, _hub, models) = isolated_engine(&root_path);
21815 let template = crate::registry::builtin_catalog()
21816 .into_iter()
21817 .find(|m| matches!(m.source, ModelSource::Local { .. }))
21818 .expect("a downloadable local row");
21819 let signed = |id: &str, name: &str| {
21820 let mut m = template.clone();
21821 m.id = id.into();
21822 m.name = name.into();
21823 m.tags.retain(|t| t != "builtin");
21824 m
21825 };
21826 let lazy = signed("signed/lazy", "signed-lazy");
21827 let copied = signed("signed/copied", "signed-copied");
21828 let state = root_path.join("state");
21829 let path = catalog::cache_path(&state);
21830 let doc = |models, revoked, version| catalog::CatalogDoc {
21831 version,
21832 models,
21833 revoked,
21834 };
21835 let (v2, key) =
21836 catalog::signed_test_catalog(doc(vec![lazy.clone(), copied.clone()], vec![], 2), 62);
21837 engine
21838 .adopt_verified_catalog(&path, &v2, &key)
21839 .await
21840 .unwrap();
21841
21842 let shared = root_path.join("hub-snapshot");
21843 std::fs::create_dir_all(&shared).unwrap();
21844 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
21845 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
21846 std::os::unix::fs::symlink(&shared, models.join(&lazy.name)).unwrap();
21847 let copy = models.join(&copied.name);
21848 std::fs::create_dir_all(©).unwrap();
21849 std::fs::write(copy.join("model.gguf"), b"copied weights").unwrap();
21850 std::fs::write(copy.join("tokenizer.json"), b"{}").unwrap();
21851
21852 let (v3, _) = catalog::signed_test_catalog(
21853 doc(vec![], vec![lazy.id.clone(), copied.id.clone()], 3),
21854 62,
21855 );
21856 engine
21857 .adopt_verified_catalog(&path, &v3, &key)
21858 .await
21859 .unwrap();
21860 assert!(
21861 std::fs::symlink_metadata(models.join(&lazy.name)).is_err(),
21862 "links unlinked"
21863 );
21864 assert!(
21865 copy.join("model.gguf").exists(),
21866 "a copy's bytes are never deleted"
21867 );
21868 let restarted =
21869 crate::registry::UnifiedRegistry::new_with_state_root(state.clone(), models.clone());
21870 assert!(
21871 restarted
21872 .list()
21873 .iter()
21874 .all(|m| m.name != lazy.name && m.name != copied.name),
21875 "neither is rediscovered"
21876 );
21877
21878 let want = vec![retire::RevokedCopy {
21879 model_id: copied.id.clone(),
21880 path: copy.clone(),
21881 bytes: (b"copied weights".len() + b"{}".len()) as u64,
21882 }];
21883 assert_eq!(retire::revoked_copies(&state, &models), want);
21884 let hub = root_path.join("hub");
21885 assert_eq!(
21886 engine
21887 .portfolio_in(&hub, &HardwareInfo::detect())
21888 .await
21889 .revoked_copies,
21890 want
21891 );
21892 let opts = doctor::DoctorOptions {
21893 deep: false,
21894 repair: false,
21895 };
21896 let report = doctor::diagnose_at_isolated(&state, &models, &hub, &opts);
21897 assert_eq!(report.revoked_copies, want);
21898 let without = doctor::DoctorReport {
21899 revoked_copies: Vec::new(),
21900 ..report.clone()
21901 };
21902 assert_eq!(
21903 report.is_healthy(),
21904 without.is_healthy(),
21905 "informational only"
21906 );
21907
21908 let outside = root_path.join("outside");
21911 std::fs::create_dir_all(&outside).unwrap();
21912 std::fs::write(outside.join("private"), b"user data").unwrap();
21913 let names_path = catalog::revoked_names_path(&state);
21914 let mut names = catalog::load_revoked_names(&names_path);
21915 names.insert("../outside".into(), copied.id.clone());
21916 catalog::save_revoked_names(&names_path, &names).unwrap();
21917 assert_eq!(retire::revoked_copies(&state, &models), want);
21918
21919 std::fs::remove_dir_all(©).unwrap();
21921 std::fs::create_dir_all(©).unwrap();
21922 std::os::unix::fs::symlink(shared.join("model.gguf"), copy.join("model.gguf")).unwrap();
21923 assert!(retire::revoked_copies(&state, &models).is_empty());
21924 }
21925
21926 #[tokio::test]
21931 async fn agents_pinning_a_revoked_model_are_reported_by_id_or_name() {
21932 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21933 let root = tempfile::tempdir().unwrap();
21934 let root_path = root.path().canonicalize().unwrap();
21935 let (engine, _hub, _models) = isolated_engine(&root_path);
21936 let mut row = crate::registry::builtin_catalog()
21937 .into_iter()
21938 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
21939 .unwrap();
21940 row.id = "signed/revoked-pinned".into();
21941 row.name = "revoked-pinned".into();
21942 let registry = engine.current_registry();
21943 let builtin = crate::registry::builtin_catalog()
21944 .into_iter()
21945 .find(|m| {
21946 !m.is_local()
21947 && registry.find_by_name(&m.name).map(|f| f.id.as_str()) == Some(m.id.as_str())
21948 })
21949 .unwrap();
21950 let fine = crate::registry::builtin_catalog()
21951 .into_iter()
21952 .find(|m| m.id != builtin.id && !m.is_local())
21953 .unwrap();
21954 let path = catalog::cache_path(&root_path.join("state"));
21955 let doc = |models, revoked, version| catalog::CatalogDoc {
21956 version,
21957 models,
21958 revoked,
21959 };
21960 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 65);
21961 engine
21962 .adopt_verified_catalog(&path, &v2, &key)
21963 .await
21964 .unwrap();
21965 let pins = std::collections::BTreeMap::from([
21966 (row.id.clone(), vec!["by-id".to_string()]),
21967 (
21968 row.name.clone(),
21969 vec!["monthly".to_string(), "digest".to_string()],
21970 ),
21971 (builtin.name.clone(), vec!["weekly".to_string()]),
21972 (fine.id.clone(), vec!["untouched".to_string()]),
21973 ]);
21974 engine.set_external_pins(pins.clone());
21975 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
21976
21977 let (v3, _) = catalog::signed_test_catalog(
21978 doc(vec![], vec![row.id.clone(), builtin.id.clone()], 3),
21979 65,
21980 );
21981 engine
21982 .adopt_verified_catalog(&path, &v3, &key)
21983 .await
21984 .unwrap();
21985 let expect = |agents: &[&str], pin: &str, model_id: &str| portfolio::RevokedPin {
21986 pin: pin.to_string(),
21987 model_id: model_id.to_string(),
21988 agents: agents.iter().map(|a| a.to_string()).collect(),
21989 };
21990 let mut want = vec![
21991 expect(&["by-id"], &row.id, &row.id),
21992 expect(&["digest", "monthly"], &row.name, &row.id),
21993 expect(&["weekly"], &builtin.name, &builtin.id),
21994 ];
21995 want.sort_by(|a, b| a.pin.cmp(&b.pin));
21996 assert_eq!(engine.revoked_pins(&engine.current_registry()), want);
21997 let state = root_path.join("state");
22000 assert!(
22001 catalog::load_revoked_names(&catalog::revoked_models_path(&state))
22002 .contains_key(&row.name)
22003 );
22004 assert!(
22005 !catalog::load_revoked_names(&catalog::revoked_names_path(&state))
22006 .contains_key(&row.name)
22007 );
22008 let hub = root_path.join("hub");
22010 let hw = HardwareInfo::detect();
22011 assert_eq!(engine.portfolio_in(&hub, &hw).await.revoked_pins, want);
22012 assert_eq!(engine.concierge_status(false).await.revoked_pins, want);
22013
22014 drop(engine);
22017 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
22018 let (restarted, _hub, _models) = isolated_engine(&root_path);
22019 std::env::remove_var("CAR_CATALOG_PUBKEY");
22020 restarted.set_external_pins(pins);
22021 assert_eq!(restarted.revoked_pins(&restarted.current_registry()), want);
22022 }
22023
22024 #[tokio::test]
22028 async fn a_pin_by_a_name_a_live_row_holds_is_not_reported() {
22029 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22030 let root = tempfile::tempdir().unwrap();
22031 let root_path = root.path().canonicalize().unwrap();
22032 let (engine, _hub, _models) = isolated_engine(&root_path);
22033 let base = crate::registry::builtin_catalog()
22034 .into_iter()
22035 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22036 .unwrap();
22037 let named = |id: &str| {
22038 let mut row = base.clone();
22039 row.id = id.into();
22040 row.name = "shared-name".into();
22041 row
22042 };
22043 let (old, fresh) = (named("signed/old"), named("signed/fresh"));
22044 let path = catalog::cache_path(&root_path.join("state"));
22045 let doc = |models, revoked, version| catalog::CatalogDoc {
22046 version,
22047 models,
22048 revoked,
22049 };
22050 let (v2, key) = catalog::signed_test_catalog(doc(vec![old.clone()], vec![], 2), 66);
22051 engine
22052 .adopt_verified_catalog(&path, &v2, &key)
22053 .await
22054 .unwrap();
22055 engine.set_external_pins(std::collections::BTreeMap::from([(
22056 "shared-name".to_string(),
22057 vec!["monthly".to_string()],
22058 )]));
22059
22060 let (v3, _) =
22061 catalog::signed_test_catalog(doc(vec![fresh.clone()], vec![old.id.clone()], 3), 66);
22062 engine
22063 .adopt_verified_catalog(&path, &v3, &key)
22064 .await
22065 .unwrap();
22066 assert!(
22067 engine.revoked_pins(&engine.current_registry()).is_empty(),
22068 "the name belongs to a live row"
22069 );
22070
22071 let (v4, _) = catalog::signed_test_catalog(doc(vec![], vec![old.id.clone()], 4), 66);
22074 engine
22075 .adopt_verified_catalog(&path, &v4, &key)
22076 .await
22077 .unwrap();
22078 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
22079 }
22080
22081 #[tokio::test]
22084 async fn a_revoked_lane_default_is_cleared_and_recorded() {
22085 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22086 let root = tempfile::tempdir().unwrap();
22087 let root_path = root.path().canonicalize().unwrap();
22088 let (engine, _hub, _models) = isolated_engine(&root_path);
22089 let mut row = crate::registry::builtin_catalog()
22090 .into_iter()
22091 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22092 .unwrap();
22093 row.id = "signed/revoked-default".into();
22094 row.name = "revoked-default".into();
22095 let other = crate::registry::builtin_catalog()
22096 .into_iter()
22097 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Code))
22098 .unwrap();
22099 let path = catalog::cache_path(&root_path.join("state"));
22100 let doc = |models, revoked, version| catalog::CatalogDoc {
22101 version,
22102 models,
22103 revoked,
22104 };
22105 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 63);
22106 engine
22107 .adopt_verified_catalog(&path, &v2, &key)
22108 .await
22109 .unwrap();
22110 engine
22111 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22112 .unwrap();
22113 engine
22114 .set_lane_default(None, crate::intent::UseCase::Coding, &other.id)
22115 .unwrap();
22116
22117 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 63);
22118 engine
22119 .adopt_verified_catalog(&path, &v3, &key)
22120 .await
22121 .unwrap();
22122 assert_eq!(
22123 engine.lane_default(None, crate::intent::UseCase::Assistant),
22124 None
22125 );
22126 assert_eq!(
22127 engine.lane_default(None, crate::intent::UseCase::Coding),
22128 Some(other.id.clone()),
22129 "a lane on another model is untouched"
22130 );
22131 let cleared = engine
22132 .concierge_actions(0)
22133 .into_iter()
22134 .find(|a| a.kind == action_ledger::ConciergeActionKind::RevokedLaneCleared)
22135 .expect("the clearing is recorded");
22136 assert_eq!(cleared.model_id, row.id);
22137 assert!(cleared.detail.contains("revoked"), "{}", cleared.detail);
22138 let now = std::time::SystemTime::now()
22140 .duration_since(std::time::UNIX_EPOCH)
22141 .unwrap()
22142 .as_secs();
22143 let shown = engine
22144 .recent_concierge_actions(now)
22145 .into_iter()
22146 .find(|a| a.kind == "lane_cleared")
22147 .expect("the clearing is in recent actions");
22148 assert_eq!(shown.subject, row.id);
22149 assert_eq!(shown.undo, None);
22150 let refused = engine
22152 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22153 .unwrap_err();
22154 assert!(refused.contains("revoked"), "{refused}");
22155 }
22156
22157 #[tokio::test]
22160 async fn a_revocation_cached_before_a_restart_still_clears_its_lane() {
22161 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22162 let root = tempfile::tempdir().unwrap();
22163 let root_path = root.path().canonicalize().unwrap();
22164 let mut row = crate::registry::builtin_catalog()
22165 .into_iter()
22166 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22167 .unwrap();
22168 row.id = "signed/revoked-while-down".into();
22169 row.name = "revoked-while-down".into();
22170 let state = root_path.join("state");
22171 let path = catalog::cache_path(&state);
22172 let doc = |models, revoked, version| catalog::CatalogDoc {
22173 version,
22174 models,
22175 revoked,
22176 };
22177 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 64);
22178 {
22179 let (engine, _hub, _models) = isolated_engine(&root_path);
22180 engine
22181 .adopt_verified_catalog(&path, &v2, &key)
22182 .await
22183 .unwrap();
22184 engine
22185 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22186 .unwrap();
22187 }
22188 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 64);
22190 catalog::save_verified(&path, &v3).unwrap();
22191 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
22192 let (engine, _hub, _models) = isolated_engine(&root_path);
22193 std::env::remove_var("CAR_CATALOG_PUBKEY");
22194 assert_eq!(
22195 engine.lane_default(None, crate::intent::UseCase::Assistant),
22196 Some(row.id.clone())
22197 );
22198 engine.enforce_catalog_revocations().await;
22199 assert_eq!(
22200 engine.lane_default(None, crate::intent::UseCase::Assistant),
22201 None
22202 );
22203 }
22204
22205 #[tokio::test]
22208 async fn a_request_reads_one_registry_generation_across_a_swap() {
22209 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22210 let root = tempfile::tempdir().unwrap();
22211 let (engine, _hub, _models) = isolated_engine(root.path());
22212 let mut added = crate::registry::builtin_catalog()
22213 .into_iter()
22214 .find(|m| !m.is_local())
22215 .unwrap();
22216 added.id = "signed/mid-request".into();
22217 added.name = "mid-request".into();
22218 let (catalog, _key) = crate::catalog::signed_test_catalog(
22219 crate::catalog::CatalogDoc {
22220 revoked: Vec::new(),
22221 version: 2,
22222 models: vec![added],
22223 },
22224 9,
22225 );
22226 engine
22227 .with_pinned_registry(async {
22228 assert!(engine.model_schema("mid-request").is_none());
22229 engine.swap_signed_catalog(catalog);
22230 assert!(
22231 engine.model_schema("mid-request").is_none(),
22232 "the request's own generation is unchanged"
22233 );
22234 assert!(engine
22235 .current_registry()
22236 .get("signed/mid-request")
22237 .is_some());
22238 })
22239 .await;
22240 assert!(engine.model_schema("mid-request").is_some());
22241 }
22242
22243 #[tokio::test]
22246 async fn a_refreshed_catalog_loads_live_and_a_stale_one_changes_nothing() {
22247 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22248 let root = tempfile::tempdir().unwrap();
22249 let (engine, _hub, _models) = isolated_engine(root.path());
22250 let template = crate::registry::builtin_catalog()
22251 .into_iter()
22252 .find(|m| !m.is_local())
22253 .unwrap();
22254 let row = |id: &str| {
22255 let mut m = template.clone();
22256 m.id = id.into();
22257 m.name = id.replace('/', "-");
22258 m
22259 };
22260 let path = crate::catalog::cache_path(&root.path().join("state"));
22261 let (v3, key) = crate::catalog::signed_test_catalog(
22262 crate::catalog::CatalogDoc {
22263 revoked: Vec::new(),
22264 version: 3,
22265 models: vec![row("signed/v3")],
22266 },
22267 11,
22268 );
22269 let (v2, _) = crate::catalog::signed_test_catalog(
22270 crate::catalog::CatalogDoc {
22271 revoked: Vec::new(),
22272 version: 2,
22273 models: vec![row("signed/v2")],
22274 },
22275 11,
22276 );
22277 assert_eq!(engine.adopt_verified_catalog(&path, &v3, &key).await, Ok(1));
22278 assert!(engine.unified_registry().get("signed/v3").is_some());
22279 assert!(engine
22280 .adopt_verified_catalog(&path, &v2, &key)
22281 .await
22282 .is_err());
22283 assert!(engine.unified_registry().get("signed/v3").is_some());
22284 assert!(engine.unified_registry().get("signed/v2").is_none());
22285 }
22286
22287 #[cfg(unix)]
22291 #[tokio::test]
22292 async fn retiring_a_whole_repo_model_frees_it_and_leaves_nothing_behind() {
22293 let _serial = RETIRE_LOCKS.lock().await;
22294 let root = tempfile::tempdir().unwrap();
22295 let root_path = root.path().canonicalize().unwrap();
22296 let (engine, hub, models) = isolated_engine(&root_path);
22297 let id = plain_mlx_id(&engine);
22298 let id = id.as_str();
22299 let schema = engine.unified_registry().get(id).unwrap().clone();
22300 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22301 unreachable!()
22302 };
22303 let repo = hub_repo(
22304 &hub,
22305 hf_repo,
22306 &[("config.json", 10), ("model.safetensors", 500)],
22307 );
22308 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22309
22310 let dry = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22311 assert!(!dry.executed);
22312 assert!(repo.exists(), "a dry run deletes nothing");
22313
22314 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22315 assert!(outcome.executed, "{outcome:?}");
22316 assert!(outcome.errors.is_empty(), "{outcome:?}");
22317 assert!(!repo.exists());
22318 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
22319 let leftovers: Vec<_> = std::fs::read_dir(&hub)
22320 .unwrap()
22321 .filter_map(Result::ok)
22322 .collect();
22323 assert!(leftovers.is_empty(), "no quarantine left: {leftovers:?}");
22324 let journals = engine
22325 .model_management_store()
22326 .management_state_dir()
22327 .join("hub-removals");
22328 assert!(std::fs::read_dir(&journals).map_or(true, |mut d| d.next().is_none()));
22329 assert!(engine
22330 .concierge_actions(10)
22331 .iter()
22332 .any(|a| a.kind == action_ledger::ConciergeActionKind::Retire && a.model_id == id));
22333
22334 let again = engine.retire_plan_in(id, &hub).unwrap();
22335 assert!(
22336 again.deletions.is_empty() && again.projections.is_empty(),
22337 "{again:?}"
22338 );
22339 }
22340
22341 #[cfg(unix)]
22344 #[tokio::test]
22345 async fn retiring_a_gguf_model_leaves_the_users_checkout_alone() {
22346 let _serial = RETIRE_LOCKS.lock().await;
22347 let root = tempfile::tempdir().unwrap();
22348 let root_path = root.path().canonicalize().unwrap();
22349 let (engine, hub, _models) = isolated_engine(&root_path);
22350 let schema = engine
22351 .unified_registry()
22352 .all()
22353 .find(|m| m.id == "qwen/qwen3-8b:q4_k_m")
22354 .unwrap()
22355 .clone();
22356 let ModelSource::Local {
22357 hf_repo,
22358 hf_filename,
22359 tokenizer_repo,
22360 } = &schema.source
22361 else {
22362 unreachable!()
22363 };
22364 let quant = hub_repo(&hub, hf_repo, &[(hf_filename.as_str(), 300)]);
22365 let base = hub_repo(
22366 &hub,
22367 tokenizer_repo,
22368 &[("tokenizer.json", 7), ("model.safetensors", 900)],
22369 );
22370
22371 let outcome = engine
22372 .retire_model_in(&schema.id, &hub, false, None)
22373 .await
22374 .unwrap();
22375 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22376 assert!(std::fs::symlink_metadata(quant.join("snapshots/s").join(hf_filename)).is_err());
22377 assert!(!quant.join("blobs/b0").exists());
22378 assert!(base.join("snapshots/s/tokenizer.json").exists());
22379 assert!(base.join("snapshots/s/model.safetensors").exists());
22380 assert_eq!(std::fs::metadata(base.join("blobs/b1")).unwrap().len(), 900);
22381 }
22382
22383 #[cfg(unix)]
22387 #[tokio::test]
22388 async fn retirement_refuses_a_changed_plan_and_a_running_download() {
22389 let _serial = RETIRE_LOCKS.lock().await;
22390 let root = tempfile::tempdir().unwrap();
22391 let root_path = root.path().canonicalize().unwrap();
22392 let (engine, hub, _models) = isolated_engine(&root_path);
22393 let id = plain_mlx_id(&engine);
22394 let id = id.as_str();
22395 let ModelSource::Mlx { hf_repo, .. } =
22396 engine.unified_registry().get(id).unwrap().source.clone()
22397 else {
22398 unreachable!()
22399 };
22400 let repo = hub_repo(&hub, &hf_repo, &[("w", 10)]);
22401 let reviewed = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22402 assert!(!reviewed.plan.digest.is_empty());
22403
22404 std::fs::write(repo.join("blobs/late"), vec![0u8; 99]).unwrap();
22405 let changed = engine
22406 .retire_model_in(id, &hub, false, Some(&reviewed.plan.digest))
22407 .await
22408 .unwrap();
22409 assert!(!changed.executed);
22410 assert!(changed
22411 .plan
22412 .refusals
22413 .contains(&retire::RetireRefusal::StateChanged));
22414 assert!(repo.join("blobs/late").exists() && repo.join("blobs/b0").exists());
22415
22416 let busy = crate::download::try_acquire_model_lock(id).unwrap();
22417 let blocked = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22418 assert!(!blocked.executed);
22419 assert!(blocked
22420 .plan
22421 .refusals
22422 .contains(&retire::RetireRefusal::Downloading));
22423 assert!(repo.exists());
22424 drop(busy);
22425
22426 let current = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22427 let done = engine
22428 .retire_model_in(id, &hub, false, Some(¤t.plan.digest))
22429 .await
22430 .unwrap();
22431 assert!(done.executed && done.errors.is_empty(), "{done:?}");
22432 assert!(!repo.exists());
22433 }
22434
22435 #[cfg(unix)]
22438 #[tokio::test]
22439 async fn a_shared_auxiliary_does_not_make_twins() {
22440 let _serial = RETIRE_LOCKS.lock().await;
22441 let root = tempfile::tempdir().unwrap();
22442 let root_path = root.path().canonicalize().unwrap();
22443 let (mut engine, hub, _models) = isolated_engine(&root_path);
22444 let ltx = engine
22445 .unified_registry()
22446 .get("mlx/ltx-2.3:q4")
22447 .unwrap()
22448 .clone();
22449 let ModelSource::Mlx {
22450 hf_repo: ltx_repo, ..
22451 } = ltx.source.clone()
22452 else {
22453 unreachable!()
22454 };
22455 let mut second = ltx.clone();
22456 second.id = "user/ltx-q8".into();
22457 second.name = "ltx-q8".into();
22458 second.source = ModelSource::Mlx {
22459 hf_repo: "someone/ltx-q8".into(),
22460 hf_weight_file: None,
22461 };
22462 engine.unified_registry_mut().register_project_model(second);
22463 let ltx_dir = hub_repo(&hub, <x_repo, &[("config.json", 5), ("w", 50)]);
22464 hub_repo(&hub, "someone/ltx-q8", &[("config.json", 5), ("w", 70)]);
22465 hub_repo(
22466 &hub,
22467 crate::registry::LTX_TEXT_ENCODER,
22468 &[("config.json", 5), ("w", 90)],
22469 );
22470 let plan = engine.retire_plan_in("mlx/ltx-2.3:q4", &hub).unwrap();
22471 assert!(plan.also_retires.is_empty(), "{plan:?}");
22472 assert!(
22473 plan.deletions.iter().all(|d| !matches!(
22474 d,
22475 retire::HubDeletion::Repo { repo, .. } if repo == "someone/ltx-q8"
22476 )),
22477 "{plan:?}"
22478 );
22479 assert!(ltx_dir.exists());
22480 }
22481
22482 #[cfg(unix)]
22484 #[tokio::test]
22485 async fn twins_sharing_a_repo_retire_together() {
22486 let _serial = RETIRE_LOCKS.lock().await;
22487 let root = tempfile::tempdir().unwrap();
22488 let root_path = root.path().canonicalize().unwrap();
22489 let (engine, hub, _models) = isolated_engine(&root_path);
22490 let id = "mlx/gemma-4-12b-it:4bit";
22491 let ModelSource::Mlx { hf_repo, .. } =
22492 engine.unified_registry().get(id).unwrap().source.clone()
22493 else {
22494 unreachable!()
22495 };
22496 let repo = hub_repo(&hub, &hf_repo, &[("config.json", 5), ("w", 50)]);
22497 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22498 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22499 assert_eq!(
22500 outcome.plan.also_retires,
22501 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
22502 );
22503 assert!(!repo.exists());
22504 let retired: Vec<String> = engine
22505 .concierge_actions(10)
22506 .into_iter()
22507 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
22508 .map(|a| a.model_id)
22509 .collect();
22510 assert!(retired.contains(&id.to_string()), "{retired:?}");
22511 assert!(
22512 retired.contains(&"vllm-mlx/gemma-4-12b-it".to_string()),
22513 "{retired:?}"
22514 );
22515
22516 let again = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22518 assert!(!again.executed);
22519 assert!(again
22520 .plan
22521 .refusals
22522 .contains(&retire::RetireRefusal::NothingToRetire));
22523 assert_eq!(
22524 engine
22525 .concierge_actions(10)
22526 .iter()
22527 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
22528 .count(),
22529 2
22530 );
22531 }
22532
22533 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22538 #[tokio::test]
22539 async fn a_receipted_projection_retires_and_the_row_stays_fetchable() {
22540 let _serial = RETIRE_LOCKS.lock().await;
22541 let root = tempfile::tempdir().unwrap();
22542 let root_path = root.path().canonicalize().unwrap();
22543 let (engine, hub, models) = isolated_engine(&root_path);
22544 let id = plain_mlx_id(&engine);
22545 let id = id.as_str();
22546 let schema = engine.unified_registry().get(id).unwrap().clone();
22547 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22548 unreachable!()
22549 };
22550 let repo = hub_repo(
22551 &hub,
22552 hf_repo,
22553 &[("config.json", 10), ("model.safetensors", 500)],
22554 );
22555 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22556 engine.adopt_model_into_car(id).await.unwrap();
22557 let store = engine.model_management_store();
22558 assert!(store.receipt_path(id).exists());
22559
22560 let plan = engine.retire_plan_in(id, &hub).unwrap();
22561 assert!(
22562 plan.projections
22563 .iter()
22564 .any(|p| p.receipt_for.as_deref() == Some(id)),
22565 "{plan:?}"
22566 );
22567 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22568 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22569 assert!(!repo.exists());
22570 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
22571 assert!(!store.receipt_path(id).exists());
22572 assert!(!store.tombstone_path(id).exists(), "retired, not disabled");
22573 assert!(store.car_enabled(id).unwrap());
22574 let intents = store.management_state_dir().join("retiring");
22575 assert!(std::fs::read_dir(&intents).map_or(true, |mut d| d.next().is_none()));
22576 }
22577
22578 #[cfg(unix)]
22580 #[test]
22581 fn a_projection_holding_a_real_file_is_not_removed() {
22582 let root = tempfile::tempdir().unwrap();
22583 let models = root.path().canonicalize().unwrap().join("weights");
22584 let projection = models.join("Some-MLX");
22585 std::fs::create_dir_all(&projection).unwrap();
22586 std::os::unix::fs::symlink("/nowhere", projection.join("link")).unwrap();
22587 std::fs::write(projection.join("notes.txt"), b"mine").unwrap();
22588 assert!(retire::remove_links_only_tree(&projection, &models).is_err());
22589 assert!(std::fs::symlink_metadata(projection.join("link")).is_ok());
22590 assert!(projection.join("notes.txt").exists());
22591 let elsewhere = root.path().canonicalize().unwrap().join("elsewhere");
22593 std::fs::create_dir_all(&elsewhere).unwrap();
22594 assert!(retire::remove_links_only_tree(&elsewhere, &models).is_err());
22595 assert!(elsewhere.exists());
22596 }
22597
22598 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22603 #[tokio::test]
22604 async fn portfolio_keeps_defaults_and_waits_for_evidence() {
22605 let _serial = RETIRE_LOCKS.lock().await;
22606 let root = tempfile::tempdir().unwrap();
22607 let root_path = root.path().canonicalize().unwrap();
22608 let (engine, hub, models) = isolated_engine(&root_path);
22609 let hw = HardwareInfo {
22610 total_ram_mb: 64 * 1024,
22611 ..HardwareInfo::detect()
22612 };
22613 let protected = engine.protected_models(&engine.unified_registry(), true);
22616 let idle_id = [
22617 "mlx/qwen3-30b-a3b:4bit",
22618 "mlx/qwen3-1.7b:3bit",
22619 "mlx/qwen3-0.6b:6bit",
22620 ]
22621 .into_iter()
22622 .find(|id| !protected.contains_key(*id))
22623 .unwrap();
22624 let mut installed = Vec::new();
22625 for id in [idle_id, "mlx/qwen3-8b:4bit"] {
22626 let schema = engine.unified_registry().get(id).unwrap().clone();
22627 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22628 unreachable!()
22629 };
22630 let repo = hub_repo(
22631 &hub,
22632 hf_repo,
22633 &[("config.json", 10), ("model.safetensors", 500)],
22634 );
22635 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
22636 .unwrap();
22637 installed.push(id);
22638 }
22639 engine.lane_defaults_cache.write().unwrap().set(
22640 None,
22641 crate::intent::UseCase::Assistant,
22642 "mlx/qwen3-8b:4bit".to_string(),
22643 1,
22644 );
22645
22646 let fresh = engine.portfolio_in(&hub, &hw).await;
22647 let ids: Vec<&str> = fresh.models.iter().map(|m| m.model_id.as_str()).collect();
22648 for id in &installed {
22649 assert!(ids.contains(id), "{ids:?}");
22650 }
22651 let verdict = |p: &portfolio::Portfolio, id: &str| {
22652 p.models.iter().find(|m| m.model_id == id).unwrap().clone()
22653 };
22654 assert_eq!(
22655 verdict(&fresh, "mlx/qwen3-8b:4bit").verdict,
22656 portfolio::Verdict::Protected
22657 );
22658 let small = verdict(&fresh, idle_id);
22659 assert_eq!(small.verdict, portfolio::Verdict::Unknown, "{small:?}");
22660 assert_eq!(small.action, portfolio::PortfolioAction::Keep);
22661
22662 engine
22664 .model_management_store()
22665 .set_usage_tracking_since_for_test(1);
22666 let aged = engine.portfolio_in(&hub, &hw).await;
22667 let small = verdict(&aged, idle_id);
22668 assert_eq!(small.verdict, portfolio::Verdict::Idle, "{small:?}");
22669 assert_eq!(small.action, portfolio::PortfolioAction::Retire);
22670 assert!(aged.reclaimable_bytes >= 510, "{aged:?}");
22671 let digest = small.retire.unwrap().digest;
22672 let done = engine
22673 .retire_model_in(idle_id, &hub, false, Some(&digest))
22674 .await
22675 .unwrap();
22676 assert!(done.executed && done.errors.is_empty(), "{done:?}");
22677 assert_eq!(
22678 verdict(&aged, "mlx/qwen3-8b:4bit").verdict,
22679 portfolio::Verdict::Protected
22680 );
22681
22682 let pinned = plain_mlx_id(&engine);
22685 engine.set_external_pins(std::collections::BTreeMap::from([(
22686 pinned.clone(),
22687 vec!["monthly".to_string()],
22688 )]));
22689 let plan = engine.retire_plan_in(&pinned, &hub).unwrap();
22690 assert!(
22691 plan.refusals.iter().any(|r| matches!(
22692 r,
22693 retire::RetireRefusal::Protected { why } if why.contains("monthly")
22694 )),
22695 "{plan:?}"
22696 );
22697 engine.set_external_pins(Default::default());
22698
22699 let tool = engine
22701 .retire_plan_in(LOCAL_TOOL_MODEL_PULL_ID, &hub)
22702 .unwrap();
22703 assert!(
22704 tool.refusals
22705 .iter()
22706 .any(|r| matches!(r, retire::RetireRefusal::Protected { .. })),
22707 "{tool:?}"
22708 );
22709 }
22710
22711 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22718 #[tokio::test]
22719 async fn maintenance_retires_under_pressure_and_respects_keep_and_churn() {
22720 use crate::update_prefs::UpdatePolicy;
22721 let _serial = RETIRE_LOCKS.lock().await;
22722 let root = tempfile::tempdir().unwrap();
22723 let root_path = root.path().canonicalize().unwrap();
22724 let (engine, hub, _models) = isolated_engine(&root_path);
22725 let hw = HardwareInfo {
22726 total_ram_mb: 64 * 1024,
22727 ..HardwareInfo::detect()
22728 };
22729 let id = plain_mlx_id(&engine);
22730 let ModelSource::Mlx { hf_repo, .. } =
22731 engine.unified_registry().get(&id).unwrap().source.clone()
22732 else {
22733 unreachable!()
22734 };
22735 let layout = || hub_repo(&hub, &hf_repo, &[("config.json", 10), ("w", 500)]);
22736 let repo = layout();
22737 engine
22738 .model_management_store()
22739 .set_usage_tracking_since_for_test(1);
22740 let gb = 1_000_000_000;
22741 let roomy = Some(maintenance::DiskSpace {
22742 free_bytes: 500 * gb,
22743 total_bytes: 1_000 * gb,
22744 });
22745 let tight = Some(maintenance::DiskSpace {
22746 free_bytes: 50 * gb,
22747 total_bytes: 1_000 * gb,
22748 });
22749 let none = crate::nudge::NudgeState::default();
22750
22751 let calm = engine
22752 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22753 .await;
22754 assert!(calm.retired.is_none(), "{calm:?}");
22755 assert!(calm.plan.held.iter().any(|h| h.model_id == id), "{calm:?}");
22756 assert!(repo.exists());
22757
22758 let notify = engine
22759 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, tight)
22760 .await;
22761 assert!(notify.retired.is_none());
22762 assert!(
22763 notify.plan.suggest.iter().any(|d| d.model_id == id),
22764 "{notify:?}"
22765 );
22766 assert!(repo.exists(), "Notify never acts");
22767
22768 let off = engine
22769 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Off, &none, tight)
22770 .await;
22771 assert_eq!(off.plan, maintenance::MaintenancePlan::default());
22772
22773 let mut kept = crate::nudge::NudgeState::default();
22774 kept.dismiss_labeled(
22775 maintenance::keep_key(&id),
22776 crate::concierge::DismissReason::Wrong,
22777 1,
22778 );
22779 let held = engine
22780 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &kept, tight)
22781 .await;
22782 assert!(held.retired.is_none());
22783 assert!(repo.exists(), "kept");
22784
22785 let pressed = engine
22786 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22787 .await;
22788 let retired = pressed.retired.as_ref().expect("retired under pressure");
22789 assert_eq!(retired.model_id, id);
22790 assert!(!repo.exists());
22791 let record = engine.portfolio_in(&hub, &hw).await.recent_actions;
22793 let entry = record
22794 .iter()
22795 .find(|a| a.kind == "retire" && a.subject == id)
22796 .expect("the retirement is on record");
22797 assert_eq!(
22798 entry.undo.as_deref(),
22799 Some(format!("car models pull {id}").as_str())
22800 );
22801 assert!(!entry.interrupted);
22802 let kinds: Vec<_> = engine
22803 .concierge_actions(10)
22804 .into_iter()
22805 .filter(|a| a.model_id == id)
22806 .map(|a| a.kind)
22807 .collect();
22808 assert!(kinds.contains(&action_ledger::ConciergeActionKind::RetireStarted));
22809 assert!(kinds.contains(&action_ledger::ConciergeActionKind::Retire));
22810
22811 let repo = layout();
22813 let again = engine
22814 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22815 .await;
22816 assert!(again.retired.is_none(), "{again:?}");
22817 assert!(again
22818 .plan
22819 .held
22820 .iter()
22821 .any(|h| h.model_id == id && h.why.contains("fetched back")));
22822 assert!(repo.exists());
22823
22824 let old = repo.join("blobs/aaaa.incomplete");
22826 let fresh = repo.join("blobs/bbbb.incomplete");
22827 std::fs::write(&old, b"half").unwrap();
22828 std::fs::write(&fresh, b"half").unwrap();
22829 let two_days_ago =
22830 std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 24 * 60 * 60);
22831 std::fs::File::options()
22832 .write(true)
22833 .open(&old)
22834 .unwrap()
22835 .set_modified(two_days_ago)
22836 .unwrap();
22837 let swept = engine
22838 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22839 .await;
22840 assert_eq!(swept.partials_discarded, vec![old.clone()], "{swept:?}");
22841 assert!(!old.exists() && fresh.exists());
22842 }
22843
22844 #[cfg(unix)]
22847 #[tokio::test]
22848 async fn maintenance_retires_an_orphan_car_fetched_only_under_pressure() {
22849 use crate::update_prefs::UpdatePolicy;
22850 let _serial = RETIRE_LOCKS.lock().await;
22851 let root = tempfile::tempdir().unwrap();
22852 let root_path = root.path().canonicalize().unwrap();
22853 let (engine, hub, models) = isolated_engine(&root_path);
22854 let hw = HardwareInfo::detect();
22855 let orphan = hub_repo(&hub, "dropped/model", &[("config.json", 10), ("w", 900)]);
22856 let theirs = hub_repo(
22857 &hub,
22858 "someone/checkout",
22859 &[("tokenizer.json", 5), ("w", 50)],
22860 );
22861 let fetched = |repo: &str, files: &[&str]| -> Vec<retire::Fetched> {
22862 files
22863 .iter()
22864 .map(|f| (repo.to_string(), f.to_string(), Some("s".to_string())))
22865 .collect()
22866 };
22867 retire::record_download(
22868 &root_path.join("state"),
22869 &hub,
22870 "old/row",
22871 &fetched("dropped/model", &["config.json", "w"]),
22872 );
22873 retire::record_download(
22874 &root_path.join("state"),
22875 &hub,
22876 "old/row",
22877 &fetched("someone/checkout", &["tokenizer.json"]),
22878 );
22879 let gb = 1_000_000_000;
22880 let none = crate::nudge::NudgeState::default();
22881 let roomy = Some(maintenance::DiskSpace {
22882 free_bytes: 500 * gb,
22883 total_bytes: 1_000 * gb,
22884 });
22885 let calm = engine
22886 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22887 .await;
22888 assert!(calm.orphan_retired.is_none());
22889 assert!(orphan.exists());
22890
22891 let tight = Some(maintenance::DiskSpace {
22892 free_bytes: 10 * gb,
22893 total_bytes: 1_000 * gb,
22894 });
22895 let link = models.join("Dropped-Model");
22898 std::os::unix::fs::symlink(orphan.join("snapshots/s"), &link).unwrap();
22899 let blocked = engine
22900 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22901 .await;
22902 assert!(blocked.orphan_retired.is_none(), "{blocked:?}");
22903 assert!(
22904 blocked
22905 .errors
22906 .iter()
22907 .any(|e| e.contains("resolves into it")),
22908 "{blocked:?}"
22909 );
22910 assert!(orphan.exists());
22911 std::fs::remove_file(&link).unwrap();
22912
22913 let pressed = engine
22914 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22915 .await;
22916 assert_eq!(
22917 pressed.orphan_retired.as_ref().map(|o| o.repo.as_str()),
22918 Some("dropped/model"),
22919 "{pressed:?}"
22920 );
22921 assert!(!orphan.exists());
22922 assert!(theirs.exists(), "a user's checkout is never touched");
22923 let again = engine
22924 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22925 .await;
22926 assert!(again.orphan_retired.is_none(), "{again:?}");
22927 assert!(theirs.exists());
22928 }
22929
22930 #[tokio::test]
22934 async fn maintenance_raises_an_unchosen_memory_policy_on_evidence() {
22935 use crate::resource_policy::{
22936 FileResourcePolicyRepository, ResourcePolicy, ResourcePolicyRepository, ResourceProfile,
22937 };
22938 use crate::update_prefs::UpdatePolicy;
22939 let _serial = RETIRE_LOCKS.lock().await;
22940 let root = tempfile::tempdir().unwrap();
22941 let root_path = root.path().canonicalize().unwrap();
22942 let hub = root_path.join("hub");
22943 std::fs::create_dir_all(&hub).unwrap();
22944 let engine = InferenceEngine::new(InferenceConfig {
22945 state_root: root_path.join("state"),
22946 models_dir: root_path.join("weights"),
22947 ..InferenceConfig::default()
22948 });
22949 let hw = HardwareInfo {
22950 total_ram_mb: 64 * 1024,
22951 ..HardwareInfo::detect()
22952 };
22953 let big = engine
22955 .unified_registry()
22956 .all()
22957 .find(|m| {
22958 m.is_local()
22959 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::everyday())).fit
22960 == recommend::ModelFitStatus::TooBig
22961 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::local_focused())).fit
22962 == recommend::ModelFitStatus::Fits
22963 })
22964 .expect("a row only Local-focused admits")
22965 .id
22966 .clone();
22967 let ledger = engine
22968 .config
22969 .state_models_dir()
22970 .join("outcome_ledger.jsonl");
22971 std::fs::create_dir_all(ledger.parent().unwrap()).unwrap();
22972 let now = std::time::SystemTime::now()
22973 .duration_since(std::time::UNIX_EPOCH)
22974 .unwrap()
22975 .as_secs();
22976 let lines: String = (0..3)
22977 .map(|i| {
22978 format!(
22979 "{}\n",
22980 serde_json::json!({
22981 "trace_id": format!("t{i}"), "model_id": big, "task": "generate",
22982 "routing_reason": "explicit model", "latency_ms": 0,
22983 "input_tokens": 0, "output_tokens": 0, "error": "refused for memory",
22984 "attribution": "resource_blocked",
22985 "timestamp": now - 60 - i * 24 * 60 * 60
22986 })
22987 )
22988 })
22989 .collect();
22990 std::fs::write(&ledger, lines).unwrap();
22991 let none = crate::nudge::NudgeState::default();
22992 let repository = FileResourcePolicyRepository::new(root_path.join("state"));
22993
22994 let unobserved = engine
22996 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
22997 .await;
22998 assert!(unobserved.resource_policy.is_none(), "{unobserved:?}");
22999
23000 engine
23001 .model_management_store()
23002 .set_usage_tracking_since_for_test(now - 4 * 24 * 60 * 60);
23003 let notify = engine
23004 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, None)
23005 .await;
23006 let suggested = notify.resource_policy.expect("suggested under Notify");
23007 assert!(!suggested.applied);
23008 assert!(!repository.path().exists(), "Notify saves nothing");
23009
23010 let auto = engine
23011 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23012 .await;
23013 assert!(
23014 auto.resource_policy.as_ref().is_some_and(|p| p.applied),
23015 "{auto:?}"
23016 );
23017 assert_eq!(repository.load().unwrap(), ResourcePolicy::local_focused());
23018 assert_eq!(
23019 repository.concierge_marker(),
23020 Some(ResourcePolicy::local_focused())
23021 );
23022 assert_eq!(
23023 engine.active_local_resource_policy().policy.profile,
23024 ResourceProfile::LocalFocused
23025 );
23026
23027 let small = engine
23030 .unified_registry()
23031 .all()
23032 .find(|m| {
23033 m.is_local()
23034 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::everyday())).fit
23035 == recommend::ModelFitStatus::Fits
23036 })
23037 .unwrap()
23038 .id
23039 .clone();
23040 let later = std::time::SystemTime::now()
23041 .duration_since(std::time::UNIX_EPOCH)
23042 .unwrap()
23043 .as_secs()
23044 + 1;
23045 let mut text = std::fs::read_to_string(&ledger).unwrap();
23046 for i in 0..3 {
23047 text.push_str(&format!(
23048 "{}\n",
23049 serde_json::json!({
23050 "trace_id": format!("live{i}"), "model_id": small, "task": "generate",
23051 "routing_reason": "explicit model", "latency_ms": 0,
23052 "input_tokens": 0, "output_tokens": 0, "error": "not enough memory free",
23053 "attribution": "resource_blocked", "timestamp": later
23054 })
23055 ));
23056 }
23057 std::fs::write(&ledger, text).unwrap();
23058 let reverted = engine
23059 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23060 .await;
23061 let undo = reverted.resource_policy.expect("reverted");
23062 assert!(
23063 undo.applied && undo.to == ResourceProfile::Everyday,
23064 "{undo:?}"
23065 );
23066 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
23067 assert!(repository.concierge_choice().unwrap().reverted);
23068 let settled = engine
23069 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23070 .await;
23071 assert!(settled.resource_policy.is_none(), "{settled:?}");
23072
23073 repository.save(&ResourcePolicy::everyday()).unwrap();
23075 repository.clear_concierge_marker().unwrap();
23076 engine.apply_local_resource_policy(ResourcePolicy::everyday());
23077 let chosen = engine
23078 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23079 .await;
23080 assert!(chosen.resource_policy.is_none(), "{chosen:?}");
23081 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
23082 }
23083
23084 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
23090 #[tokio::test]
23091 async fn a_curated_upgrade_switches_the_lane_that_used_the_old_model() {
23092 let _serial = RETIRE_LOCKS.lock().await;
23093 let root = tempfile::tempdir().unwrap();
23094 let root_path = root.path().canonicalize().unwrap();
23095 let (engine, hub, models) = isolated_engine(&root_path);
23096 let from = plain_mlx_id(&engine);
23097 let to = "mlx/qwen3-8b:4bit".to_string();
23098 for id in [&from, &to] {
23099 let schema = engine.unified_registry().get(id).unwrap().clone();
23100 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23101 unreachable!()
23102 };
23103 let repo = hub_repo(
23104 &hub,
23105 hf_repo,
23106 &[("config.json", 10), ("model.safetensors", 500)],
23107 );
23108 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
23109 .unwrap();
23110 }
23111 let finding = |source| crate::upgrade::UpgradeFinding {
23112 from_id: from.clone(),
23113 from_name: from.clone(),
23114 to_id: to.clone(),
23115 to_name: to.clone(),
23116 reason: "newer".into(),
23117 trust_tier: crate::schema::TrustTier::Curated,
23118 source,
23119 target_pullable: true,
23120 };
23121 let now = 1;
23122 let none = crate::nudge::NudgeState::default();
23123
23124 let idle = engine
23126 .apply_one_curated_upgrade_from(
23127 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23128 &hub,
23129 &none,
23130 now,
23131 )
23132 .await
23133 .unwrap();
23134 assert!(idle.is_none());
23135
23136 engine
23137 .set_lane_default(None, crate::intent::UseCase::Assistant, &from)
23138 .unwrap();
23139 let community = engine
23141 .apply_one_curated_upgrade_from(
23142 vec![finding(crate::upgrade::UpgradeSource::Upstream)],
23143 &hub,
23144 &none,
23145 now,
23146 )
23147 .await
23148 .unwrap();
23149 assert!(community.is_none());
23150 let mut dismissed = crate::nudge::NudgeState::default();
23151 dismissed.dismiss(&format!("{from}=>{to}"));
23152 assert!(engine
23153 .apply_one_curated_upgrade_from(
23154 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23155 &hub,
23156 &dismissed,
23157 now,
23158 )
23159 .await
23160 .unwrap()
23161 .is_none());
23162
23163 let upgraded = engine
23164 .apply_one_curated_upgrade_from(
23165 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23166 &hub,
23167 &none,
23168 now,
23169 )
23170 .await
23171 .unwrap()
23172 .expect("upgraded");
23173 assert_eq!(upgraded.lanes, vec!["assistant".to_string()]);
23174 assert_eq!(
23175 engine
23176 .lane_default(None, crate::intent::UseCase::Assistant)
23177 .as_deref(),
23178 Some(to.as_str())
23179 );
23180 let record = engine.recent_concierge_actions(now);
23181 let entry = record
23182 .iter()
23183 .find(|a| a.kind == "upgrade")
23184 .expect("on record");
23185 assert_eq!(
23186 entry.undo.as_deref(),
23187 Some("car concierge rollback assistant")
23188 );
23189 assert_eq!(entry.use_case.as_deref(), Some("assistant"));
23190 assert_eq!(entry.prior.as_deref(), Some(from.as_str()));
23191 }
23192
23193 #[tokio::test]
23196 async fn concierge_ask_without_a_question_states_the_facts_without_a_model() {
23197 let _serial = RETIRE_LOCKS.lock().await;
23198 let root = tempfile::tempdir().unwrap();
23199 let engine = InferenceEngine::new(InferenceConfig {
23200 state_root: root.path().join("state"),
23201 models_dir: root.path().join("weights"),
23202 ..InferenceConfig::default()
23203 });
23204 let answer = engine.concierge_ask(None).await.unwrap();
23205 assert!(answer.model_used.is_none());
23206 assert_eq!(answer.answer, answer.summary);
23207 assert!(
23208 answer.summary.contains("Installed local models:"),
23209 "{}",
23210 answer.summary
23211 );
23212 assert!(answer.summary.contains("Memory policy for local models:"));
23213 }
23214
23215 #[cfg(unix)]
23217 #[tokio::test]
23218 async fn a_refused_retirement_deletes_nothing() {
23219 let _serial = RETIRE_LOCKS.lock().await;
23220 let root = tempfile::tempdir().unwrap();
23221 let root_path = root.path().canonicalize().unwrap();
23222 let (engine, hub, _models) = isolated_engine(&root_path);
23223 let id = "mlx/qwen3-4b:4bit";
23224 let schema = engine.unified_registry().get(id).unwrap().clone();
23225 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23226 unreachable!()
23227 };
23228 let repo = hub_repo(&hub, hf_repo, &[("w", 10)]);
23229 engine.lane_defaults_cache.write().unwrap().set(
23230 None,
23231 crate::intent::UseCase::Assistant,
23232 id.to_string(),
23233 1,
23234 );
23235 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23236 assert!(!outcome.executed);
23237 assert!(repo.exists());
23238 }
23239
23240 #[test]
23243 fn retire_plan_refuses_a_model_stored_outside_the_hub() {
23244 let root = tempfile::tempdir().unwrap();
23245 let engine = InferenceEngine::new(InferenceConfig {
23246 state_root: root.path().join("state"),
23247 models_dir: root.path().join("weights"),
23248 ..InferenceConfig::default()
23249 });
23250 let whisper = engine
23251 .unified_registry()
23252 .all()
23253 .find(|m| matches!(m.source, ModelSource::WhisperCpp { .. }))
23254 .expect("a whisper.cpp row")
23255 .id
23256 .clone();
23257 let plan = engine.retire_plan_in(&whisper, root.path()).unwrap();
23258 assert!(
23259 matches!(
23260 &plan.refusals[..],
23261 [retire::RetireRefusal::Unsupported { .. }]
23262 ),
23263 "{plan:?}"
23264 );
23265 }
23266
23267 #[test]
23269 fn downloads_are_recorded_per_hub() {
23270 let root = tempfile::tempdir().unwrap();
23271 let fetched = [(
23272 "renamed/repo".to_string(),
23273 "model.safetensors".to_string(),
23274 Some("r1".to_string()),
23275 )];
23276 let hub = root.path().join("hub");
23277 retire::record_download(root.path(), &hub, "old/row", &fetched);
23278 let recorded = retire::recorded_files(root.path(), &hub);
23279 assert_eq!(
23280 recorded.get("renamed/repo"),
23281 Some(&retire::Owned::Files(std::collections::BTreeSet::from([
23282 retire::OwnedFile {
23283 path: "model.safetensors".into(),
23284 revision: Some("r1".into()),
23285 }
23286 ])))
23287 );
23288 assert!(retire::recorded_files(root.path(), Path::new("/elsewhere")).is_empty());
23289 }
23290
23291 #[cfg(unix)]
23296 #[tokio::test]
23297 async fn a_failed_ledger_write_leaves_the_profiles_untouched_and_retries_once() {
23298 use std::os::unix::fs::PermissionsExt;
23299 let root = tempfile::tempdir().unwrap();
23300 let state = root.path().join("state");
23301 let models = state.join("models");
23302 std::fs::create_dir_all(&models).unwrap();
23303 let ledger = models.join("outcome_ledger.jsonl");
23304 let real = "mlx/qwen3-4b:4bit";
23305 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
23306 let mut seed = OutcomeTracker::new();
23307 seed.import_profiles(profiles);
23308 seed.save_to_file(&models.join("outcome_profiles.json"))
23309 .unwrap();
23310 let engine = InferenceEngine::new(InferenceConfig {
23311 state_root: state.clone(),
23312 models_dir: root.path().join("weights"),
23313 ..InferenceConfig::default()
23314 });
23315 let before = profile_json(&*engine.outcome_tracker().read().await, real);
23316 let dirty_before = engine.outcome_tracker().read().await.is_dirty();
23317 assert!(
23318 engine
23319 .outcome_tracker()
23320 .read()
23321 .await
23322 .profile(real)
23323 .unwrap()
23324 .fail_count
23325 > 1
23326 );
23327
23328 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o555)).unwrap();
23330 let failed = engine.reconcile_outcome_ledger().await;
23331 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o755)).unwrap();
23332 assert!(failed.is_err(), "the write was refused");
23333 {
23334 let tracker = engine.outcome_tracker();
23335 let tracker = tracker.read().await;
23336 assert_eq!(profile_json(&tracker, real), before, "profiles untouched");
23337 assert_eq!(
23338 tracker.is_dirty(),
23339 dirty_before,
23340 "the failed reconcile left nothing new for the periodic save to persist"
23341 );
23342 }
23343
23344 engine.reconcile_outcome_ledger().await.unwrap();
23346 engine.reconcile_outcome_ledger().await.unwrap();
23347 let p = engine
23348 .outcome_tracker()
23349 .read()
23350 .await
23351 .profile(real)
23352 .unwrap()
23353 .clone();
23354 assert_eq!(
23355 p.fail_count, 1,
23356 "only the genuine failure remains, taken back once"
23357 );
23358 }
23359
23360 #[test]
23363 fn engine_construction_never_rewrites_the_ledger() {
23364 let root = tempfile::tempdir().unwrap();
23365 let state = root.path().join("state");
23366 let models = state.join("models");
23367 std::fs::create_dir_all(&models).unwrap();
23368 let ledger = models.join("outcome_ledger.jsonl");
23369 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
23370 let before = std::fs::read(&ledger).unwrap();
23371 let _engine = InferenceEngine::new(InferenceConfig {
23372 state_root: state,
23373 models_dir: root.path().join("weights"),
23374 ..InferenceConfig::default()
23375 });
23376 assert_eq!(std::fs::read(&ledger).unwrap(), before);
23377 }
23378
23379 #[test]
23380 fn dispatch_outcome_guards_keep_real_failures_and_existing_rejections_distinct() {
23381 let model_id = "test/model";
23382
23383 let real_failure = InferenceError::InferenceFailed("provider returned HTTP 500".into());
23384 let mut tracker = OutcomeTracker::new();
23385 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
23386 record_dispatch_failure(&mut tracker, &trace, &real_failure);
23387 let profile = tracker.profile(model_id).expect("real failure profile");
23388 assert_eq!(profile.success_count, 0);
23389 assert_eq!(profile.fail_count, 1);
23390 let receipt = tracker.drain_ledger().pop().unwrap();
23391 assert_eq!(receipt.success, Some(false));
23392 assert!(error_counts_against_circuit_breaker(&real_failure));
23393 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
23394 breakers.record_failure(model_id);
23395 assert_eq!(
23396 breakers.state(model_id),
23397 Some(crate::routing_ext::CircuitState::Open)
23398 );
23399
23400 for rejection in [
23401 InferenceError::GatewayUnconfigured {
23402 provider: "parslee".into(),
23403 namespace: "parslee/openrouter/".into(),
23404 status: 503,
23405 message: "not configured".into(),
23406 },
23407 InferenceError::ContentRefused {
23408 provider: "parslee".into(),
23409 kind: Some("invalid_request_error".into()),
23410 code: Some("content_policy_violation".into()),
23411 message: "content refused".into(),
23412 },
23413 ] {
23414 let mut tracker = OutcomeTracker::new();
23415 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
23416 record_dispatch_failure(&mut tracker, &trace, &rejection);
23417 assert!(
23418 tracker.profile(model_id).is_none(),
23419 "existing rejection changed model profile: {rejection}"
23420 );
23421 let receipt = tracker.drain_ledger().pop().unwrap();
23422 assert_eq!(receipt.success, None, "{rejection}");
23423 assert_eq!(receipt.quality, None, "{rejection}");
23424 assert!(!error_counts_against_circuit_breaker(&rejection));
23425 }
23426 }
23427
23428 #[tokio::test(flavor = "current_thread")]
23441 async fn openrouter_out_of_credits_does_not_degrade_the_model() {
23442 use wiremock::matchers::{method, path};
23443 use wiremock::{Mock, MockServer, ResponseTemplate};
23444
23445 let _credential_scope = crate::openrouter::test_credential_scope();
23446 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
23447 let server = MockServer::start().await;
23448 Mock::given(method("POST"))
23449 .and(path("/v1/chat/completions"))
23450 .respond_with(
23451 ResponseTemplate::new(402)
23452 .set_body_string(r#"{"error":{"code":402,"message":"Insufficient credits"}}"#),
23453 )
23454 .mount(&server)
23455 .await;
23456
23457 let tmp = TempDir::new().unwrap();
23458 let models_dir = tmp.path().join("models");
23459 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
23460 let mut schema = crate::openrouter::curated_schemas()
23461 .into_iter()
23462 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
23463 .unwrap();
23464 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
23465 *endpoint = server.uri();
23466 }
23467 let model_id = schema.id.clone();
23468 engine.unified_registry_mut().register_project_model(schema);
23469
23470 let err = engine
23471 .generate_tracked(GenerateRequest {
23472 prompt: "bill me".into(),
23473 model: Some(model_id.clone()),
23474 params: GenerateParams {
23475 strict_model: true,
23476 ..Default::default()
23477 },
23478 ..Default::default()
23479 })
23480 .await
23481 .expect_err("402 must not succeed");
23482
23483 match &err {
23484 InferenceError::ProviderAccount {
23485 provider, status, ..
23486 } => {
23487 assert_eq!(status, &402);
23488 assert_eq!(provider, "openrouter");
23489 }
23490 other => panic!("expected ProviderAccount, got {other:?}"),
23491 }
23492 assert!(
23493 !error_counts_against_circuit_breaker(&err),
23494 "an account rejection must not feed the per-model breaker"
23495 );
23496
23497 let tracker_handle = engine.outcome_tracker();
23499 let tracker = tracker_handle.read().await;
23500 let profile = tracker.profile(&model_id).cloned();
23501 assert!(
23502 profile.as_ref().is_none_or(|p| p.fail_count == 0),
23503 "account rejection degraded the model profile: {profile:?}"
23504 );
23505 drop(tracker);
23506
23507 let ledger = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
23511 let entry = ledger
23512 .iter()
23513 .find(|e| e.model_id == model_id)
23514 .expect("the attempt must still produce a receipt");
23515 assert!(
23516 entry.success.is_none() && entry.quality.is_none(),
23517 "receipt must record the attempt without a verdict: {entry:?}"
23518 );
23519
23520 assert!(
23522 engine
23523 .adaptive_router
23524 .circuit_breakers
23525 .lock()
23526 .unwrap()
23527 .state(&model_id)
23528 .is_none(),
23529 "account rejection must not create breaker state for the model"
23530 );
23531 }
23532
23533 #[tokio::test(flavor = "current_thread")]
23534 async fn openrouter_stream_account_error_is_unattributed_and_never_completes_successfully() {
23535 use wiremock::matchers::{method, path};
23536 use wiremock::{Mock, MockServer, ResponseTemplate};
23537
23538 let _credential_scope = crate::openrouter::test_credential_scope();
23539 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
23540 let server = MockServer::start().await;
23541 Mock::given(method("POST"))
23542 .and(path("/v1/chat/completions"))
23543 .respond_with(ResponseTemplate::new(200).set_body_raw(
23544 "data: {\"error\":{\"code\":402,\"message\":\"private balance details\"}}\n\n",
23545 "text/event-stream",
23546 ))
23547 .mount(&server)
23548 .await;
23549
23550 let tmp = TempDir::new().unwrap();
23551 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23552 let mut schema = crate::openrouter::curated_schemas()
23553 .into_iter()
23554 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
23555 .unwrap();
23556 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
23557 *endpoint = server.uri();
23558 }
23559 let model_id = schema.id.clone();
23560 engine.unified_registry_mut().register_project_model(schema);
23561
23562 let mut handle = engine
23563 .generate_tracked_stream(GenerateRequest {
23564 prompt: "fail after headers".into(),
23565 model: Some(model_id.clone()),
23566 ..Default::default()
23567 })
23568 .await
23569 .unwrap();
23570 let mut events = Vec::new();
23571 while let Some(event) = handle.events.recv().await {
23572 events.push(event);
23573 }
23574 assert!(matches!(
23575 events.as_slice(),
23576 [StreamEvent::Error(message)] if message == "OpenRouter account is out of credits"
23577 ));
23578 assert!(!events
23579 .iter()
23580 .any(|event| matches!(event, StreamEvent::Done { .. })));
23581
23582 let mut receipts = Vec::new();
23585 for _ in 0..50 {
23586 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
23587 if !receipts.is_empty() {
23588 break;
23589 }
23590 tokio::task::yield_now().await;
23591 }
23592 let [receipt] = receipts.as_slice() else {
23593 panic!("expected one receipt, got {receipts:?}");
23594 };
23595 assert_eq!(receipt.success, None);
23596 assert_eq!(
23597 receipt.attribution,
23598 Some(crate::outcome::UnattributedReason::Account)
23599 );
23600 assert!(
23601 engine
23602 .outcome_tracker()
23603 .read()
23604 .await
23605 .profile(&model_id)
23606 .is_none(),
23607 "an account rejection must not touch the model's profile"
23608 );
23609 }
23610
23611 #[tokio::test(flavor = "current_thread")]
23614 async fn tracked_stream_dropped_by_the_caller_is_cancelled_not_a_failure() {
23615 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23616 let tmp = TempDir::new().unwrap();
23617 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23618 pin_test_local_admission_memory(&mut engine);
23619 let model_id = install_small_local_fixture(&engine);
23620 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
23621
23622 let stream = engine
23623 .generate_tracked_stream(GenerateRequest {
23624 prompt: "interrupted".into(),
23625 model: Some(model_id.clone()),
23626 params: GenerateParams {
23627 strict_model: true,
23628 ..Default::default()
23629 },
23630 ..Default::default()
23631 })
23632 .await
23633 .unwrap();
23634 drop(stream);
23635
23636 let mut receipts = Vec::new();
23637 for _ in 0..200 {
23638 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
23639 if !receipts.is_empty() {
23640 break;
23641 }
23642 tokio::task::yield_now().await;
23643 }
23644 crate::offload::set_local_offload(None);
23645 let [receipt] = receipts.as_slice() else {
23646 panic!("expected one receipt, got {receipts:?}");
23647 };
23648 assert_eq!(receipt.success, None);
23649 assert_eq!(
23650 receipt.attribution,
23651 Some(crate::outcome::UnattributedReason::Cancelled)
23652 );
23653 assert!(engine
23654 .outcome_tracker()
23655 .read()
23656 .await
23657 .profile(&model_id)
23658 .is_none_or(|p| p.fail_count == 0));
23659 }
23660
23661 #[test]
23665 fn an_apple_guardrail_decline_does_not_end_the_fallback_chain() {
23666 let refusal = |provider: &str| InferenceError::ContentRefused {
23667 provider: provider.into(),
23668 kind: Some("guardrailViolation".into()),
23669 code: None,
23670 message: "declined".into(),
23671 };
23672 assert!(!error_ends_fallback_chain(&refusal("apple")));
23673 assert!(error_ends_fallback_chain(&refusal("parslee")));
23674 assert!(!error_counts_against_circuit_breaker(&refusal("apple")));
23675 }
23676
23677 #[tokio::test(flavor = "current_thread")]
23678 async fn tracked_stream_without_done_records_failure_not_success() {
23679 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23680 let tmp = TempDir::new().unwrap();
23681 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23682 pin_test_local_admission_memory(&mut engine);
23683 let model_id = install_small_local_fixture(&engine);
23684 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: false })));
23685
23686 let mut stream = engine
23687 .generate_tracked_stream(GenerateRequest {
23688 prompt: "must not count as success".into(),
23689 model: Some(model_id.clone()),
23690 params: GenerateParams {
23691 strict_model: true,
23692 ..Default::default()
23693 },
23694 ..Default::default()
23695 })
23696 .await
23697 .unwrap();
23698 while stream.events.recv().await.is_some() {}
23699 crate::offload::set_local_offload(None);
23700
23701 for _ in 0..50 {
23702 if engine
23703 .outcome_tracker()
23704 .read()
23705 .await
23706 .profile(&model_id)
23707 .is_some_and(|profile| profile.fail_count == 1)
23708 {
23709 break;
23710 }
23711 tokio::task::yield_now().await;
23712 }
23713 let profile = engine
23714 .outcome_tracker()
23715 .read()
23716 .await
23717 .profile(&model_id)
23718 .cloned()
23719 .unwrap();
23720 assert_eq!(profile.fail_count, 1);
23721 assert_eq!(profile.success_count, 0);
23722 }
23723
23724 #[tokio::test(flavor = "current_thread")]
23725 async fn google_vertex_terminal_matrix_records_only_deliberate_finishes_as_success() {
23726 use wiremock::matchers::{method, path};
23727 use wiremock::{Mock, MockServer, ResponseTemplate};
23728
23729 let _provider_env = crate::openrouter::test_environment_scope_async().await;
23730 let _env = ENV_MUTEX.lock().await;
23731 unsafe { std::env::set_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY", "matrix-key") };
23732
23733 for (protocol, reason, should_succeed) in [
23734 (schema::ApiProtocol::Google, "STOP", true),
23735 (schema::ApiProtocol::Google, "SAFETY", false),
23736 (schema::ApiProtocol::VertexAi, "MAX_TOKENS", true),
23737 (
23738 schema::ApiProtocol::VertexAi,
23739 "MALFORMED_FUNCTION_CALL",
23740 false,
23741 ),
23742 ] {
23743 let server = MockServer::start().await;
23744 let expected_path = match protocol {
23745 schema::ApiProtocol::Google => "/v1beta/models/gemini-test:streamGenerateContent",
23746 schema::ApiProtocol::VertexAi => {
23747 "/publishers/google/models/gemini-test:streamGenerateContent"
23748 }
23749 _ => unreachable!(),
23750 };
23751 Mock::given(method("POST"))
23752 .and(path(expected_path))
23753 .respond_with(ResponseTemplate::new(200).set_body_raw(
23754 format!(
23755 "data: {{\"candidates\":[{{\"content\":{{\"parts\":[{{\"text\":\"matrix\"}}]}},\"finishReason\":\"{reason}\"}}]}}\n\n"
23756 ),
23757 "text/event-stream",
23758 ))
23759 .mount(&server)
23760 .await;
23761
23762 let id = format!("test/{protocol:?}-{reason}");
23763 let tmp = TempDir::new().unwrap();
23764 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23765 engine.register_model(remote_stream_fixture_schema(
23766 &id,
23767 server.uri(),
23768 protocol,
23769 "CAR_GOOGLE_OUTCOME_MATRIX_KEY",
23770 ));
23771 let mut stream = engine
23772 .generate_tracked_stream(GenerateRequest {
23773 prompt: "matrix".into(),
23774 model: Some(id.clone()),
23775 params: GenerateParams {
23776 strict_model: true,
23777 ..Default::default()
23778 },
23779 ..Default::default()
23780 })
23781 .await
23782 .unwrap();
23783 let mut events = Vec::new();
23784 while let Some(event) = stream.events.recv().await {
23785 events.push(event);
23786 }
23787 for _ in 0..50 {
23788 if engine
23789 .outcome_tracker()
23790 .read()
23791 .await
23792 .profile(&id)
23793 .is_some_and(|profile| profile.total_calls == 1)
23794 {
23795 break;
23796 }
23797 tokio::task::yield_now().await;
23798 }
23799 let profile = engine
23800 .outcome_tracker()
23801 .read()
23802 .await
23803 .profile(&id)
23804 .cloned()
23805 .unwrap();
23806 assert_eq!(
23807 (profile.success_count, profile.fail_count),
23808 if should_succeed { (1, 0) } else { (0, 1) },
23809 "{protocol:?}/{reason}: {events:?}"
23810 );
23811 }
23812
23813 unsafe { std::env::remove_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY") };
23814 }
23815
23816 #[tokio::test(flavor = "current_thread")]
23817 async fn remote_primary_stream_setup_failure_falls_back_to_installed_local_dispatch() {
23818 use wiremock::matchers::{method, path};
23819 use wiremock::{Mock, MockServer, ResponseTemplate};
23820
23821 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23822 let _env = ENV_MUTEX.lock().await;
23823 let server = MockServer::start().await;
23824 Mock::given(method("POST"))
23825 .and(path("/v1/chat/completions"))
23826 .respond_with(ResponseTemplate::new(503))
23827 .mount(&server)
23828 .await;
23829 unsafe { std::env::set_var("CAR_STREAM_FALLBACK_TEST_KEY", "fixture") };
23830
23831 let tmp = TempDir::new().unwrap();
23832 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23833 pin_test_local_admission_memory(&mut engine);
23834 let _local_id = install_small_local_fixture(&engine);
23835 let remote_id = "test/remote-primary";
23836 engine.register_model(ModelSchema {
23837 id: remote_id.into(),
23838 name: "remote-primary".into(),
23839 provider: "test".into(),
23840 family: "test".into(),
23841 version: "1".into(),
23842 capabilities: vec![ModelCapability::Generate],
23843 context_length: 8_192,
23844 max_output_tokens: Some(1_024),
23845 param_count: String::new(),
23846 quantization: None,
23847 performance: Default::default(),
23848 cost: Default::default(),
23849 source: ModelSource::RemoteApi {
23850 endpoint: server.uri(),
23851 api_key_env: "CAR_STREAM_FALLBACK_TEST_KEY".into(),
23852 api_key_envs: vec![],
23853 api_version: None,
23854 protocol: schema::ApiProtocol::OpenAiCompat,
23855 },
23856 tags: vec![],
23857 supported_params: vec![],
23858 public_benchmarks: vec![],
23859 trust_tier: TrustTier::Community,
23860 deprecated: false,
23861 available: true,
23862 weights_ready: true,
23863 });
23864 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
23865
23866 let mut stream = engine
23867 .generate_tracked_stream(GenerateRequest {
23868 prompt: "fall back".into(),
23869 model: Some(remote_id.into()),
23870 ..Default::default()
23871 })
23872 .await
23873 .expect("compatible installed local model should be dispatched");
23874 assert_ne!(stream.model_used, remote_id);
23875 assert!(
23876 engine
23877 .unified_registry()
23878 .get(&stream.model_used)
23879 .is_some_and(ModelSchema::is_local),
23880 "fallback must stay on a compatible local model: {}",
23881 stream.model_used
23882 );
23883 let mut saw_done = false;
23884 while let Some(event) = stream.events.recv().await {
23885 saw_done |= matches!(event, StreamEvent::Done { .. });
23886 }
23887 assert!(saw_done);
23888
23889 crate::offload::set_local_offload(None);
23890 unsafe { std::env::remove_var("CAR_STREAM_FALLBACK_TEST_KEY") };
23891 }
23892
23893 #[tokio::test(flavor = "current_thread")]
23894 async fn exact_model_id_rejects_display_name_without_dispatch() {
23895 let tmp = TempDir::new().unwrap();
23896 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23897 let mut first = remote_stream_fixture_schema(
23898 "test/ambiguous-a:1",
23899 "http://127.0.0.1:9".into(),
23900 schema::ApiProtocol::OpenAiCompat,
23901 "CAR_AMBIGUOUS_EXACT_KEY",
23902 );
23903 first.name = "Shared Alias".into();
23904 let mut second = first.clone();
23905 second.id = "test/ambiguous-b:1".into();
23906 engine.register_model(first);
23907 engine.register_model(second);
23908 let mut request = GenerateRequest {
23909 prompt: "must not dispatch".into(),
23910 ..Default::default()
23911 };
23912 pin_exact_model_id(&mut request, "Shared Alias".into()).unwrap();
23913
23914 let error = engine
23915 .generate_tracked(request)
23916 .await
23917 .expect_err("an exact-id pin must never resolve a display name");
23918 assert!(matches!(error, InferenceError::ModelNotFound(_)));
23919 }
23920
23921 #[tokio::test(flavor = "current_thread")]
23922 async fn exact_openai_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
23923 let _env = ENV_MUTEX.lock().await;
23924 assert_remote_model_identity_contract(
23925 schema::ApiProtocol::OpenAiCompat,
23926 "openai/newsroom-gpt-5.5-2026-04-23:test",
23927 "newsroom-gpt-5.5-2026-04-23-test",
23928 "CAR_OPENAI_IDENTITY_TEST_KEY",
23929 )
23930 .await;
23931 }
23932
23933 #[tokio::test(flavor = "current_thread")]
23934 async fn exact_anthropic_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
23935 let _env = ENV_MUTEX.lock().await;
23936 assert_remote_model_identity_contract(
23937 schema::ApiProtocol::Anthropic,
23938 "anthropic/newsroom-claude-opus-4-8:test",
23939 "newsroom-claude-opus-4-8-test",
23940 "CAR_ANTHROPIC_IDENTITY_TEST_KEY",
23941 )
23942 .await;
23943 }
23944
23945 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
23946 #[tokio::test(flavor = "current_thread")]
23947 async fn exact_model_id_nonstream_bypasses_mlx_equivalent_substitution() {
23948 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23949 let tmp = TempDir::new().unwrap();
23950 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23951 let exact_id = install_exact_pin_equivalent_fixture(&engine);
23952 let offload = Arc::new(ExactPinCaptureOffload::new());
23953 crate::offload::set_local_offload(Some(offload.clone()));
23954
23955 let mut request = GenerateRequest {
23956 prompt: "use only the immutable row I selected".into(),
23957 ..Default::default()
23958 };
23959 pin_exact_model_id(&mut request, exact_id.clone()).unwrap();
23960 let error = engine.generate_tracked(request).await.unwrap_err();
23961
23962 assert!(offload.dispatched_models().is_empty());
23963 assert!(error.to_string().contains(&exact_id), "{error}");
23964 assert!(
23965 error
23966 .to_string()
23967 .contains("MLX-equivalent substitution is disabled for exact pins"),
23968 "{error}"
23969 );
23970 crate::offload::set_local_offload(None);
23971 }
23972
23973 #[tokio::test(flavor = "current_thread")]
23974 async fn catalog_identity_mismatch_rejects_before_provider_dispatch() {
23975 use wiremock::matchers::{method, path};
23976 use wiremock::{Mock, MockServer, ResponseTemplate};
23977
23978 let _env = ENV_MUTEX.lock().await;
23979 let server = MockServer::start().await;
23980 Mock::given(method("POST"))
23981 .and(path("/v1/chat/completions"))
23982 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
23983 "choices": [{"message": {"content": "must not run"}, "finish_reason": "stop"}],
23984 })))
23985 .mount(&server)
23986 .await;
23987 unsafe { std::env::set_var("CAR_PRECONDITION_TEST_KEY", "fixture") };
23988
23989 let tmp = TempDir::new().unwrap();
23990 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23991 let model_id = "test/catalog-precondition:1";
23992 engine.register_model(remote_stream_fixture_schema(
23993 model_id,
23994 server.uri(),
23995 schema::ApiProtocol::OpenAiCompat,
23996 "CAR_PRECONDITION_TEST_KEY",
23997 ));
23998 let snapshot = engine.catalog_snapshot().unwrap();
23999 let row_digest = snapshot
24000 .model_by_exact_id(model_id)
24001 .unwrap()
24002 .row_digest
24003 .clone();
24004
24005 for (expected_row_digest, expected_catalog_revision) in [
24006 (
24007 Some("0".repeat(64)),
24008 Some(snapshot.catalog_revision.clone()),
24009 ),
24010 (Some(row_digest.clone()), Some("f".repeat(64))),
24011 ] {
24012 let mut request = GenerateRequest {
24013 prompt: "must not dispatch".into(),
24014 expected_row_digest,
24015 expected_catalog_revision,
24016 ..Default::default()
24017 };
24018 pin_exact_model_id(&mut request, model_id.into()).unwrap();
24019 let error = engine.generate_tracked(request).await.unwrap_err();
24020 assert!(matches!(
24021 error,
24022 InferenceError::CatalogPreconditionMismatch { .. }
24023 ));
24024 }
24025 assert!(
24026 server.received_requests().await.unwrap().is_empty(),
24027 "catalog identity mismatches must fail before provider dispatch"
24028 );
24029
24030 unsafe { std::env::remove_var("CAR_PRECONDITION_TEST_KEY") };
24031 }
24032
24033 #[tokio::test(flavor = "current_thread")]
24034 async fn exact_nonstream_identity_survives_thinking_retry() {
24035 use std::sync::atomic::{AtomicUsize, Ordering};
24036 use wiremock::matchers::{method, path};
24037 use wiremock::{Mock, MockServer, ResponseTemplate};
24038
24039 let _env = ENV_MUTEX.lock().await;
24040 let server = MockServer::start().await;
24041 let calls = Arc::new(AtomicUsize::new(0));
24042 let response_calls = calls.clone();
24043 Mock::given(method("POST"))
24044 .and(path("/v1/chat/completions"))
24045 .respond_with(move |_request: &wiremock::Request| {
24046 let attempt = response_calls.fetch_add(1, Ordering::SeqCst);
24047 let content = if attempt == 0 { "" } else { "recovered" };
24048 ResponseTemplate::new(200).set_body_json(serde_json::json!({
24049 "choices": [{
24050 "message": {"content": content},
24051 "finish_reason": if attempt == 0 { "length" } else { "stop" },
24052 }],
24053 "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
24054 }))
24055 })
24056 .mount(&server)
24057 .await;
24058 unsafe { std::env::set_var("CAR_IDENTITY_RETRY_KEY", "fixture") };
24059
24060 let tmp = TempDir::new().unwrap();
24061 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24062 let model_id = "test/identity-retry:1";
24063 engine.register_model(remote_stream_fixture_schema(
24064 model_id,
24065 server.uri(),
24066 schema::ApiProtocol::OpenAiCompat,
24067 "CAR_IDENTITY_RETRY_KEY",
24068 ));
24069 let snapshot = engine.catalog_snapshot().unwrap();
24070 let expected_row = snapshot.model_by_exact_id(model_id).unwrap();
24071 let mut request = GenerateRequest {
24072 prompt: "retry once".into(),
24073 expected_row_digest: Some(expected_row.row_digest.clone()),
24074 expected_catalog_revision: Some(snapshot.catalog_revision.clone()),
24075 ..Default::default()
24076 };
24077 pin_exact_model_id(&mut request, model_id.into()).unwrap();
24078
24079 let result = engine.generate_tracked(request).await.unwrap();
24080 assert_eq!(calls.load(Ordering::SeqCst), 2, "must exercise retry path");
24081 assert_eq!(result.stop_reason.as_deref(), Some("thinking_recovered"));
24082 assert_eq!(
24083 result.model_identity.requested_model_id.as_deref(),
24084 Some(model_id)
24085 );
24086 assert_eq!(result.model_identity.resolved_model_id, model_id);
24087 assert_eq!(result.model_identity.row_digest, expected_row.row_digest);
24088 assert_eq!(
24089 result.model_identity.catalog_revision,
24090 snapshot.catalog_revision
24091 );
24092
24093 unsafe { std::env::remove_var("CAR_IDENTITY_RETRY_KEY") };
24094 }
24095
24096 #[tokio::test(flavor = "current_thread")]
24097 async fn managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns() {
24098 if !crate::run_in_isolated_test_process(
24099 "tests::managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns",
24100 "CAR_MANAGED_REASONING_ROUNDTRIP_CHILD",
24101 ) {
24102 return;
24103 }
24104 let _home = crate::openrouter::StateRootScope::new();
24105 use wiremock::matchers::{header, method, path};
24106 use wiremock::{Mock, MockServer, ResponseTemplate};
24107
24108 let _provider_env = crate::openrouter::test_environment_scope_async().await;
24109 let _env = ENV_MUTEX.lock().await;
24110 let bearer = "managed-reasoning-roundtrip-bearer";
24111
24112 let server = MockServer::start().await;
24113 unsafe {
24114 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
24115 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
24116 }
24117 Mock::given(method("GET"))
24118 .and(path("/api/v1/organizations/me"))
24119 .and(header("authorization", format!("Bearer {bearer}")))
24120 .respond_with(
24121 ResponseTemplate::new(200)
24122 .set_body_json(serde_json::json!({"organizationId": "org-roundtrip"})),
24123 )
24124 .mount(&server)
24125 .await;
24126 Mock::given(method("GET"))
24127 .and(path("/connect/session"))
24128 .respond_with(
24129 ResponseTemplate::new(200)
24130 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
24131 )
24132 .mount(&server)
24133 .await;
24134 Mock::given(method("POST"))
24135 .and(path("/api/v1/orgs/org-roundtrip/inference/responses"))
24136 .respond_with(ResponseTemplate::new(200).set_body_raw(
24137 include_str!("../tests/fixtures/parslee-openrouter-reasoning-roundtrip.sse"),
24138 "text/event-stream",
24139 ))
24140 .expect(2)
24141 .mount(&server)
24142 .await;
24143
24144 let tmp = TempDir::new().unwrap();
24145 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24146 let schema = crate::openrouter::curated_schemas()
24147 .into_iter()
24148 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
24149 .unwrap();
24150 engine.unified_registry_mut().register_project_model(schema);
24151
24152 let first = engine
24153 .generate_tracked(GenerateRequest {
24154 prompt: "first".into(),
24155 model: Some("parslee/openrouter/frontier-general".into()),
24156 ..Default::default()
24157 })
24158 .await
24159 .expect("first managed turn");
24160 let expected_reasoning = serde_json::json!({
24161 "type": "reasoning",
24162 "id": "rs_car_roundtrip",
24163 "status": "completed",
24164 "summary": [{"type": "summary_text", "text": "safe summary"}],
24165 "encrypted_content": "opaque-encrypted-reasoning",
24166 });
24167 assert_eq!(
24168 first.provider_output_items,
24169 vec![expected_reasoning.clone()]
24170 );
24171
24172 let mut history = vec![Message::User {
24173 content: "first".into(),
24174 }];
24175 first.append_assistant_history(&mut history, first.tool_calls.clone());
24176 history.push(Message::User {
24177 content: "continue".into(),
24178 });
24179
24180 let second = engine
24181 .generate_tracked(GenerateRequest {
24182 prompt: String::new(),
24183 model: Some("parslee/openrouter/frontier-general".into()),
24184 messages: Some(history),
24185 ..Default::default()
24186 })
24187 .await
24188 .expect("second managed turn");
24189 assert_eq!(second.text, "first answer");
24190
24191 let requests = server.received_requests().await.unwrap();
24192 let posts: Vec<serde_json::Value> = requests
24193 .iter()
24194 .filter(|request| {
24195 request.method.as_str() == "POST"
24196 && request.url.path() == "/api/v1/orgs/org-roundtrip/inference/responses"
24197 })
24198 .map(|request| serde_json::from_slice(&request.body).unwrap())
24199 .collect();
24200 assert_eq!(posts.len(), 2);
24201 for body in &posts {
24202 assert_eq!(body["store"], false);
24203 assert_eq!(
24204 body["include"],
24205 serde_json::json!(["reasoning.encrypted_content"])
24206 );
24207 }
24208 let second_input = posts[1]["input"].as_array().unwrap();
24209 let reasoning_index = second_input
24210 .iter()
24211 .position(|item| item == &expected_reasoning)
24212 .expect("second request must replay the exact reasoning item");
24213 let assistant_index = second_input
24214 .iter()
24215 .position(|item| item["role"] == "assistant")
24216 .expect("second request assistant turn");
24217 let user_index = second_input
24218 .iter()
24219 .position(|item| item["content"] == "continue")
24220 .expect("second request user turn");
24221 assert!(
24222 reasoning_index < assistant_index && assistant_index < user_index,
24223 "provider output order must be reasoning, assistant text, then the next user turn"
24224 );
24225
24226 unsafe {
24227 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
24228 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
24229 }
24230 }
24231
24232 #[tokio::test(flavor = "current_thread")]
24233 async fn managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures() {
24234 if !crate::run_in_isolated_test_process(
24235 "tests::managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures",
24236 "CAR_MANAGED_PARTIAL_EOF_CHILD",
24237 ) {
24238 return;
24239 }
24240 let _home = crate::openrouter::StateRootScope::new();
24241 use wiremock::matchers::{header, method, path};
24242 use wiremock::{Mock, MockServer, ResponseTemplate};
24243
24244 let _provider_env = crate::openrouter::test_environment_scope_async().await;
24245 let _env = ENV_MUTEX.lock().await;
24246 let bearer = "managed-partial-outcome-bearer";
24247
24248 let server = MockServer::start().await;
24249 unsafe {
24250 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
24251 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
24252 }
24253 Mock::given(method("GET"))
24254 .and(path("/api/v1/organizations/me"))
24255 .and(header("authorization", format!("Bearer {bearer}")))
24256 .respond_with(
24257 ResponseTemplate::new(200)
24258 .set_body_json(serde_json::json!({"organizationId": "org-partial-outcome"})),
24259 )
24260 .mount(&server)
24261 .await;
24262 Mock::given(method("GET"))
24263 .and(path("/connect/session"))
24264 .respond_with(
24265 ResponseTemplate::new(200)
24266 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
24267 )
24268 .mount(&server)
24269 .await;
24270 Mock::given(method("POST"))
24271 .and(path("/api/v1/orgs/org-partial-outcome/inference/responses"))
24272 .respond_with(ResponseTemplate::new(200).set_body_raw(
24273 "event: response.output_text.delta\ndata: {\"delta\":\"partial must fail\"}\n\n",
24274 "text/event-stream",
24275 ))
24276 .expect(2)
24277 .mount(&server)
24278 .await;
24279
24280 let tmp = TempDir::new().unwrap();
24281 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24282 let model_id = "parslee/openrouter/frontier-general";
24283 let schema = crate::openrouter::curated_schemas()
24284 .into_iter()
24285 .find(|schema| schema.id == model_id)
24286 .unwrap();
24287 engine.unified_registry_mut().register_project_model(schema);
24288
24289 let buffered_error = engine
24290 .generate_tracked(GenerateRequest {
24291 prompt: "buffered".into(),
24292 model: Some(model_id.into()),
24293 params: GenerateParams {
24294 strict_model: true,
24295 ..Default::default()
24296 },
24297 ..Default::default()
24298 })
24299 .await
24300 .expect_err("buffered partial EOF must fail");
24301 assert!(
24302 buffered_error.to_string().contains("response.completed"),
24303 "unexpected buffered error: {buffered_error}"
24304 );
24305
24306 let mut stream = engine
24307 .generate_tracked_stream(GenerateRequest {
24308 prompt: "streamed".into(),
24309 model: Some(model_id.into()),
24310 params: GenerateParams {
24311 strict_model: true,
24312 ..Default::default()
24313 },
24314 ..Default::default()
24315 })
24316 .await
24317 .expect("HTTP streaming request starts");
24318 let mut events = Vec::new();
24319 while let Some(event) = stream.events.recv().await {
24320 events.push(event);
24321 }
24322 assert!(
24323 matches!(events.last(), Some(StreamEvent::Error(message)) if message.contains("response.completed"))
24324 );
24325 assert!(!events
24326 .iter()
24327 .any(|event| matches!(event, StreamEvent::Done { .. })));
24328
24329 for _ in 0..50 {
24330 if engine
24331 .outcome_tracker()
24332 .read()
24333 .await
24334 .profile(model_id)
24335 .is_some_and(|profile| profile.fail_count == 2)
24336 {
24337 break;
24338 }
24339 tokio::task::yield_now().await;
24340 }
24341 let profile = engine
24342 .outcome_tracker()
24343 .read()
24344 .await
24345 .profile(model_id)
24346 .cloned()
24347 .unwrap();
24348 assert_eq!(profile.fail_count, 2);
24349 assert_eq!(profile.success_count, 0);
24350
24351 unsafe {
24352 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
24353 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
24354 }
24355 }
24356
24357 #[tokio::test]
24358 async fn tokenize_rejects_known_remote_model_with_unsupported_mode() {
24359 let tmp = TempDir::new().unwrap();
24364 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24365 let remote_id = engine
24366 .list_schemas()
24367 .into_iter()
24368 .find(|s| !s.is_local())
24369 .map(|s| s.id)
24370 .expect("built-in catalog should include at least one remote model schema");
24371
24372 let err = engine
24373 .tokenize(&remote_id, "hello")
24374 .await
24375 .expect_err("remote tokenize must error");
24376 match err {
24377 InferenceError::UnsupportedMode { mode, backend, .. } => {
24378 assert_eq!(mode, "tokenize/detokenize");
24379 assert_eq!(backend, "remote");
24380 }
24381 other => panic!("expected UnsupportedMode, got {other:?}"),
24382 }
24383
24384 let err = engine
24385 .detokenize(&remote_id, &[1, 2, 3])
24386 .await
24387 .expect_err("remote detokenize must error");
24388 assert!(
24389 matches!(err, InferenceError::UnsupportedMode { .. }),
24390 "expected UnsupportedMode, got {err:?}"
24391 );
24392 }
24393
24394 #[test]
24395 fn unsupported_mode_does_not_trip_circuit_breaker() {
24396 let unsupported = InferenceError::UnsupportedMode {
24402 mode: "structured-output-json-schema",
24403 backend: "anthropic",
24404 reason: "not wired under the pinned API version",
24405 };
24406 assert!(!error_counts_against_circuit_breaker(&unsupported));
24407 assert!(error_counts_against_circuit_breaker(
24408 &InferenceError::InferenceFailed("API returned 500".into())
24409 ));
24410 assert!(error_counts_against_circuit_breaker(
24411 &InferenceError::InferenceFailed("API returned 429".into())
24412 ));
24413 }
24414
24415 #[test]
24423 fn a_content_refusal_does_not_trip_the_circuit_breaker() {
24424 let refused = InferenceError::ContentRefused {
24425 provider: "parslee".into(),
24426 kind: Some("invalid_request_error".into()),
24427 code: Some("content_policy_violation".into()),
24428 message: "content refused".into(),
24429 };
24430 assert!(!error_counts_against_circuit_breaker(&refused));
24431
24432 let rendered = refused.to_string();
24435 assert!(rendered.contains("content grounds"), "{rendered}");
24436 assert!(rendered.contains("content_policy_violation"), "{rendered}");
24437
24438 assert!(error_counts_against_circuit_breaker(
24440 &InferenceError::InferenceFailed("managed inference failed".into())
24441 ));
24442 }
24443
24444 #[test]
24454 fn a_content_refusal_ends_the_fallback_chain() {
24455 let refused = InferenceError::ContentRefused {
24456 provider: "parslee".into(),
24457 kind: Some("invalid_request_error".into()),
24458 code: Some("content_policy_violation".into()),
24459 message: "content refused".into(),
24460 };
24461 assert!(error_ends_fallback_chain(&refused));
24462
24463 for still_advances in [
24467 InferenceError::InferenceFailed("managed inference failed".into()),
24468 InferenceError::InferenceFailed("API returned 503".into()),
24469 InferenceError::ModelNotFound("parslee/reasoning".into()),
24470 InferenceError::UnsupportedMode {
24471 mode: "json_schema",
24472 backend: "anthropic",
24473 reason: "structured output is not supported by this protocol",
24474 },
24475 InferenceError::GatewayUnconfigured {
24476 provider: "parslee".into(),
24477 namespace: "parslee/openrouter/".into(),
24478 status: 503,
24479 message: "not configured".into(),
24480 },
24481 ] {
24482 assert!(
24483 !error_ends_fallback_chain(&still_advances),
24484 "must keep advancing the chain: {still_advances}"
24485 );
24486 }
24487 }
24488
24489 #[test]
24498 fn recovery_hints_do_not_rewrite_a_content_refusal() {
24499 let refused = InferenceError::ContentRefused {
24500 provider: "parslee".into(),
24501 kind: Some("invalid_request_error".into()),
24502 code: Some("content_policy_violation".into()),
24503 message: "blocked: 403 forbidden by the content filter".into(),
24505 };
24506 let out = apply_exhaustion_recovery_hint(refused);
24507 assert!(
24508 matches!(out, InferenceError::ContentRefused { .. }),
24509 "{out:?}"
24510 );
24511
24512 let signed_out = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
24514 "no credential for proprietary provider 'parslee'".into(),
24515 ));
24516 assert!(
24517 matches!(signed_out, InferenceError::InferenceFailed(ref m) if m.contains("car auth")),
24518 "{signed_out:?}"
24519 );
24520
24521 let transient = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
24523 "API returned 500".into(),
24524 ));
24525 assert_eq!(transient.to_string(), "inference failed: API returned 500");
24526 }
24527
24528 #[test]
24536 fn unconfigured_gateway_does_not_trip_circuit_breaker() {
24537 let unconfigured = InferenceError::GatewayUnconfigured {
24538 provider: "parslee".into(),
24539 namespace: "parslee/openrouter/".into(),
24540 status: 503,
24541 message: "OpenRouter inference is not configured on this Parslee environment.".into(),
24542 };
24543 assert!(!error_counts_against_circuit_breaker(&unconfigured));
24544 let rendered = unconfigured.to_string();
24548 assert!(rendered.contains("parslee/openrouter/"), "{rendered}");
24549 assert!(rendered.contains("not configured"), "{rendered}");
24550 }
24551
24552 #[test]
24558 fn namespace_drop_spares_siblings_outside_the_prefix() {
24559 let namespace = "parslee/openrouter/";
24560 let mut queue: std::collections::VecDeque<String> = [
24561 "parslee/openrouter/open-fast",
24562 "parslee/reasoning",
24563 "parslee/openrouter/frontier-general",
24564 "parslee/advisor",
24565 "anthropic/claude-opus-4-8:latest",
24566 ]
24567 .into_iter()
24568 .map(String::from)
24569 .collect();
24570
24571 queue.retain(|id| !id.starts_with(namespace));
24572
24573 assert_eq!(
24574 queue.iter().collect::<Vec<_>>(),
24575 vec![
24576 "parslee/reasoning",
24577 "parslee/advisor",
24578 "anthropic/claude-opus-4-8:latest"
24579 ],
24580 "only the unconfigured namespace may be dropped"
24581 );
24582 }
24583
24584 #[test]
24585 fn engine_loads_benchmark_priors_on_startup() {
24586 let _env = ENV_MUTEX.blocking_lock();
24587 let tmp = TempDir::new().unwrap();
24588 let priors_path = tmp.path().join("benchmark_priors.json");
24589 std::fs::write(
24590 &priors_path,
24591 serde_json::json!({
24592 "model_id": "qwen/qwen3-8b:q4_k_m",
24593 "overall_score": 0.88
24594 })
24595 .to_string(),
24596 )
24597 .unwrap();
24598
24599 unsafe {
24600 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
24601 }
24602
24603 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24604 let tracker = engine.outcome_tracker.blocking_read();
24605 let profile = tracker
24606 .profile("qwen/qwen3-8b:q4_k_m")
24607 .expect("benchmark prior should create a profile");
24608 assert!((profile.ema_quality - 0.88).abs() < 0.01);
24609
24610 unsafe {
24611 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
24612 }
24613 }
24614
24615 #[test]
24616 fn benchmark_priors_do_not_override_observed_profiles() {
24617 let _env = ENV_MUTEX.blocking_lock();
24618 let tmp = TempDir::new().unwrap();
24619 let models_dir = tmp.path().join("models");
24620 std::fs::create_dir_all(&models_dir).unwrap();
24621
24622 let observed = vec![ModelProfile {
24623 model_id: "qwen/qwen3-8b:q4_k_m".into(),
24624 total_calls: 12,
24625 success_count: 3,
24626 fail_count: 9,
24627 total_latency_ms: 1200,
24628 total_input_tokens: 0,
24629 total_output_tokens: 0,
24630 total_cache_read_input_tokens: 0,
24631 total_cache_creation_input_tokens: 0,
24632 task_stats: std::collections::HashMap::new(),
24633 ema_quality: 0.21,
24634 prior_sample_size: 0,
24635 quality_observations: 0,
24636 quality_per_1k_tokens: 0.0,
24637 updated_at: 1,
24638 }];
24639 std::fs::write(
24640 models_dir.join("outcome_profiles.json"),
24641 serde_json::to_string(&observed).unwrap(),
24642 )
24643 .unwrap();
24644
24645 let priors_path = tmp.path().join("benchmark_priors.json");
24646 std::fs::write(
24647 &priors_path,
24648 serde_json::json!({
24649 "model_id": "qwen/qwen3-8b:q4_k_m",
24650 "overall_score": 0.95
24651 })
24652 .to_string(),
24653 )
24654 .unwrap();
24655
24656 unsafe {
24657 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
24658 }
24659
24660 let engine = InferenceEngine::new(test_config(models_dir));
24661 let tracker = engine.outcome_tracker.blocking_read();
24662 let profile = tracker
24663 .profile("qwen/qwen3-8b:q4_k_m")
24664 .expect("observed profile should remain present");
24665 assert!((profile.ema_quality - 0.21).abs() < 0.01);
24666 assert_eq!(profile.total_calls, 12);
24667
24668 unsafe {
24669 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
24670 }
24671 }
24672
24673 #[test]
24674 fn speech_runtime_package_spec_defaults_and_overrides() {
24675 let _env = ENV_MUTEX.blocking_lock();
24676 unsafe {
24677 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
24678 }
24679 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.2");
24680
24681 unsafe {
24682 std::env::set_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC", "mlx-audio==0.4.1");
24683 }
24684 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.1");
24685
24686 unsafe {
24687 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
24688 }
24689 }
24690
24691 #[test]
24692 fn speech_runtime_spacy_model_spec_defaults_and_overrides() {
24693 let _env = ENV_MUTEX.blocking_lock();
24694 unsafe {
24695 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
24696 }
24697 assert!(
24698 speech_runtime_spacy_model_spec().starts_with("en-core-web-sm @ https://github.com/")
24699 );
24700
24701 unsafe {
24702 std::env::set_var(
24703 "CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC",
24704 "en-core-web-sm==3.8.0",
24705 );
24706 }
24707 assert_eq!(speech_runtime_spacy_model_spec(), "en-core-web-sm==3.8.0");
24708
24709 unsafe {
24710 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
24711 }
24712 }
24713
24714 #[test]
24715 fn kokoro_runtime_fallback_defaults_on() {
24716 unsafe {
24717 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
24718 }
24719 assert!(kokoro_runtime_fallback_enabled());
24720
24721 unsafe {
24722 std::env::set_var("CAR_SPEECH_KOKORO_FALLBACK", "false");
24723 }
24724 assert!(!kokoro_runtime_fallback_enabled());
24725
24726 unsafe {
24727 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
24728 }
24729 }
24730
24731 #[test]
24732 fn preferred_local_tts_wins_over_builtin_rank() {
24733 let tmp = TempDir::new().unwrap();
24734 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24735 engine.set_speech_policy(SpeechPolicy {
24736 prefer_local: true,
24737 allow_remote_fallback: false,
24738 preferred_local_stt: None,
24739 preferred_local_tts: Some("Kokoro-82M-6bit".into()),
24740 preferred_remote_stt: None,
24741 preferred_remote_tts: None,
24742 });
24743
24744 let schema = engine
24745 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
24746 .expect("preferred local TTS should resolve");
24747 #[cfg(not(target_os = "windows"))]
24754 assert_eq!(schema.name, "Kokoro-82M-6bit");
24755 #[cfg(target_os = "windows")]
24756 assert_eq!(schema.name, "Windows-Speech");
24757 }
24758
24759 #[test]
24760 fn preferred_discovered_vllm_mlx_model_wins_generate_routing() {
24761 let tmp = TempDir::new().unwrap();
24762 let mut config = test_config(tmp.path().join("models"));
24763 config.preferred_generation_model =
24764 Some("vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit".into());
24765 let mut engine = InferenceEngine::new(config);
24766 let schema = crate::vllm_mlx::to_model_schema(
24767 &crate::vllm_mlx::DiscoveredModel {
24768 id: "mlx-community/gemma-3n-E2B-it-lm-4bit".into(),
24769 owned_by: Some("mlx-community".into()),
24770 },
24771 "http://127.0.0.1:8001",
24772 );
24773 engine.register_model(schema);
24774
24775 let rt = tokio::runtime::Runtime::new().unwrap();
24776 let decision = rt.block_on(engine.route_adaptive("say hello in one sentence"));
24777 assert_eq!(
24778 decision.model_id,
24779 "vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit"
24780 );
24781 assert_eq!(decision.strategy, RoutingStrategy::Explicit);
24782 assert_eq!(decision.reason, "preferred generation model override");
24783 }
24784
24785 #[test]
24794 fn tools_request_on_non_tool_model_terminates_not_spins() {
24795 let tmp = TempDir::new().unwrap();
24796 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24797 let schema = crate::vllm_mlx::to_model_schema(
24799 &crate::vllm_mlx::DiscoveredModel {
24800 id: "test-org/embed-only-model".into(),
24801 owned_by: None,
24802 },
24803 "http://127.0.0.1:1", );
24805 let model_id = schema.id.clone();
24806 engine.register_model(schema);
24807
24808 let mut req = GenerateRequest {
24809 prompt: "call a tool".to_string(),
24810 model: Some(model_id),
24811 tools: Some(vec![serde_json::json!({
24812 "name": "noop", "description": "n", "parameters": {"type": "object"}
24813 })]),
24814 ..Default::default()
24815 };
24816 req.params.strict_model = true;
24817
24818 let rt = tokio::runtime::Runtime::new().unwrap();
24819 let out = rt.block_on(async {
24820 tokio::time::timeout(
24821 std::time::Duration::from_secs(10),
24822 engine.generate_tracked(req),
24823 )
24824 .await
24825 });
24826 let completed = out.expect("fallback loop must terminate, not spin");
24829 assert!(completed.is_err());
24830 }
24831
24832 fn fake_ready_speech_runtime(root: &Path) {
24843 managed_venv::seed_ready_venv(root);
24844 let runtime = SpeechRuntime::new(root.to_path_buf());
24845 for program in [&runtime.stt_program, &runtime.tts_program] {
24846 std::fs::create_dir_all(program.parent().expect("program has a parent")).unwrap();
24847 std::fs::write(program, b"").unwrap();
24848 }
24849 }
24850
24851 #[tokio::test]
24878 async fn prepare_speech_runtime_returns_the_root_doctor_reports() {
24879 let _env = ENV_MUTEX.lock().await;
24880 let tmp = TempDir::new().unwrap();
24881 let runtime_root = tmp.path().join("speech-runtime");
24882 fake_ready_speech_runtime(&runtime_root);
24883 unsafe {
24884 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
24885 }
24886
24887 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24888 let root = engine
24889 .prepare_speech_runtime()
24890 .await
24891 .expect("prepare should succeed against a ready runtime");
24892 let health = engine.speech_health();
24893
24894 assert_eq!(
24895 root, health.runtime.root,
24896 "prepare returned a different root than doctor reports"
24897 );
24898 assert!(
24899 root.exists(),
24900 "prepare returned {} but it does not exist",
24901 root.display()
24902 );
24903 assert!(
24904 health.runtime.installed,
24905 "doctor should report a ready runtime as installed"
24906 );
24907 assert_eq!(
24909 engine
24910 .prepare_speech_runtime()
24911 .await
24912 .expect("second prepare should succeed"),
24913 root
24914 );
24915
24916 unsafe {
24917 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
24918 }
24919 }
24920
24921 #[tokio::test]
24935 async fn prepare_speech_runtime_fails_when_bootstrap_fails() {
24936 let _env = ENV_MUTEX.lock().await;
24937 let tmp = TempDir::new().unwrap();
24938 let runtime_root = tmp.path().join("speech-runtime");
24939 assert!(!runtime_root.exists(), "precondition: root absent");
24940 unsafe {
24941 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
24942 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
24943 }
24944
24945 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24946 let err = engine.prepare_speech_runtime().await.err();
24947 let health = engine.speech_health();
24948
24949 assert!(
24950 err.is_some(),
24951 "a failed bootstrap must fail the install, not return a root"
24952 );
24953 assert!(
24954 !health.runtime.installed,
24955 "doctor must not claim a runtime the bootstrap never built"
24956 );
24957
24958 unsafe {
24959 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
24960 std::env::remove_var("CAR_SPEECH_PYTHON");
24961 }
24962 }
24963
24964 #[cfg(target_os = "linux")]
24977 #[tokio::test]
24978 async fn a_runtime_that_cannot_be_built_no_longer_blocks_the_whisper_install() {
24979 let _env = ENV_MUTEX.lock().await;
24980 let tmp = TempDir::new().unwrap();
24981
24982 let cached = tmp
24986 .path()
24987 .join(".tokhn")
24988 .join("whisper")
24989 .join("ggml-large-v3-turbo-q5_0.bin");
24990 std::fs::create_dir_all(cached.parent().unwrap()).unwrap();
24991 std::fs::write(&cached, b"stand-in for the ggml weights").unwrap();
24992
24993 let previous_home = std::env::var_os("HOME");
24994 unsafe {
24995 std::env::set_var("HOME", tmp.path());
24996 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", tmp.path().join("speech-runtime"));
24997 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
25001 }
25002
25003 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25004 let installed = engine.install_curated_speech().await;
25005
25006 let restore = || unsafe {
25007 match &previous_home {
25008 Some(value) => std::env::set_var("HOME", value),
25009 None => std::env::remove_var("HOME"),
25010 }
25011 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
25012 std::env::remove_var("CAR_SPEECH_PYTHON");
25013 };
25014 let installed = match installed {
25015 Ok(installed) => installed,
25016 Err(error) => {
25017 restore();
25018 panic!("a runtime that cannot be built must not abort the install: {error}");
25019 }
25020 };
25021 let runtime_installed = engine.speech_health().runtime.installed;
25022 restore();
25023
25024 assert!(
25025 !runtime_installed,
25026 "precondition: the bootstrap must actually have failed, or this \
25027 test would pass without exercising anything"
25028 );
25029 let whisper = installed
25030 .iter()
25031 .find(|report| report.hf_repo == "ggerganov/whisper.cpp")
25032 .unwrap_or_else(|| {
25033 panic!(
25034 "the cross-platform whisper model must still be installed, got: {installed:?}"
25035 )
25036 });
25037 assert_eq!(whisper.snapshot_path, cached);
25038 assert!(
25039 installed
25040 .iter()
25041 .all(|report| report.hf_repo == "ggerganov/whisper.cpp"),
25042 "MLX weights are Apple-only and must not be pulled here — over a \
25043 gigabyte of them, for models this host can never load; got: {installed:?}"
25044 );
25045 }
25046
25047 #[test]
25055 fn alias_resolves_to_canonical_id_for_outcome_keying() {
25056 let tmp = TempDir::new().unwrap();
25057 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25058
25059 let cases = [
25061 ("claude-sonnet-4-6", "anthropic/claude-sonnet-4-6:latest"),
25062 ("gpt-5.4", "openai/gpt-5.4:latest"),
25063 ("gemini-2.5-flash", "google/gemini-2.5-flash:latest"),
25064 ];
25065 for (alias, canonical) in cases {
25066 let resolved = engine
25069 .resolve_schema(alias)
25070 .map(|s| s.id)
25071 .unwrap_or_else(|| alias.to_string());
25072 assert_eq!(
25073 resolved, canonical,
25074 "alias `{alias}` must resolve to canonical `{canonical}` for outcome keying, got `{resolved}`"
25075 );
25076 assert_ne!(
25077 resolved, alias,
25078 "alias `{alias}` must NOT be recorded raw — that is the fragmentation bug"
25079 );
25080 }
25081 }
25082
25083 #[test]
25088 fn inference_result_serializes_with_full_shape() {
25089 use crate::tasks::generate::ToolCall;
25090 use std::collections::HashMap;
25091
25092 let mut args = HashMap::new();
25093 args.insert("path".to_string(), serde_json::json!("README.md"));
25094
25095 let result = InferenceResult {
25096 text: String::new(),
25097 bounding_boxes: Vec::new(),
25098 tool_calls: vec![ToolCall {
25099 id: None,
25100 name: "read_file".into(),
25101 arguments: args,
25102 }],
25103 trace_id: "trace-abc".into(),
25104 model_used: "test-model".into(),
25105 model_identity: InferenceModelIdentity {
25106 requested_model_id: Some("test/model:1".into()),
25107 resolved_model_id: "test/model:1".into(),
25108 row_digest: "a".repeat(64),
25109 catalog_revision: "b".repeat(64),
25110 },
25111 latency_ms: 1234,
25112 time_to_first_token_ms: Some(180),
25113 usage: Some(TokenUsage {
25114 prompt_tokens: 100,
25115 completion_tokens: 50,
25116 total_tokens: 150,
25117 context_window: 8192,
25118 ..Default::default()
25119 }),
25120 provider_output_items: Vec::new(),
25121 thinking: Vec::new(),
25122 stop_reason: Some("tool_use".into()),
25123 auth_fallback_from: None,
25124 local_last_resort: false,
25125 fallback_from: Vec::new(),
25126 };
25127
25128 let json = serde_json::to_value(&result).expect("serialize");
25129
25130 assert_eq!(json["stop_reason"].as_str(), Some("tool_use"));
25132
25133 assert_eq!(json["text"].as_str(), Some(""));
25135 assert_eq!(json["trace_id"].as_str(), Some("trace-abc"));
25136 assert_eq!(json["model_used"].as_str(), Some("test-model"));
25137 assert_eq!(json["requested_model_id"].as_str(), Some("test/model:1"));
25138 assert_eq!(json["resolved_model_id"].as_str(), Some("test/model:1"));
25139 assert_eq!(json["row_digest"].as_str().unwrap(), "a".repeat(64));
25140 assert_eq!(json["catalog_revision"].as_str().unwrap(), "b".repeat(64));
25141 assert_eq!(json["latency_ms"].as_u64(), Some(1234));
25142
25143 let tool_calls = json["tool_calls"].as_array().expect("tool_calls array");
25145 assert_eq!(tool_calls.len(), 1);
25146 assert_eq!(tool_calls[0]["name"].as_str(), Some("read_file"));
25147 assert_eq!(
25148 tool_calls[0]["arguments"]["path"].as_str(),
25149 Some("README.md")
25150 );
25151
25152 let usage = &json["usage"];
25154 assert_eq!(usage["prompt_tokens"].as_u64(), Some(100));
25155 assert_eq!(usage["completion_tokens"].as_u64(), Some(50));
25156 assert_eq!(usage["total_tokens"].as_u64(), Some(150));
25157 assert_eq!(usage["context_window"].as_u64(), Some(8192));
25158
25159 assert_eq!(json["time_to_first_token_ms"].as_u64(), Some(180));
25161 }
25162
25163 #[test]
25169 fn inference_result_top_level_keys_are_locked() {
25170 use std::collections::BTreeSet;
25171
25172 let result = InferenceResult {
25173 text: "anything".into(),
25174 bounding_boxes: Vec::new(),
25175 tool_calls: vec![],
25176 trace_id: "t".into(),
25177 model_used: "m".into(),
25178 model_identity: InferenceModelIdentity::default(),
25179 latency_ms: 0,
25180 time_to_first_token_ms: None,
25181 usage: None,
25182 provider_output_items: Vec::new(),
25183 thinking: Vec::new(),
25184 stop_reason: None,
25185 auth_fallback_from: None,
25186 local_last_resort: false,
25187 fallback_from: Vec::new(),
25188 };
25189
25190 let json = serde_json::to_value(&result).expect("serialize");
25191 let keys: BTreeSet<&str> = json
25192 .as_object()
25193 .expect("top-level object")
25194 .keys()
25195 .map(String::as_str)
25196 .collect();
25197
25198 let expected: BTreeSet<&str> = [
25199 "text",
25200 "tool_calls",
25201 "trace_id",
25202 "model_used",
25203 "requested_model_id",
25204 "resolved_model_id",
25205 "row_digest",
25206 "catalog_revision",
25207 "latency_ms",
25208 "time_to_first_token_ms",
25209 "usage",
25210 "stop_reason",
25211 ]
25212 .into_iter()
25213 .collect();
25214
25215 assert_eq!(
25216 keys, expected,
25217 "infer response top-level keys drifted -- update both the test \
25218 and the WebSocket protocol documentation if this is intentional"
25219 );
25220
25221 for key in &keys {
25223 assert!(
25224 !key.chars().any(|c| c.is_uppercase()) && !key.contains('-'),
25225 "key '{}' is not snake_case",
25226 key
25227 );
25228 }
25229 }
25230
25231 #[test]
25235 fn inference_result_serializes_plain_text_response() {
25236 let result = InferenceResult {
25237 text: "hello world".into(),
25238 bounding_boxes: Vec::new(),
25239 tool_calls: vec![],
25240 trace_id: "trace-xyz".into(),
25241 model_used: "test-model".into(),
25242 model_identity: InferenceModelIdentity::default(),
25243 latency_ms: 42,
25244 time_to_first_token_ms: None,
25245 usage: None,
25246 provider_output_items: Vec::new(),
25247 thinking: Vec::new(),
25248 stop_reason: None,
25249 auth_fallback_from: None,
25250 local_last_resort: false,
25251 fallback_from: Vec::new(),
25252 };
25253
25254 let json = serde_json::to_value(&result).expect("serialize");
25255 assert_eq!(json["text"], "hello world");
25256 assert!(json["stop_reason"].is_null());
25258 assert!(json["tool_calls"].is_array());
25259 assert_eq!(json["tool_calls"].as_array().unwrap().len(), 0);
25260 assert_eq!(json["model_used"], "test-model");
25261 assert!(json["usage"].is_null());
25262 assert!(json["time_to_first_token_ms"].is_null());
25265 }
25266
25267 #[test]
25268 fn append_assistant_history_preserves_responses_items_in_provider_order() {
25269 let reasoning = serde_json::json!({
25270 "type": "reasoning",
25271 "id": "rs_history",
25272 "status": "completed",
25273 "summary": [{"type": "summary_text", "text": "safe"}],
25274 "encrypted_content": "opaque",
25275 });
25276 let result: InferenceResult = serde_json::from_value(serde_json::json!({
25277 "text": "calling",
25278 "tool_calls": [{
25279 "id": "call_1",
25280 "name": "read_file",
25281 "arguments": {"path": "README.md"}
25282 }],
25283 "trace_id": "trace",
25284 "model_used": "gateway-alias",
25285 "resolved_model_id": "openrouter/anthropic/claude-sonnet-4.5",
25286 "local_last_resort": true,
25287 "latency_ms": 1,
25288 "provider_output_items": [reasoning.clone()],
25289 }))
25290 .unwrap();
25291 let mut history = vec![crate::tasks::generate::Message::User {
25292 content: "inspect".into(),
25293 }];
25294
25295 result.append_assistant_history(&mut history, result.tool_calls.clone());
25296
25297 assert!(matches!(
25298 &history[1],
25299 crate::tasks::generate::Message::ProviderOutputItems { protocol, items }
25300 if protocol == crate::protocol::OPENAI_RESPONSES_PROTOCOL
25301 && items == &vec![reasoning]
25302 ));
25303 assert!(matches!(
25304 &history[2],
25305 crate::tasks::generate::Message::Assistant {
25306 content,
25307 tool_calls,
25308 model_id,
25309 local_last_resort,
25310 ..
25311 } if content == "calling"
25312 && tool_calls[0].id.as_deref() == Some("call_1")
25313 && model_id.as_deref() == Some("openrouter/anthropic/claude-sonnet-4.5")
25314 && *local_last_resort
25315 ));
25316 }
25317
25318 #[test]
25319 fn append_assistant_history_leaves_personal_chat_history_unchanged() {
25320 let result: InferenceResult = serde_json::from_value(serde_json::json!({
25321 "text": "plain",
25322 "tool_calls": [],
25323 "trace_id": "trace",
25324 "model_used": "openrouter/openai/gpt-4.1-mini",
25325 "latency_ms": 1,
25326 }))
25327 .unwrap();
25328 let mut history = Vec::new();
25329
25330 result.append_assistant_history(&mut history, Vec::new());
25331
25332 assert_eq!(history.len(), 1);
25333 assert!(matches!(
25334 &history[0],
25335 crate::tasks::generate::Message::Assistant { content, .. } if content == "plain"
25336 ));
25337 }
25338
25339 #[test]
25351 fn generate_request_deserializes_intent_field_from_json_rpc_params() {
25352 use crate::intent::TaskHint;
25353 use crate::schema::ModelCapability;
25354
25355 let params = serde_json::json!({
25358 "prompt": "summarize this email",
25359 "intent": {
25360 "task": "chat",
25361 "prefer_local": true,
25362 "require": ["tool_use"],
25363 },
25364 });
25365
25366 let req: GenerateRequest =
25367 serde_json::from_value(params).expect("GenerateRequest deserialize");
25368
25369 let intent = req.intent.as_ref().expect("intent field deserialized");
25370 assert_eq!(intent.task, Some(TaskHint::Chat));
25371 assert!(intent.prefer_local);
25372 assert_eq!(intent.require, vec![ModelCapability::ToolUse]);
25373
25374 let back: serde_json::Value =
25378 serde_json::to_value(&req).expect("re-serialize GenerateRequest");
25379 assert_eq!(back["intent"]["task"], "chat");
25380 assert_eq!(back["intent"]["prefer_local"], true);
25381 assert_eq!(back["intent"]["require"][0], "tool_use");
25382
25383 let default_req: GenerateRequest = serde_json::from_value(serde_json::json!({
25388 "prompt": "x",
25389 "intent": {},
25390 }))
25391 .unwrap();
25392 let default_intent = default_req.intent.expect("present but empty");
25393 assert_eq!(default_intent.task, None);
25394 assert!(!default_intent.prefer_local);
25395 assert!(default_intent.require.is_empty());
25396
25397 let no_intent: GenerateRequest =
25400 serde_json::from_value(serde_json::json!({"prompt": "x"})).unwrap();
25401 assert!(no_intent.intent.is_none());
25402 }
25403
25404 #[test]
25405 fn rerank_prompt_matches_upstream_template_shape() {
25406 let p = rerank_prompt(
25407 "retrieve relevant passages",
25408 "who runs the treasury?",
25409 "doc x",
25410 );
25411 assert!(p.contains("<|im_start|>system"));
25412 assert!(p.contains("Note that the answer can only be \"yes\" or \"no\"."));
25413 assert!(p.contains("<|im_start|>user\n<Instruct>: retrieve relevant passages"));
25414 assert!(p.contains("<Query>: who runs the treasury?"));
25415 assert!(p.contains("<Document>: doc x<|im_end|>"));
25416 assert!(p.contains("<|im_start|>assistant\n<think>\n\n</think>\n\n"));
25417 }
25418
25419 #[test]
25420 fn rerank_score_yes_and_no_exactly() {
25421 assert_eq!(score_from_rerank_output("yes", "m"), 1.0);
25422 assert_eq!(score_from_rerank_output("no", "m"), 0.0);
25423 }
25424
25425 #[test]
25426 fn rerank_score_handles_case_leading_space_and_chat_sentinels() {
25427 assert_eq!(score_from_rerank_output(" Yes", "m"), 1.0);
25430 assert_eq!(score_from_rerank_output("\nno.", "m"), 0.0);
25431 assert_eq!(score_from_rerank_output("<|im_end|>yes", "m"), 1.0);
25432 }
25433
25434 #[test]
25435 fn rerank_score_scans_up_to_three_tokens() {
25436 assert_eq!(score_from_rerank_output("_bos_ yes", "m"), 1.0);
25439 }
25440
25441 #[test]
25442 fn rerank_score_unexpected_is_neutral() {
25443 assert_eq!(score_from_rerank_output("maybe", "m"), 0.5);
25446 assert_eq!(score_from_rerank_output("", "m"), 0.5);
25447 }
25448
25449 #[tokio::test]
25450 async fn pull_reuses_a_valid_directory_receipt_and_reports_removability() {
25451 let root = tempfile::tempdir().unwrap();
25452 let models_dir = root.path().join("models");
25453 std::fs::create_dir_all(&models_dir).unwrap();
25454 let engine = InferenceEngine::new(InferenceConfig {
25455 state_root: root.path().join("state"),
25456 models_dir: models_dir.clone(),
25457 ..InferenceConfig::default()
25458 });
25459 let schema = engine
25460 .unified_registry()
25461 .all()
25462 .find(|schema| matches!(schema.source, ModelSource::Local { .. }))
25463 .unwrap()
25464 .clone();
25465 let managed = models_dir.join(&schema.name);
25466 std::fs::create_dir_all(&managed).unwrap();
25467 std::fs::write(managed.join("model.gguf"), b"owned").unwrap();
25468 std::fs::write(managed.join("tokenizer.json"), b"{}").unwrap();
25469 engine
25470 .model_management
25471 .record_managed_artifact(
25472 &schema.id,
25473 model_source_identity(&schema),
25474 None,
25475 1,
25476 false,
25477 managed.clone(),
25478 )
25479 .unwrap();
25480 let row = engine
25481 .list_models_unified()
25482 .into_iter()
25483 .find(|row| row.id == schema.id)
25484 .unwrap();
25485 if model_management::directory_removal_supported() {
25486 assert!(engine.model_management.can_remove(&schema.id).unwrap());
25487 assert!(row.can_remove);
25488 assert_eq!(row.management_evidence.as_deref(), Some("install_receipt"));
25489 } else {
25490 assert!(!engine.model_management.can_remove(&schema.id).unwrap());
25491 assert!(!row.can_remove);
25492 assert_eq!(
25493 row.management_evidence.as_deref(),
25494 Some("install_receipt_directory_cleanup_unsupported")
25495 );
25496 }
25497
25498 let reused = engine.pull_model(&schema.id).await.unwrap();
25499 assert_eq!(reused, managed);
25500 }
25501}
25502
25503#[cfg(test)]
25504mod response_format_support_tests {
25505 use super::*;
25506
25507 #[test]
25511 fn rejection_reason_tracks_the_protocol_handler() {
25512 let root = tempfile::tempdir().unwrap();
25515 let engine = InferenceEngine::new(InferenceConfig {
25516 state_root: root.path().join("state"),
25517 models_dir: root.path().join("models"),
25518 ..InferenceConfig::default()
25519 });
25520 let rf = crate::tasks::generate::ResponseFormat::JsonObject;
25521 let models = engine.list_models_unified();
25522 let anthropic = models
25523 .iter()
25524 .find(|m| m.provider.eq_ignore_ascii_case("anthropic"));
25525 if let Some(m) = anthropic {
25526 let reason = engine
25527 .response_format_rejection_reason(&m.id, &rf)
25528 .expect("the Anthropic protocol rejects response_format");
25529 assert!(reason.contains("protocol rejects"), "{reason}");
25530 }
25531 let openrouter = models
25532 .iter()
25533 .find(|m| m.provider.eq_ignore_ascii_case("openrouter"));
25534 if let Some(m) = openrouter {
25535 assert_eq!(
25536 engine.response_format_rejection_reason(&m.id, &rf),
25537 None,
25538 "OpenRouter forwards the format upstream"
25539 );
25540 }
25541 assert!(
25542 anthropic.is_some() || openrouter.is_some(),
25543 "the builtin catalog should list at least one of the two providers this pins"
25544 );
25545 assert_eq!(
25546 engine.response_format_rejection_reason("no/such-model", &rf),
25547 None,
25548 "an unknown model is not a rejection"
25549 );
25550 }
25551}