1pub mod action_ledger;
26pub mod adaptive_router;
27pub mod aws_sigv4;
28pub mod backend;
29pub mod backend_cache;
30pub mod billing;
31pub mod calibration;
32pub mod catalog;
33pub mod catalog_identity;
34pub mod concierge;
35pub mod concierge_answer;
36pub mod discovery;
37pub mod doctor;
38pub mod download;
39pub mod handle;
40pub mod hardware;
41pub mod hf_cache;
42pub mod hf_schema;
43pub mod intent;
44pub mod key_pool;
45pub mod lane_defaults;
46pub mod maintenance;
47pub mod managed_venv;
48pub mod media_tokens;
49pub mod model_management;
50pub mod models;
51pub mod nudge;
52pub mod offload;
53pub mod openrouter;
54pub mod outcome;
55pub mod parslee_credential;
56pub mod portfolio;
57pub mod protocol;
58pub mod recommend;
59pub mod registry;
60pub mod remote;
61pub mod resource_policy;
62pub mod retire;
63pub mod router;
64pub mod routing_ext;
65pub mod runner;
66pub mod schema;
67pub mod scoreboard;
68pub mod search;
69pub mod service;
70pub mod stream;
71pub mod tasks;
72pub(crate) mod tls_client;
76pub mod uninstall;
77pub mod update_prefs;
78pub mod upgrade;
79pub mod usage_profile;
80pub mod vllm_mlx;
81pub mod vllm_pool;
82pub mod vllm_runtime;
83
84use std::path::{Path, PathBuf};
85use std::sync::Arc;
86use std::time::Instant;
87use std::time::{SystemTime, UNIX_EPOCH};
88
89use reqwest::multipart::{Form, Part};
90use serde::Serialize;
91use thiserror::Error;
92use tokio::io::AsyncReadExt;
93use tokio::process::Command;
94use tokio::sync::Mutex;
95use tokio::sync::RwLock;
96use tracing::{debug, instrument};
97
98pub use action_ledger::{ConciergeActionEntry, ConciergeActionKind};
100pub use adaptive_router::{
101 AdaptiveRouter, AdaptiveRoutingDecision, RoutingConfig, RoutingStrategy,
102};
103pub use catalog_identity::{CatalogModelRow, CatalogSnapshot};
104pub use concierge::{
105 decide_concierge, evaluate_concierge, ConciergeDecision, ConciergeMode, ConciergeStatus,
106 ConciergeSuggestion, DismissReason, DismissalRecord, ModelHealth,
107 DEFAULT_CONCIERGE_THROTTLE_SECS, DEFAULT_WATCHED_USE_CASES,
108};
109pub use download::{DownloadEvent, DownloadProgress, ProgressSink};
110pub use handle::InferenceHandle;
111pub use intent::{IntentHint, Privacy, QualityTier, TaskHint, TierWeights, UseCase, UseCaseRole};
112pub use key_pool::{KeyPool, KeyStats};
113pub use lane_defaults::{LaneDefault, LaneDefaults};
114pub use nudge::{NudgeDecision, NudgeState, UpgradeNudge};
115pub use outcome::{
116 prune_ledger, read_ledger, CodeOutcome, InferenceOutcome, InferenceTask, InferredOutcome,
117 ModelProfile, OutcomeLedgerEntry, OutcomeTracker,
118};
119pub use recommend::{
120 model_fit, platform_compatible, recommend, recommend_with_policy, FitStatus, ModelFit,
121 ModelFitStatus, Recommendation, RecommendationSet,
122};
123pub use resource_policy::{
124 estimate_model_memory, estimate_model_memory_with_measured_weights, evaluate_resources,
125 AcceleratorResourceBudget, EffectiveResourceBudget, FileResourcePolicyRepository,
126 LocalAdmissionCoordinator, LocalLoadPreflight, LocalLoadVerdict, ModelMemoryEstimate,
127 ModelResourceEvidence, ResourceEvaluation, ResourcePolicy, ResourcePolicyError,
128 ResourcePolicyLoadEvidence, ResourcePolicyLoadSource, ResourcePolicyRepository,
129 ResourceProfile, RECOMMENDATION_CONTEXT_TOKENS,
130};
131pub use update_prefs::{UpdateChannel, UpdatePolicy, UpdatePreferences};
132pub use upgrade::{HuggingFaceProbe, UpgradeFinding, UpgradeSource, UpstreamProbe};
133pub use usage_profile::{use_case_for_task, LaneUsage, UsageProfile};
134
135fn now_unix() -> u64 {
137 std::time::SystemTime::now()
138 .duration_since(std::time::UNIX_EPOCH)
139 .map(|d| d.as_secs())
140 .unwrap_or(0)
141}
142pub use offload::{
143 clear_remote_deadline, current_controlled_termination_token, current_inference_control_id,
144 current_local_offload, current_remote_deadline, ensure_not_controlled_terminated,
145 is_offload_worker, scope_inference_control_id, set_local_offload, set_remote_deadline,
146 ControlledTerminationToken, InferenceTerminationAck, LocalGenerationOffload,
147 LocalOffloadResult, LocalOffloadStream, LocalWorkerAdmission, LocalWorkerResidency,
148 RemoteDeadline,
149};
150pub use registry::{
151 is_builtin_subscription_billed, model_unavailable_reason, ModelFilter, ModelInfo,
152 ModelRuntimeRequirement, ModelUpgrade, UnifiedRegistry,
153};
154pub use remote::RemoteBackend;
155pub use routing_ext::{
156 CircuitBreaker, CircuitBreakerRegistry, CircuitState, ImplicitSignal, ImplicitSignalType,
157 RoutingMode, SpendControl, SpendLimitExceeded, SpendLimits, SpendStatus,
158};
159pub use runner::{
160 current_inference_runner, set_inference_runner, EventEmitter, InferenceRunner, RunnerError,
161 RunnerResult,
162};
163pub use schema::{
164 ApiProtocol, ApproxCost, BenchmarkScore, CostModel, ModelCapability, ModelSchema, ModelSource,
165 PerformanceEnvelope, ProprietaryAuth, ProprietaryProtocol, ProprietaryWire, QuantScheme,
166 Quantization, TrustTier, OPENAI_CODEX_SIGN_IN_HINT,
167};
168
169pub use adaptive_router::{RouterExclusion, RouterExclusionReason, TaskComplexity};
171#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
172pub use backend::CandleBackend;
173#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
174pub use backend::EmbeddingBackend;
175#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
176pub use backend::SwiftLmBackend;
177pub use hardware::HardwareInfo;
178pub use models::{ModelRegistry, ModelRole};
179pub use router::{ModelRouter, RoutingDecision};
180pub use stream::{StreamAccumulator, StreamEvent};
181pub use tasks::{
182 parse_boxes, BoundingBox, ClassifyRequest, ClassifyResult, ContentBlock, EmbedRequest,
183 GenerateImageRequest, GenerateImageResult, GenerateParams, GenerateRequest,
184 GenerateVideoRequest, GenerateVideoResult, GroundRequest, GroundResult, Message,
185 OptionProbabilities, Provenance, RerankRequest, RerankResult, RerankedDocument, ResponseFormat,
186 RoutingWorkload, SynthesizeRequest, SynthesizeResult, ThinkingMode, ToolCall,
187 TranscribeRequest, TranscribeResult, VideoMode,
188};
189#[derive(Debug, Clone, PartialEq, Eq)]
197pub struct InferenceRetryProgress {
198 pub model: String,
200 pub attempt: u32,
202 pub reason: &'static str,
204 pub backoff_ms: u64,
206}
207
208#[derive(Error, Debug)]
209pub enum InferenceError {
210 #[error("model not found: {0}")]
211 ModelNotFound(String),
212
213 #[error("no eligible model remains after required exclusions: {excluded_models}")]
217 NoEligibleModel { excluded_models: String },
218
219 #[error("model download failed: {0}")]
220 DownloadFailed(String),
221
222 #[error("inference failed: {0}")]
223 InferenceFailed(String),
224
225 #[error("classify: the model's reply {reply:?} {reason}")]
229 ClassifyNoAnswer { reply: String, reason: String },
230
231 #[error("classify: invalid labels: {0}")]
235 InvalidClassifyLabels(String),
236
237 #[error("catalog precondition mismatch: {detail}")]
242 CatalogPreconditionMismatch { detail: String },
243
244 #[error("controlled inference termination confirmed")]
249 ControlledTermination,
250
251 #[error(transparent)]
252 ModelManagement(#[from] model_management::ModelManagementError),
253
254 #[error("{recovery}")]
257 LocalResourceBlocked {
258 preflight: resource_policy::LocalLoadPreflight,
259 recovery: String,
260 },
261
262 #[error("transient remote failure after retries (status={status:?}): {message}")]
271 Transient {
272 status: Option<u16>,
273 message: String,
274 },
275
276 #[error("deadline exceeded: the caller's {applied_ms} ms infer deadline elapsed after {elapsed_ms} ms; last attempt: {last_error}")]
284 DeadlineExceeded {
285 applied_ms: u64,
286 elapsed_ms: u64,
287 last_error: String,
288 },
289
290 #[error("mode {mode} not implemented on backend {backend}: {reason}")]
295 UnsupportedMode {
296 mode: &'static str,
297 backend: &'static str,
298 reason: &'static str,
299 },
300
301 #[error("{provider} account rejected the request (HTTP {status}): {message}")]
316 ProviderAccount {
317 provider: String,
318 status: u16,
319 message: String,
320 },
321
322 #[error("inference failed: {message}")]
331 ProviderKeyMissing {
332 provider: String,
333 model: String,
334 env_vars: Vec<String>,
337 message: String,
340 },
341
342 #[error("no credential for proprietary provider '{provider}' (model {model}): {detail}")]
359 CredentialUnavailable {
360 provider: String,
361 model: String,
362 reason: CredentialFailure,
364 detail: String,
366 },
367
368 #[error("{provider} account has no workspace yet: {detail}")]
385 WorkspaceRequired {
386 provider: String,
388 detail: String,
390 },
391
392 #[error("{provider} refused this request on content grounds{}: {message}",
412 content_refusal_detail(.kind.as_deref(), .code.as_deref()))]
413 ContentRefused {
414 provider: String,
415 kind: Option<String>,
417 code: Option<String>,
418 message: String,
419 },
420
421 #[error("{provider} gateway has no upstream configured for '{namespace}' (HTTP {status}): {message}")]
439 GatewayUnconfigured {
440 provider: String,
441 namespace: String,
442 status: u16,
443 message: String,
444 },
445
446 #[error("tokenization error: {0}")]
447 TokenizationError(String),
448
449 #[error("device error: {0}")]
450 DeviceError(String),
451
452 #[error("io error: {0}")]
453 Io(#[from] std::io::Error),
454}
455
456impl From<resource_policy::LocalAdmissionError> for InferenceError {
457 fn from(error: resource_policy::LocalAdmissionError) -> Self {
458 let recovery = match error.preflight.verdict {
459 resource_policy::LocalLoadVerdict::DisabledByPolicy => {
460 "Local model loading is disabled by the 0 GB allocation. Increase Local Models RAM in Settings, or choose a remote model. Downloads remain available.".to_string()
461 }
462 resource_policy::LocalLoadVerdict::ExceedsConfiguredCeiling => format!(
463 "This model needs about {} MB for this request, beyond the configured {} MB local-model allocation. Increase the allocation or choose a smaller model.",
464 error.preflight.estimated_incremental_mb,
465 error.preflight.configured_ceiling_mb
466 ),
467 resource_policy::LocalLoadVerdict::InsufficientLiveMemory => format!(
468 "Not enough memory is free to start this model while preserving CAR's {} MB emergency reserve. Close memory-heavy apps or choose a smaller model.",
469 error.preflight.emergency_reserve_mb
470 ),
471 resource_policy::LocalLoadVerdict::LiveMemoryUnknown => {
472 "CAR could not measure live memory. The static allocation fits, but current safety is unknown.".to_string()
473 }
474 resource_policy::LocalLoadVerdict::ModelMaintenance => {
475 "This local model is being removed or maintained. Wait for that operation to finish, then retry.".to_string()
476 }
477 resource_policy::LocalLoadVerdict::PendingTeardown => {
478 "CAR is still confirming that the previous local model process exited. Wait for teardown to finish, then retry.".to_string()
479 }
480 resource_policy::LocalLoadVerdict::Allowed => error.to_string(),
481 };
482 Self::LocalResourceBlocked {
483 preflight: error.preflight,
484 recovery,
485 }
486 }
487}
488
489fn error_counts_against_circuit_breaker(e: &InferenceError) -> bool {
526 failure_attribution(e).is_none()
527}
528
529fn content_refusal_detail(kind: Option<&str>, code: Option<&str>) -> String {
535 match (kind, code) {
536 (Some(k), Some(c)) => format!(" (type={k}, code={c})"),
537 (Some(k), None) => format!(" (type={k})"),
538 (None, Some(c)) => format!(" (code={c})"),
539 (None, None) => String::new(),
540 }
541}
542
543fn record_dispatch_failure(tracker: &mut OutcomeTracker, trace_id: &str, error: &InferenceError) {
549 match failure_attribution(error) {
550 Some(reason) => tracker.record_not_model_fault(trace_id, &error.to_string(), reason),
551 None => tracker.record_failure(trace_id, &error.to_string()),
552 }
553}
554
555fn failure_attribution(error: &InferenceError) -> Option<outcome::UnattributedReason> {
561 use outcome::UnattributedReason as R;
562 match error {
563 InferenceError::UnsupportedMode { .. }
564 | InferenceError::CatalogPreconditionMismatch { .. }
565 | InferenceError::InvalidClassifyLabels { .. } => Some(R::Capability),
567 InferenceError::ProviderAccount { .. }
568 | InferenceError::ProviderKeyMissing { .. }
569 | InferenceError::CredentialUnavailable { .. }
570 | InferenceError::WorkspaceRequired { .. }
571 | InferenceError::GatewayUnconfigured { .. } => Some(R::Account),
572 InferenceError::ContentRefused { .. } => Some(R::ContentRefused),
573 InferenceError::LocalResourceBlocked { .. } => Some(R::ResourceBlocked),
574 InferenceError::ControlledTermination => Some(R::Cancelled),
575 InferenceError::ModelNotFound { .. }
576 | InferenceError::NoEligibleModel { .. }
577 | InferenceError::DownloadFailed { .. }
578 | InferenceError::InferenceFailed { .. }
579 | InferenceError::ClassifyNoAnswer { .. }
580 | InferenceError::ModelManagement { .. }
581 | InferenceError::Transient { .. }
582 | InferenceError::DeadlineExceeded { .. }
583 | InferenceError::TokenizationError { .. }
584 | InferenceError::DeviceError { .. }
585 | InferenceError::Io { .. } => None,
586 }
587}
588
589#[cfg(any(
595 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
596 all(target_os = "ios", target_arch = "aarch64")
597))]
598fn send_stream_failure_blocking(
599 tx: &tokio::sync::mpsc::Sender<stream::StreamEvent>,
600 error: &InferenceError,
601) {
602 if let Some(reason) = failure_attribution(error) {
603 let _ = tx.blocking_send(stream::StreamEvent::Attribution(reason));
604 }
605 let _ = tx.blocking_send(stream::StreamEvent::Error(error.to_string()));
606}
607
608fn error_ends_fallback_chain(e: &InferenceError) -> bool {
630 if matches!(e, InferenceError::ContentRefused { provider, .. } if provider == "apple") {
637 return false;
638 }
639 matches!(
640 e,
641 InferenceError::ContentRefused { .. }
642 | InferenceError::CatalogPreconditionMismatch { .. }
643 | InferenceError::ControlledTermination
644 | InferenceError::WorkspaceRequired { .. }
650 )
651}
652
653fn apply_exhaustion_recovery_hint(underlying: InferenceError) -> InferenceError {
664 if matches!(
665 underlying,
666 InferenceError::ContentRefused { .. }
667 | InferenceError::CatalogPreconditionMismatch { .. }
668 | InferenceError::ControlledTermination
669 | InferenceError::WorkspaceRequired { .. }
674 ) {
675 return underlying;
676 }
677 let underlying_str = underlying.to_string();
678 match no_backend_recovery_hint(&underlying_str)
679 .or_else(|| auth_expired_recovery_hint(&underlying_str))
680 {
681 Some(msg) => InferenceError::InferenceFailed(msg),
682 None => underlying,
683 }
684}
685
686const AUTH_LOGIN_MARKER: &str = "auth login";
687const AUTH_STORE_UNREADABLE_MARKER: &str = "credential store unreadable";
688const AUTH_ENV_MISSING_MARKER: &str = "credential environment variable missing";
689
690const AUTH_FAILURE_MESSAGE_MARKERS: &[&str] = &[
695 "no credential for proprietary",
696 AUTH_LOGIN_MARKER,
697 "session has expired",
698 "cannot read parslee credentials",
699 AUTH_STORE_UNREADABLE_MARKER,
700 AUTH_ENV_MISSING_MARKER,
701 "credential was rejected",
707 "repair its provider login",
708];
709
710#[derive(Debug, Clone, PartialEq, Eq)]
720struct RouteCredentialFailure {
721 summary: String,
722 source_error: String,
723}
724
725fn parslee_signed_out_route_failure() -> RouteCredentialFailure {
726 let summary = "Parslee login is absent — run `car auth login` before retrying".to_string();
727 RouteCredentialFailure {
728 source_error: summary.clone(),
729 summary,
730 }
731}
732
733fn route_credential_failure(
734 candidate: &str,
735 error: &InferenceError,
736 promote_missing_credential: bool,
737) -> Option<String> {
738 match error {
739 InferenceError::CredentialUnavailable {
740 provider,
741 reason,
742 detail,
743 ..
744 } => {
745 let provider_name = if provider.eq_ignore_ascii_case("parslee") {
746 "Parslee".to_string()
747 } else {
748 provider.clone()
749 };
750 let summary = match reason {
751 CredentialFailure::Expired { .. } => format!(
752 "{provider_name} login expired for `{candidate}` — run `car auth login`"
753 ),
754 CredentialFailure::SignedOut => format!(
755 "{provider_name} login is absent for `{candidate}` — run `car auth login`"
756 ),
757 CredentialFailure::StoreUnreadable => format!(
758 "{provider_name} {AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
759 ),
760 CredentialFailure::EnvVarMissing { .. } if !promote_missing_credential => {
761 return None;
762 }
763 CredentialFailure::EnvVarMissing { env_var } => format!(
764 "{provider_name} {AUTH_ENV_MISSING_MARKER}: `{env_var}` for explicitly requested `{candidate}` — {detail}"
765 ),
766 CredentialFailure::RaceRetryable => return None,
769 };
770 Some(summary)
771 }
772 InferenceError::ProviderAccount {
773 provider, status, ..
774 } if matches!(*status, 401 | 403) => {
775 if provider.eq_ignore_ascii_case("parslee") {
776 Some(format!(
777 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
778 ))
779 } else {
780 Some(format!(
781 "{provider} credential was rejected for `{candidate}` (HTTP {status})"
782 ))
783 }
784 }
785 _ => {
786 let rendered = error.to_string();
787 let lower = rendered.to_ascii_lowercase();
788 if is_auth_rejection_message(&rendered) {
789 if candidate
790 .split_once('/')
791 .is_some_and(|(provider, _)| provider.eq_ignore_ascii_case("parslee"))
792 {
793 Some(format!(
794 "Parslee login expired or was rejected for `{candidate}` — run `car auth login`"
795 ))
796 } else {
797 Some(format!(
798 "credential expired or was rejected for `{candidate}` — repair its provider login"
799 ))
800 }
801 } else if lower.contains(AUTH_STORE_UNREADABLE_MARKER)
802 || (promote_missing_credential && lower.contains("keychain lookup failed"))
803 {
804 Some(format!(
805 "{AUTH_STORE_UNREADABLE_MARKER} for `{candidate}` — unlock the credential store, then retry"
806 ))
807 } else {
808 None
809 }
810 }
811 }
812}
813
814fn record_route_credential_failure(
815 slot: &mut Option<RouteCredentialFailure>,
816 candidate: &str,
817 error: &InferenceError,
818 promote_missing_credential: bool,
819) {
820 if let Some(summary) = route_credential_failure(candidate, error, promote_missing_credential) {
821 *slot = Some(RouteCredentialFailure {
826 summary,
827 source_error: error.to_string(),
828 });
829 }
830}
831
832fn chain_includes_parslee_route<'a>(
837 mut resolve: impl FnMut(&str) -> Option<&'a ModelSchema>,
838 chain: &[String],
839) -> bool {
840 chain.iter().any(|candidate| {
841 resolve(candidate).is_some_and(|schema| schema.provider.eq_ignore_ascii_case("parslee"))
842 })
843}
844
845fn apply_route_failure_context(
865 underlying: InferenceError,
866 credential: Option<&RouteCredentialFailure>,
867) -> InferenceError {
868 if matches!(
869 underlying,
870 InferenceError::ContentRefused { .. }
871 | InferenceError::CatalogPreconditionMismatch { .. }
872 | InferenceError::ControlledTermination
873 | InferenceError::WorkspaceRequired { .. }
881 ) {
882 return underlying;
883 }
884 let Some(credential) = credential else {
885 return apply_exhaustion_recovery_hint(underlying);
886 };
887 let rendered = underlying.to_string();
888 let summary = credential.summary.as_str();
889 let setup_hint = no_backend_recovery_hint(&rendered);
890 let connective = if rendered == credential.source_error {
891 "provider detail"
892 } else {
893 "fallback then failed"
894 };
895 let augment = |field: String| match &setup_hint {
896 Some(hint) => format!("{summary}; {hint}"),
898 None => format!("{summary}; {connective}: {field}"),
899 };
900 match underlying {
901 InferenceError::InferenceFailed(message) => {
902 InferenceError::InferenceFailed(augment(message))
903 }
904 InferenceError::ProviderKeyMissing {
905 provider,
906 model,
907 env_vars,
908 message,
909 } => InferenceError::ProviderKeyMissing {
910 provider,
911 model,
912 env_vars,
913 message: augment(message),
914 },
915 InferenceError::Transient { status, message } => InferenceError::Transient {
916 status,
917 message: augment(message),
918 },
919 InferenceError::ProviderAccount {
920 provider,
921 status,
922 message,
923 } => InferenceError::ProviderAccount {
924 provider,
925 status,
926 message: augment(message),
927 },
928 InferenceError::CredentialUnavailable {
929 provider,
930 model,
931 reason,
932 detail,
933 } => InferenceError::CredentialUnavailable {
934 provider,
935 model,
936 reason,
937 detail: augment(detail),
938 },
939 InferenceError::GatewayUnconfigured {
940 provider,
941 namespace,
942 status,
943 message,
944 } => InferenceError::GatewayUnconfigured {
945 provider,
946 namespace,
947 status,
948 message: augment(message),
949 },
950 InferenceError::LocalResourceBlocked {
951 preflight,
952 recovery,
953 } => InferenceError::LocalResourceBlocked {
954 preflight,
955 recovery: augment(recovery),
956 },
957 _ => InferenceError::InferenceFailed(augment(rendered)),
960 }
961}
962
963#[derive(Debug, Clone, PartialEq, Eq)]
972pub enum CredentialFailure {
973 Expired { expires_at: u64 },
980 SignedOut,
983 StoreUnreadable,
987 EnvVarMissing { env_var: String },
989 RaceRetryable,
992}
993
994#[derive(Debug, Clone, Copy, PartialEq, Eq)]
996pub enum Device {
997 Cpu,
998 Metal,
999 Cuda(usize), }
1001
1002impl Device {
1003 pub fn auto() -> Self {
1011 #[cfg(all(target_os = "macos", feature = "metal"))]
1012 {
1013 return Device::Metal;
1014 }
1015 #[cfg(all(
1016 any(target_os = "linux", target_os = "windows"),
1017 target_arch = "x86_64",
1018 not(car_skip_cuda)
1019 ))]
1020 {
1021 return Device::Cuda(0);
1022 }
1023 #[cfg(not(any(
1024 all(target_os = "macos", feature = "metal"),
1025 all(
1026 any(target_os = "linux", target_os = "windows"),
1027 target_arch = "x86_64",
1028 not(car_skip_cuda)
1029 )
1030 )))]
1031 {
1032 Device::Cpu
1033 }
1034 }
1035}
1036
1037#[derive(Debug, Clone)]
1039pub struct InferenceConfig {
1040 pub models_dir: std::path::PathBuf,
1046 pub state_root: std::path::PathBuf,
1055 pub device: Option<Device>,
1057 pub generation_model: String,
1059 pub preferred_generation_model: Option<String>,
1061 pub embedding_model: String,
1063 pub preferred_embedding_model: Option<String>,
1065 pub classification_model: String,
1067 pub preferred_classification_model: Option<String>,
1069}
1070
1071impl Default for InferenceConfig {
1072 fn default() -> Self {
1073 let models_dir = default_models_dir();
1090
1091 let hw = HardwareInfo::detect();
1092
1093 Self {
1094 models_dir,
1095 state_root: car_home::root_or_relative(),
1096 device: None,
1097 generation_model: hw.recommended_model,
1098 preferred_generation_model: None,
1099 embedding_model: "Qwen3-Embedding-0.6B".to_string(),
1100 preferred_embedding_model: None,
1101 classification_model: "Qwen3-0.6B".to_string(),
1102 preferred_classification_model: None,
1103 }
1104 }
1105}
1106
1107impl InferenceConfig {
1108 pub fn state_models_dir(&self) -> std::path::PathBuf {
1117 self.state_root.join("models")
1118 }
1119}
1120
1121pub fn default_models_dir() -> std::path::PathBuf {
1128 dirs_next()
1129 .unwrap_or_else(|| std::path::PathBuf::from("."))
1130 .join(".car")
1131 .join("models")
1132}
1133
1134fn dirs_next() -> Option<std::path::PathBuf> {
1135 std::env::var_os("HOME")
1139 .or_else(|| std::env::var_os("USERPROFILE"))
1140 .map(std::path::PathBuf::from)
1141}
1142
1143fn model_source_identity(schema: &ModelSchema) -> &str {
1144 match &schema.source {
1145 ModelSource::Local { hf_repo, .. }
1146 | ModelSource::Mlx { hf_repo, .. }
1147 | ModelSource::ManagedVllmMlx { hf_repo, .. } => hf_repo,
1148 ModelSource::WhisperCpp { model } | ModelSource::CodexCli { model } => model,
1149 _ => &schema.id,
1150 }
1151}
1152
1153#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1155pub struct TokenUsage {
1156 pub prompt_tokens: u64,
1166 pub completion_tokens: u64,
1168 pub total_tokens: u64,
1170 pub context_window: u64,
1172 #[serde(default)]
1177 pub cache_read_input_tokens: u64,
1178 #[serde(default)]
1183 pub cache_creation_input_tokens: u64,
1184}
1185
1186#[derive(
1188 Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq, schemars::JsonSchema,
1189)]
1190pub struct InferenceModelIdentity {
1191 #[serde(default)]
1194 pub requested_model_id: Option<String>,
1195 #[serde(default)]
1209 pub resolved_model_id: String,
1210 #[serde(default)]
1212 pub row_digest: String,
1213 #[serde(default)]
1215 pub catalog_revision: String,
1216}
1217
1218#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1220pub struct InferenceResult {
1221 pub text: String,
1223 pub tool_calls: Vec<crate::tasks::generate::ToolCall>,
1225 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1232 pub bounding_boxes: Vec<crate::tasks::grounding::BoundingBox>,
1233 pub trace_id: String,
1235 pub model_used: String,
1238 #[serde(flatten)]
1241 pub model_identity: InferenceModelIdentity,
1242 pub latency_ms: u64,
1244 #[serde(default)]
1256 pub time_to_first_token_ms: Option<u64>,
1257 pub usage: Option<TokenUsage>,
1276 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1289 pub provider_output_items: Vec<serde_json::Value>,
1290 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1298 pub thinking: Vec<crate::tasks::generate::ThinkingBlock>,
1299 #[serde(default)]
1315 pub stop_reason: Option<String>,
1316 #[serde(default, skip_serializing_if = "Option::is_none")]
1325 pub auth_fallback_from: Option<String>,
1326 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
1333 pub local_last_resort: bool,
1334 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1354 pub fallback_from: Vec<FallbackFrom>,
1355}
1356
1357#[derive(
1359 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1360)]
1361pub struct FallbackFrom {
1362 pub candidate: String,
1364 pub reason: FallbackReason,
1365}
1366
1367#[derive(
1375 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
1376)]
1377#[serde(rename_all = "snake_case")]
1378pub enum FallbackReason {
1379 CredentialRejected,
1388 CredentialAbsent,
1391 RateLimited,
1394 QuotaExhausted,
1400 TimedOut,
1408 Failed,
1414}
1415
1416pub fn classify_fallback_reason(error: &InferenceError) -> FallbackReason {
1449 use FallbackReason as R;
1450 match error {
1451 InferenceError::ProviderAccount { status, .. } => match status {
1454 402 => R::QuotaExhausted,
1455 _ => R::CredentialRejected,
1456 },
1457 InferenceError::CredentialUnavailable { reason, .. } => match reason {
1459 CredentialFailure::Expired { .. } => R::CredentialRejected,
1461 CredentialFailure::SignedOut | CredentialFailure::EnvVarMissing { .. } => {
1463 R::CredentialAbsent
1464 }
1465 CredentialFailure::StoreUnreadable | CredentialFailure::RaceRetryable => R::Failed,
1468 },
1469 InferenceError::Transient {
1470 status: Some(429), ..
1471 } => R::RateLimited,
1472 InferenceError::Transient { status: None, .. } => R::Failed,
1479 InferenceError::Transient { .. } => R::Failed,
1480 InferenceError::DeadlineExceeded { .. } => R::TimedOut,
1483 InferenceError::InferenceFailed(msg)
1484 | InferenceError::ProviderKeyMissing { message: msg, .. } => classify_untyped_failure(msg),
1485 _ => R::Failed,
1486 }
1487}
1488
1489fn classify_untyped_failure(msg: &str) -> FallbackReason {
1497 use FallbackReason as R;
1498 if let Some(status) = parse_api_returned_status(msg) {
1501 return match status {
1502 401 | 403 => R::CredentialRejected,
1503 402 => R::QuotaExhausted,
1504 429 => R::RateLimited,
1505 408 | 504 => R::TimedOut,
1506 _ => R::Failed,
1507 };
1508 }
1509 let l = msg.to_ascii_lowercase();
1510 if l.contains("org lookup failed") {
1513 return if l.contains("http 401") || l.contains("http 403") {
1514 R::CredentialRejected
1515 } else if l.contains("http 429") {
1516 R::RateLimited
1517 } else {
1518 R::Failed
1519 };
1520 }
1521 if l.contains("authentication required")
1522 || l.contains("invalid_grant")
1523 || l.contains("token expired")
1524 {
1525 return R::CredentialRejected;
1526 }
1527 if l.contains("no credential for proprietary") || l.contains("no api key") {
1528 return R::CredentialAbsent;
1529 }
1530 if l.contains("too many requests") || l.contains("rate limit") {
1531 return R::RateLimited;
1532 }
1533 if l.contains("timed out") || l.contains("deadline exceeded") {
1534 return R::TimedOut;
1535 }
1536 R::Failed
1537}
1538
1539pub fn record_fallback_from(hops: &mut Vec<FallbackFrom>, candidate: &str, error: &InferenceError) {
1545 hops.push(FallbackFrom {
1546 candidate: candidate.to_string(),
1547 reason: classify_fallback_reason(error),
1548 });
1549}
1550
1551pub struct TrackedStream {
1558 pub model_used: String,
1560 pub trace_id: String,
1562 pub events: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
1564}
1565
1566struct AbortOnDropTask<T>(Option<tokio::task::JoinHandle<T>>);
1567
1568impl<T> AbortOnDropTask<T> {
1569 async fn join(mut self) -> Result<T, tokio::task::JoinError> {
1570 self.0.take().expect("owned task handle available").await
1571 }
1572}
1573
1574impl<T> Drop for AbortOnDropTask<T> {
1575 fn drop(&mut self) {
1576 if let Some(task) = self.0.take() {
1577 task.abort();
1578 }
1579 }
1580}
1581
1582tokio::task_local! {
1583 static PINNED_REGISTRY: (usize, Arc<UnifiedRegistry>);
1587}
1588
1589fn bound_model_identity(
1590 snapshot: &CatalogSnapshot,
1591 requested_model_id: Option<&str>,
1592 resolved_model_id: &str,
1593) -> Result<InferenceModelIdentity, InferenceError> {
1594 let row = snapshot.model_by_exact_id(resolved_model_id).ok_or_else(|| {
1595 InferenceError::InferenceFailed(format!(
1596 "resolved model `{resolved_model_id}` was absent from the catalog snapshot bound to this request"
1597 ))
1598 })?;
1599 Ok(InferenceModelIdentity {
1600 requested_model_id: requested_model_id.map(str::to_string),
1601 resolved_model_id: row.model.id.clone(),
1602 row_digest: row.row_digest.clone(),
1603 catalog_revision: snapshot.catalog_revision.clone(),
1604 })
1605}
1606
1607fn validate_expected_catalog_revision(
1608 req: &GenerateRequest,
1609 snapshot: &CatalogSnapshot,
1610) -> Result<(), InferenceError> {
1611 if let Some(expected) = req.expected_catalog_revision.as_deref() {
1612 if expected != snapshot.catalog_revision {
1613 return Err(InferenceError::CatalogPreconditionMismatch {
1614 detail: format!(
1615 "expected catalog revision {expected}, got {}",
1616 snapshot.catalog_revision
1617 ),
1618 });
1619 }
1620 }
1621 Ok(())
1622}
1623
1624fn validate_expected_catalog_row(
1625 req: &GenerateRequest,
1626 snapshot: &CatalogSnapshot,
1627 resolved_model_id: &str,
1628) -> Result<(), InferenceError> {
1629 let Some(expected) = req.expected_row_digest.as_deref() else {
1630 return Ok(());
1631 };
1632 let row = snapshot
1633 .model_by_exact_id(resolved_model_id)
1634 .ok_or_else(|| InferenceError::CatalogPreconditionMismatch {
1635 detail: format!(
1636 "resolved model `{resolved_model_id}` is absent from the bound catalog snapshot"
1637 ),
1638 })?;
1639 if expected != row.row_digest {
1640 return Err(InferenceError::CatalogPreconditionMismatch {
1641 detail: format!(
1642 "expected row digest {expected} for `{resolved_model_id}`, got {}",
1643 row.row_digest
1644 ),
1645 });
1646 }
1647 Ok(())
1648}
1649
1650const EXACT_MODEL_ID_PREFIX: &str = "\0car-exact-model-id:";
1651
1652pub fn pin_exact_model_id(req: &mut GenerateRequest, model_id: String) -> Result<(), String> {
1655 if req.model.is_some() {
1656 return Err("`model` and `model_id` are mutually exclusive".to_string());
1657 }
1658 if model_id.trim().is_empty() {
1659 return Err("`model_id` must be a non-empty immutable id".to_string());
1660 }
1661 req.model = Some(format!("{EXACT_MODEL_ID_PREFIX}{model_id}"));
1662 req.params.strict_model = true;
1663 Ok(())
1664}
1665
1666fn default_system_one_classifier<'a>(
1671 schemas: impl Iterator<Item = &'a ModelSchema>,
1672) -> Option<String> {
1673 schemas
1674 .filter(|schema| {
1675 schema.has_capability(ModelCapability::Classify)
1676 && schema.available_now()
1677 && matches!(
1678 &schema.source,
1679 ModelSource::Proprietary { protocol, .. }
1680 if protocol.wire == ProprietaryWire::SystemOne
1681 )
1682 })
1683 .map(|schema| schema.id.clone())
1684 .min()
1685}
1686
1687pub fn exact_pinned_model_id(req: &GenerateRequest) -> Option<&str> {
1688 req.model
1689 .as_deref()
1690 .and_then(|model| model.strip_prefix(EXACT_MODEL_ID_PREFIX))
1691}
1692
1693fn auto_thinking_budget(
1702 is_code_intent: bool,
1703 is_complex: bool,
1704 supports_thinking: bool,
1705) -> Option<usize> {
1706 if !supports_thinking {
1707 return None;
1708 }
1709 if is_code_intent {
1710 Some(24_000)
1711 } else if is_complex {
1712 Some(8_000)
1713 } else {
1714 None
1715 }
1716}
1717
1718fn is_explicit_code_intent(intent: Option<&intent::IntentHint>) -> bool {
1728 intent.and_then(|h| h.task) == Some(intent::TaskHint::Code)
1729}
1730
1731fn should_append_local_last_resort(chain_has_local: bool, strict_model: bool) -> bool {
1736 !chain_has_local && !strict_model
1737}
1738
1739fn resolved_max_tokens(requested: usize, schema: &ModelSchema) -> usize {
1763 if requested != crate::tasks::generate::DEFAULT_MAX_TOKENS
1764 || schema.decodes_in_process()
1765 || schema.is_codex_cli()
1766 {
1767 return requested;
1768 }
1769 schema.effective_max_output()
1770}
1771
1772const DEFAULT_LOCAL_DECODE_TIMEOUT_SECS: u64 = 300;
1785
1786pub const LOCAL_DECODE_TIMEOUT_STOP_REASON: &str = "local_decode_timeout";
1793
1794#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1802const LOCAL_DECODE_HEARTBEAT_SECS: u64 = 10;
1803
1804fn parse_decode_timeout(raw: Option<&str>) -> Option<std::time::Duration> {
1807 let secs = match raw {
1808 Some(v) => v
1809 .trim()
1810 .parse::<u64>()
1811 .unwrap_or(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS),
1812 None => DEFAULT_LOCAL_DECODE_TIMEOUT_SECS,
1813 };
1814 (secs > 0).then(|| std::time::Duration::from_secs(secs))
1815}
1816
1817#[cfg_attr(
1830 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
1831 allow(dead_code)
1832)]
1833fn deadline_exceeded(elapsed: std::time::Duration, timeout: Option<std::time::Duration>) -> bool {
1834 timeout.is_some_and(|limit| elapsed >= limit)
1835}
1836
1837#[cfg_attr(
1843 not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))),
1844 allow(dead_code)
1845)]
1846fn heartbeat_due(
1847 elapsed: std::time::Duration,
1848 last: std::time::Duration,
1849 interval: std::time::Duration,
1850) -> bool {
1851 elapsed.saturating_sub(last) >= interval
1852}
1853
1854#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1858enum EmptyPassAction {
1859 RetryWithoutThinking,
1862 FailDecodeCeiling,
1866 Accept,
1868}
1869
1870fn classify_empty_pass(
1878 recover: bool,
1879 stop_reason: Option<&str>,
1880 text: &str,
1881 tool_calls_empty: bool,
1882) -> EmptyPassAction {
1883 if !text.trim().is_empty() || !tool_calls_empty {
1884 return EmptyPassAction::Accept;
1885 }
1886 if stop_reason == Some(LOCAL_DECODE_TIMEOUT_STOP_REASON) {
1887 return EmptyPassAction::FailDecodeCeiling;
1888 }
1889 if recover {
1890 return EmptyPassAction::RetryWithoutThinking;
1891 }
1892 EmptyPassAction::Accept
1893}
1894
1895fn local_decode_timeout() -> Option<std::time::Duration> {
1896 parse_decode_timeout(
1897 std::env::var("CAR_LOCAL_DECODE_TIMEOUT_SECS")
1898 .ok()
1899 .as_deref(),
1900 )
1901}
1902
1903impl InferenceResult {
1904 pub fn served_model_id(&self) -> &str {
1909 if self.model_identity.resolved_model_id.is_empty() {
1910 &self.model_used
1911 } else {
1912 &self.model_identity.resolved_model_id
1913 }
1914 }
1915
1916 pub fn has_tool_calls(&self) -> bool {
1918 !self.tool_calls.is_empty()
1919 }
1920
1921 pub fn was_truncated(&self) -> bool {
1933 matches!(
1934 self.stop_reason.as_deref(),
1935 Some(
1936 "length"
1937 | "max_tokens"
1938 | "max_output_tokens"
1939 | "MAX_TOKENS"
1940 | crate::LOCAL_DECODE_TIMEOUT_STOP_REASON
1941 )
1942 )
1943 }
1944
1945 pub fn append_assistant_history(
1954 &self,
1955 messages: &mut Vec<crate::tasks::generate::Message>,
1956 tool_calls: Vec<crate::tasks::generate::ToolCall>,
1957 ) {
1958 if !self.provider_output_items.is_empty() {
1959 messages.push(crate::tasks::generate::Message::ProviderOutputItems {
1960 protocol: crate::protocol::OPENAI_RESPONSES_PROTOCOL.to_string(),
1961 items: self.provider_output_items.clone(),
1962 });
1963 }
1964 messages.push(crate::tasks::generate::Message::Assistant {
1965 content: self.text.clone(),
1966 tool_calls,
1967 thinking: self.thinking.clone(),
1968 model_id: Some(self.served_model_id().to_string()),
1969 local_last_resort: self.local_last_resort,
1970 });
1971 }
1972}
1973
1974#[derive(Debug, Clone, Serialize)]
1975pub struct SpeechRuntimeHealth {
1976 pub root: PathBuf,
1977 pub installed: bool,
1978 pub python: PathBuf,
1979 pub stt_command: PathBuf,
1980 pub tts_command: PathBuf,
1981 pub configured_python: Option<String>,
1982 pub detected_python: Option<String>,
1983}
1984
1985#[derive(Debug, Clone, Serialize)]
1986pub struct SpeechModelHealth {
1987 pub id: String,
1988 pub name: String,
1989 pub provider: String,
1990 pub capability: ModelCapability,
1991 pub is_local: bool,
1992 pub available: bool,
1993 pub cached: bool,
1994 pub selected_by_default: bool,
1995 pub source: String,
1996}
1997
1998#[derive(Debug, Clone, Serialize)]
1999pub struct SpeechHealthReport {
2000 pub runtime: SpeechRuntimeHealth,
2001 pub local_models: Vec<SpeechModelHealth>,
2002 pub remote_models: Vec<SpeechModelHealth>,
2003 pub elevenlabs_configured: bool,
2004 pub prefer_local: bool,
2005 pub allow_remote_fallback: bool,
2006 pub preferred_local_stt: Option<String>,
2007 pub preferred_local_tts: Option<String>,
2008 pub preferred_remote_stt: Option<String>,
2009 pub preferred_remote_tts: Option<String>,
2010 pub local_stt_default: Option<String>,
2011 pub local_tts_default: Option<String>,
2012 pub remote_stt_default: Option<String>,
2013 pub remote_tts_default: Option<String>,
2014}
2015
2016#[derive(Debug, Clone, Serialize)]
2017pub struct ModelDefaultHealth {
2018 pub capability: ModelCapability,
2019 pub configured_model: String,
2020 pub available: bool,
2021 pub is_local: bool,
2022 pub provider: Option<String>,
2023}
2024
2025#[derive(Debug, Clone, Serialize)]
2026pub struct ModelProviderHealth {
2027 pub provider: String,
2028 pub configured: bool,
2029 pub local_models: usize,
2030 pub remote_models: usize,
2031 pub available_models: usize,
2032 pub capabilities: Vec<ModelCapability>,
2033}
2034
2035#[derive(Debug, Clone, Serialize)]
2036pub struct ModelCapabilityHealth {
2037 pub capability: ModelCapability,
2038 pub total_models: usize,
2039 pub available_models: usize,
2040 pub local_available_models: usize,
2041 pub remote_available_models: usize,
2042}
2043
2044#[derive(Debug, Clone, Serialize)]
2045pub struct RoutingScenarioHealth {
2046 pub name: String,
2047 pub workload: RoutingWorkload,
2048 pub task_family: String,
2049 pub has_tools: bool,
2050 pub has_vision: bool,
2051 pub prefer_local: bool,
2052 pub quality_first_cold_start: bool,
2053 pub bootstrap_min_task_observations: u64,
2054 pub bootstrap_quality_floor: f64,
2055 pub model_id: String,
2056 pub model_name: String,
2057 pub reason: String,
2058 pub strategy: RoutingStrategy,
2059}
2060
2061#[derive(Debug, Clone, Serialize)]
2062pub struct ModelBenchmarkPriorHealth {
2063 pub model_id: String,
2064 pub model_name: Option<String>,
2065 pub overall_score: f64,
2066 pub overall_latency_ms: Option<f64>,
2067 pub task_scores: std::collections::HashMap<String, f64>,
2068 pub task_latency_ms: std::collections::HashMap<String, f64>,
2069 pub source_path: PathBuf,
2070}
2071
2072#[derive(Debug, Clone, Serialize)]
2073pub struct ModelHealthReport {
2074 pub total_models: usize,
2075 pub available_models: usize,
2076 pub local_models: usize,
2077 pub remote_models: usize,
2078 pub defaults: Vec<ModelDefaultHealth>,
2079 pub providers: Vec<ModelProviderHealth>,
2080 pub capabilities: Vec<ModelCapabilityHealth>,
2081 pub routing_prefer_local: bool,
2082 pub routing_quality_first_cold_start: bool,
2083 pub routing_min_observations: u64,
2084 pub routing_bootstrap_min_task_observations: u64,
2085 pub routing_bootstrap_quality_floor: f64,
2086 pub routing_quality_weight: f64,
2087 pub routing_latency_weight: f64,
2088 pub routing_cost_weight: f64,
2089 pub routing_scenarios: Vec<RoutingScenarioHealth>,
2090 pub benchmark_priors: Vec<ModelBenchmarkPriorHealth>,
2091 pub speech: SpeechHealthReport,
2092}
2093
2094#[derive(Debug, Clone, Serialize)]
2095pub struct SpeechInstallReport {
2096 pub name: String,
2097 pub hf_repo: String,
2098 pub snapshot_path: PathBuf,
2099 pub files_downloaded: usize,
2100}
2101
2102#[derive(Debug, Clone, Serialize)]
2103pub struct SpeechSmokePathReport {
2104 pub path: String,
2105 pub tts_model: String,
2106 pub stt_model: String,
2107 pub audio_path: PathBuf,
2108 pub transcript: String,
2109}
2110
2111#[derive(Debug, Clone, Serialize, Default)]
2112pub struct SpeechSmokeReport {
2113 pub local: Option<SpeechSmokePathReport>,
2114 pub remote: Option<SpeechSmokePathReport>,
2115 pub skipped: Vec<String>,
2116}
2117
2118#[derive(Debug, Clone, Serialize, Default)]
2119pub struct SpeechPolicy {
2120 pub prefer_local: bool,
2121 pub allow_remote_fallback: bool,
2122 pub preferred_local_stt: Option<String>,
2123 pub preferred_local_tts: Option<String>,
2124 pub preferred_remote_stt: Option<String>,
2125 pub preferred_remote_tts: Option<String>,
2126}
2127
2128fn render_for_local_backend(mut req: GenerateRequest) -> GenerateRequest {
2137 let has_msgs = req.messages.as_ref().is_some_and(|m| !m.is_empty());
2138 let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
2139 if has_msgs || has_tools {
2140 req.prompt = tasks::generate::render_chat_prompt(&req);
2141 req.messages = None;
2142 req.tools = None;
2143 }
2144 req
2145}
2146
2147#[cfg(test)]
2148static UNRETIRED_RELEASE_WARNING_COUNT: std::sync::atomic::AtomicUsize =
2149 std::sync::atomic::AtomicUsize::new(0);
2150
2151#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2165fn foundation_models_usage(
2166 instructions: Option<&str>,
2167 prompt: &str,
2168 completion: &str,
2169 context_window: u64,
2170) -> Option<crate::TokenUsage> {
2171 let (prompt_tokens, completion_tokens) =
2172 crate::backend::foundation_models::count_tokens(instructions, prompt, completion)?;
2173 Some(crate::TokenUsage {
2174 prompt_tokens,
2175 completion_tokens,
2176 total_tokens: prompt_tokens + completion_tokens,
2177 context_window,
2178 ..Default::default()
2180 })
2181}
2182
2183pub struct InferenceEngine {
2184 pub config: InferenceConfig,
2185 unified: std::sync::RwLock<Arc<UnifiedRegistry>>,
2191 registry_write: std::sync::Mutex<()>,
2194 pub adaptive_router: AdaptiveRouter,
2196 pub outcome_tracker: Arc<RwLock<OutcomeTracker>>,
2198 last_outcome_flush: Arc<std::sync::Mutex<Option<Instant>>>,
2204 outcome_profiles_loaded: bool,
2208 ledger_io_lock: Arc<tokio::sync::Mutex<()>>,
2213 spend_limits: Arc<std::sync::RwLock<Option<SpendLimits>>>,
2219 lane_defaults_cache: Arc<std::sync::RwLock<crate::lane_defaults::LaneDefaults>>,
2224 external_pins: Arc<std::sync::RwLock<std::collections::BTreeMap<String, Vec<String>>>>,
2228 disk_pressured: Arc<std::sync::atomic::AtomicBool>,
2231 failed_upgrades: Arc<std::sync::Mutex<std::collections::HashMap<String, u64>>>,
2235 concierge_action_lock: Arc<tokio::sync::Mutex<()>>,
2241 concierge_action_seq: Arc<std::sync::atomic::AtomicU64>,
2245 remote_backend: RemoteBackend,
2247 model_management: model_management::ModelManagementStore,
2250 local_admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2252 _runtime_scope: Arc<ScopedInferenceRuntime>,
2257 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2258 resource_policy_evidence: Arc<std::sync::RwLock<resource_policy::ResourcePolicyLoadEvidence>>,
2262 model_budget: Arc<backend_cache::SharedModelBudget>,
2265 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2271 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2272 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2279 local_backends:
2280 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2281 pub registry: models::ModelRegistry,
2283 pub router: ModelRouter,
2284 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2285 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2286 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2287 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2288 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2289 speech_policy: SpeechPolicy,
2290 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2294}
2295
2296enum SpeechCandidateAdmission {
2297 Proceed(Option<resource_policy::LocalLoadReservation>),
2298 SkipBlocked(InferenceError),
2299 FailBlocked(InferenceError),
2300}
2301
2302struct ScopedInferenceRuntime {
2303 model_budget: Arc<backend_cache::SharedModelBudget>,
2304 resource_policy_generation: Arc<std::sync::atomic::AtomicU64>,
2305 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2306 text_backends: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
2307 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2308 local_backends:
2309 Arc<backend_cache::BackendCache<Box<dyn backend::local::LocalInferenceBackend>>>,
2310 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2311 backend: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
2312 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2313 embedding_backend: Arc<RwLock<Option<EmbeddingBackend>>>,
2314 speech_runtime: Arc<Mutex<Option<SpeechRuntime>>>,
2315 vllm_pool: Arc<vllm_pool::VllmServerPool>,
2316 #[cfg(test)]
2317 load_probe: Arc<backend_cache::BackendCache<()>>,
2318}
2319
2320fn scoped_inference_runtime_registry() -> &'static std::sync::Mutex<
2321 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2322> {
2323 static REGISTRY: std::sync::OnceLock<
2324 std::sync::Mutex<
2325 std::collections::HashMap<PathBuf, std::sync::Weak<ScopedInferenceRuntime>>,
2326 >,
2327 > = std::sync::OnceLock::new();
2328 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
2329}
2330
2331fn configured_cache_budget_mb(default_mb: u64) -> u64 {
2332 std::env::var("CAR_INFERENCE_MODEL_CACHE_MB")
2333 .ok()
2334 .and_then(|value| value.parse::<u64>().ok())
2335 .unwrap_or(default_mb)
2336}
2337
2338fn scoped_inference_runtime(
2339 state_root: &Path,
2340 configured_ceiling_mb: u64,
2341 admission: Arc<resource_policy::LocalAdmissionCoordinator>,
2342) -> Arc<ScopedInferenceRuntime> {
2343 let state_root = resource_policy::normalized_state_root_key(state_root);
2344 let mut registry = scoped_inference_runtime_registry()
2345 .lock()
2346 .unwrap_or_else(std::sync::PoisonError::into_inner);
2347 registry.retain(|_, runtime| runtime.strong_count() > 0);
2348 if let Some(runtime) = registry.get(&state_root).and_then(std::sync::Weak::upgrade) {
2349 runtime.model_budget.set_budget_bytes(
2350 configured_cache_budget_mb(configured_ceiling_mb).saturating_mul(1024 * 1024),
2351 );
2352 return runtime;
2353 }
2354
2355 let model_budget = backend_cache::SharedModelBudget::from_env_or(configured_ceiling_mb);
2356 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2357 let cache_idle = backend_cache::idle_ttl_from_env();
2358 let runtime = Arc::new(ScopedInferenceRuntime {
2359 model_budget: model_budget.clone(),
2360 resource_policy_generation: Arc::new(std::sync::atomic::AtomicU64::new(1)),
2361 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2362 text_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2363 model_budget.clone(),
2364 cache_idle,
2365 Some(admission.clone()),
2366 )),
2367 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2368 local_backends: Arc::new(backend_cache::BackendCache::from_shared_with_admission(
2369 model_budget.clone(),
2370 cache_idle,
2371 Some(admission.clone()),
2372 )),
2373 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2374 backend: Arc::new(RwLock::new(std::collections::HashMap::new())),
2375 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2376 embedding_backend: Arc::new(RwLock::new(None)),
2377 speech_runtime: Arc::new(Mutex::new(None)),
2378 vllm_pool: Arc::new(
2382 vllm_pool::VllmServerPool::with_admission(
2383 std::time::Duration::from_secs(
2384 std::env::var("CAR_VLLM_IDLE_SECS")
2385 .ok()
2386 .and_then(|value| value.parse().ok())
2387 .unwrap_or(300),
2388 ),
2389 admission,
2390 )
2391 .with_process_records(state_root.join("run").join("vllm-mlx")),
2392 ),
2393 #[cfg(test)]
2394 load_probe: Arc::new(backend_cache::BackendCache::new(1024)),
2395 });
2396 registry.insert(state_root, Arc::downgrade(&runtime));
2397 runtime
2398}
2399
2400impl InferenceEngine {
2401 fn requires_local_admission(schema: &ModelSchema) -> bool {
2402 schema.is_car_managed_vllm_mlx() || Self::supports_worker_offload(schema)
2403 }
2404
2405 fn reserve_in_outer_dispatch(schema: &ModelSchema) -> bool {
2411 Self::requires_local_admission(schema) && !schema.is_car_managed_vllm_mlx()
2412 }
2413
2414 fn supports_worker_offload(schema: &ModelSchema) -> bool {
2415 matches!(
2416 schema.source,
2417 ModelSource::Local { .. } | ModelSource::Mlx { .. }
2418 )
2419 }
2420
2421 pub fn local_model_retention(&self, model_id: &str) -> backend_cache::BackendRetention {
2424 if self.local_admission.is_resident(model_id) {
2425 backend_cache::BackendRetention::Resident
2426 } else {
2427 backend_cache::BackendRetention::Transient
2428 }
2429 }
2430 pub fn local_model_preflight(
2432 &self,
2433 model_id: &str,
2434 context_tokens: usize,
2435 ) -> Result<resource_policy::LocalLoadPreflight, InferenceError> {
2436 let schema = self
2437 .resolve_schema(model_id)
2438 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
2439 self.ensure_model_enabled(&schema.id)?;
2440 Ok(self.local_admission.preflight(&schema, context_tokens))
2441 }
2442
2443 pub fn active_local_resource_policy(&self) -> resource_policy::ResourcePolicyLoadEvidence {
2450 let mut evidence = self
2451 .resource_policy_evidence
2452 .read()
2453 .unwrap_or_else(|poisoned| poisoned.into_inner())
2454 .clone();
2455 evidence.policy = self.local_admission.policy();
2456 evidence
2457 }
2458
2459 pub fn apply_local_resource_policy(&self, policy: resource_policy::ResourcePolicy) {
2463 let ceiling_mb = policy
2464 .effective_budget(HardwareInfo::detect().total_ram_mb)
2465 .configured_model_ceiling_mb;
2466 self.local_admission.set_policy(policy.clone());
2467 *self
2468 .resource_policy_evidence
2469 .write()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner()) =
2471 resource_policy::ResourcePolicyLoadEvidence {
2472 policy,
2473 source: resource_policy::ResourcePolicyLoadSource::Loaded,
2474 warning: None,
2475 };
2476 let cache_ceiling_mb = configured_cache_budget_mb(ceiling_mb);
2477 self.model_budget
2478 .set_budget_bytes(cache_ceiling_mb.saturating_mul(1024 * 1024));
2479 let generation = self
2480 .resource_policy_generation
2481 .fetch_add(1, std::sync::atomic::Ordering::AcqRel)
2482 .saturating_add(1);
2483 if let Some(offload) = crate::offload::current_local_offload() {
2484 offload.refresh_resource_policy(generation);
2485 }
2486 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2487 {
2488 self.text_backends.enforce_budget();
2489 self.local_backends.enforce_budget();
2490 }
2491 }
2492
2493 pub fn begin_local_model_maintenance(
2494 &self,
2495 model_id: &str,
2496 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2497 {
2498 self.local_admission.begin_model_maintenance(model_id)
2499 }
2500
2501 pub async fn prepare_local_model_removal(
2502 &self,
2503 model_id: &str,
2504 ) -> Result<resource_policy::LocalModelMaintenanceGuard, resource_policy::ModelMaintenanceError>
2505 {
2506 let maintenance = self.local_admission.begin_model_maintenance(model_id)?;
2507 if let Some(offload) = crate::offload::current_local_offload() {
2508 if offload
2509 .resident_models()
2510 .await
2511 .iter()
2512 .any(|resident| resident == model_id)
2513 {
2514 let acknowledged = offload.release_model(model_id).await.map_err(|error| {
2515 resource_policy::ModelMaintenanceError::ReleaseFailed(error.to_string())
2516 })?;
2517 if !acknowledged
2518 || offload
2519 .resident_models()
2520 .await
2521 .iter()
2522 .any(|resident| resident == model_id)
2523 {
2524 return Err(
2525 resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
2526 model_id.to_string(),
2527 ),
2528 );
2529 }
2530 }
2531 }
2532 if self
2533 .vllm_pool
2534 .release_model_if_present(model_id)
2535 .await
2536 .is_err()
2537 {
2538 return Err(
2539 resource_policy::ModelMaintenanceError::ProcessReleaseUnacknowledged(
2540 model_id.to_string(),
2541 ),
2542 );
2543 }
2544 if !self.evict_local_model_if_idle(model_id) {
2545 return Err(resource_policy::ModelMaintenanceError::CacheReleaseBlocked(
2546 model_id.to_string(),
2547 ));
2548 }
2549 let allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2550 if !allocation_ids.is_empty() {
2551 return Err(resource_policy::ModelMaintenanceError::ResidualResidency {
2552 model_id: model_id.to_string(),
2553 allocation_ids,
2554 });
2555 }
2556 Ok(maintenance)
2557 }
2558
2559 pub fn evict_local_model_if_idle(&self, model_id: &str) -> bool {
2563 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2564 {
2565 let mut allocation_ids = self.local_admission.resident_allocation_ids(model_id);
2566 if !allocation_ids.iter().any(|id| id == model_id) {
2567 allocation_ids.push(model_id.to_string());
2568 }
2569 allocation_ids.into_iter().all(|allocation_id| {
2570 [
2571 self.text_backends.evict_if_idle(&allocation_id),
2572 self.local_backends.evict_if_idle(&allocation_id),
2573 ]
2574 .into_iter()
2575 .all(|evicted| evicted)
2576 })
2577 }
2578 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2579 {
2580 let _ = model_id;
2581 let backend_released = self.backend.try_write().is_ok_and(|mut backends| {
2582 if backends.remove(model_id).is_some() {
2583 self.local_admission.mark_evicted(model_id);
2584 }
2585 true
2586 });
2587 backend_released
2591 && self
2592 .embedding_backend
2593 .try_write()
2594 .is_ok_and(|slot| slot.is_none())
2595 }
2596 }
2597
2598 fn reserve_local_request(
2599 &self,
2600 schema: &ModelSchema,
2601 context_tokens: usize,
2602 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2603 self.ensure_model_enabled(&schema.id)?;
2604 let activity_lease = self.model_management.acquire_lease(&schema.id)?;
2605 let mut reservation = self
2606 .local_admission
2607 .reserve(schema, context_tokens)
2608 .map_err(InferenceError::from)?;
2609 reservation.attach_activity_lease(activity_lease);
2610 Ok(reservation)
2611 }
2612
2613 async fn reserve_local_request_with_worker_retry(
2618 &self,
2619 schema: &ModelSchema,
2620 context_tokens: usize,
2621 ) -> Result<resource_policy::LocalLoadReservation, InferenceError> {
2622 let original_error = match self.reserve_local_request(schema, context_tokens) {
2623 Ok(reservation) => return Ok(reservation),
2624 Err(error) => error,
2625 };
2626 let first_verdict = match &original_error {
2627 InferenceError::LocalResourceBlocked { preflight, .. } => preflight.verdict.clone(),
2628 _ => return Err(original_error),
2629 };
2630 if first_verdict != resource_policy::LocalLoadVerdict::InsufficientLiveMemory {
2631 return Err(original_error);
2632 }
2633
2634 let Some(offload) = crate::offload::current_local_offload() else {
2635 tracing::info!(
2636 model = %schema.id,
2637 first_verdict = ?first_verdict,
2638 release_happened = false,
2639 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2640 "local admission worker release-and-retry decision"
2641 );
2642 return Err(original_error);
2643 };
2644 if !offload
2645 .resident_models()
2646 .await
2647 .iter()
2648 .any(|model_id| model_id == &schema.id)
2649 {
2650 tracing::info!(
2651 model = %schema.id,
2652 first_verdict = ?first_verdict,
2653 release_happened = false,
2654 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2655 "local admission worker release-and-retry decision"
2656 );
2657 return Err(original_error);
2658 }
2659 let Some(scoped_allocation_id) = offload.resident_allocation_id(&schema.id) else {
2660 tracing::debug!(
2661 model = %schema.id,
2662 "release-and-retry skipped because the offload exposes no allocation id"
2663 );
2664 return Err(original_error);
2665 };
2666 match offload.release_model(&schema.id).await {
2667 Ok(true) => {
2668 let offload_still_resident = offload
2672 .resident_models()
2673 .await
2674 .iter()
2675 .any(|model_id| model_id == &schema.id);
2676 let allocation_still_charged = self
2679 .local_admission
2680 .resident_allocation_ids(&schema.id)
2681 .iter()
2682 .any(|allocation_id| allocation_id == &scoped_allocation_id);
2683 if offload_still_resident || allocation_still_charged {
2684 tracing::warn!(
2688 model = %schema.id,
2689 allocation = %scoped_allocation_id,
2690 offload_still_resident,
2691 allocation_still_charged,
2692 "release acknowledged but allocation {} still charged/pending",
2693 scoped_allocation_id
2694 );
2695 #[cfg(test)]
2696 UNRETIRED_RELEASE_WARNING_COUNT
2697 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2698 return Err(original_error);
2699 }
2700 let second = self.reserve_local_request(schema, context_tokens);
2701 let second_verdict = match &second {
2702 Ok(_) => Some("admitted".to_string()),
2703 Err(InferenceError::LocalResourceBlocked { preflight, .. }) => {
2704 Some(format!("{:?}", preflight.verdict))
2705 }
2706 Err(_) => None,
2707 };
2708 tracing::info!(
2709 model = %schema.id,
2710 first_verdict = ?first_verdict,
2711 release_happened = true,
2712 second_verdict = ?second_verdict,
2713 "local admission worker release-and-retry decision"
2714 );
2715 second
2716 }
2717 Ok(false) | Err(_) => {
2718 tracing::info!(
2719 model = %schema.id,
2720 first_verdict = ?first_verdict,
2721 release_happened = false,
2722 second_verdict = ?Option::<resource_policy::LocalLoadVerdict>::None,
2723 "local admission worker release-and-retry decision"
2724 );
2725 Err(original_error)
2726 }
2727 }
2728 }
2729
2730 #[cfg(any(
2735 test,
2736 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))
2737 ))]
2738 fn reconcile_transient_local_allocation(
2739 reservation: &mut resource_policy::LocalLoadReservation,
2740 measured_weights_bytes: u64,
2741 ) -> Result<(), InferenceError> {
2742 reservation
2743 .reconcile_measured_weights(measured_weights_bytes)
2744 .map(|_| ())
2745 .map_err(InferenceError::from)
2746 }
2747
2748 fn prepare_worker_admission(
2749 &self,
2750 schema: &ModelSchema,
2751 reservation: &mut resource_policy::LocalLoadReservation,
2752 ) -> Result<crate::offload::LocalWorkerAdmission, InferenceError> {
2753 let installed = self.config.models_dir.join(&schema.name);
2754 let installed_weights_bytes = backend_cache::estimate_model_size(&installed);
2755 let measured_weights_bytes =
2762 installed_weights_bytes.max(reservation.reconciled_weights_bytes());
2763 reservation
2764 .reconcile_measured_weights(measured_weights_bytes)
2765 .map_err(InferenceError::from)?;
2766 Ok(crate::offload::LocalWorkerAdmission {
2767 policy: self.local_admission.policy(),
2768 policy_generation: self
2769 .resource_policy_generation
2770 .load(std::sync::atomic::Ordering::Acquire),
2771 state_root: resource_policy::normalized_state_root_key(&self.config.state_root),
2772 measured_weights_bytes,
2773 })
2774 }
2775
2776 async fn reconcile_worker_residency(
2777 offload: &dyn crate::offload::LocalGenerationOffload,
2778 expected_model_id: &str,
2779 residency: &crate::offload::LocalWorkerResidency,
2780 retention: backend_cache::BackendRetention,
2781 reservation: &mut resource_policy::LocalLoadReservation,
2782 ) -> Result<(), InferenceError> {
2783 if residency.model_id != expected_model_id {
2784 if retention == backend_cache::BackendRetention::Resident {
2785 let released = offload
2786 .release_model(&residency.model_id)
2787 .await
2788 .unwrap_or(false);
2789 if !released {
2790 let allocation_id = offload
2791 .resident_allocation_id(&residency.model_id)
2792 .unwrap_or_else(|| {
2793 resource_policy::worker_process_allocation_id(&residency.model_id)
2794 });
2795 reservation.publish_resident_weights_as(
2796 &allocation_id,
2797 residency.measured_weights_bytes,
2798 );
2799 }
2800 }
2801 return Err(InferenceError::InferenceFailed(format!(
2802 "local worker acknowledged model '{}' for requested '{}'",
2803 residency.model_id, expected_model_id
2804 )));
2805 }
2806 if retention == backend_cache::BackendRetention::Resident {
2807 let allocation_id = offload
2808 .resident_allocation_id(expected_model_id)
2809 .unwrap_or_else(|| {
2810 resource_policy::worker_process_allocation_id(expected_model_id)
2811 });
2812 reservation
2813 .publish_resident_weights_as(&allocation_id, residency.measured_weights_bytes);
2814 }
2815 Ok(())
2816 }
2817
2818 fn admit_speech_candidate(
2819 &self,
2820 schema: &ModelSchema,
2821 explicit: bool,
2822 ) -> SpeechCandidateAdmission {
2823 if !schema.is_local()
2824 || matches!(
2825 schema.source,
2826 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. }
2827 )
2828 {
2829 return SpeechCandidateAdmission::Proceed(None);
2830 }
2831 match self.reserve_local_request(schema, 0) {
2832 Ok(reservation) => SpeechCandidateAdmission::Proceed(Some(reservation)),
2833 Err(error) if explicit => SpeechCandidateAdmission::FailBlocked(error),
2834 Err(error) => SpeechCandidateAdmission::SkipBlocked(error),
2835 }
2836 }
2837
2838 fn hold_local_reservation_for_stream(
2839 mut source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
2840 reservation: resource_policy::LocalLoadReservation,
2841 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
2842 let (tx, rx) = tokio::sync::mpsc::channel(64);
2843 tokio::spawn(async move {
2844 let _reservation = reservation;
2845 while let Some(event) = source.recv().await {
2846 if tx.send(event).await.is_err() {
2847 break;
2848 }
2849 }
2850 });
2851 rx
2852 }
2853
2854 fn hold_optional_reservation_for_stream(
2855 source: tokio::sync::mpsc::Receiver<stream::StreamEvent>,
2856 reservation: Option<resource_policy::LocalLoadReservation>,
2857 ) -> tokio::sync::mpsc::Receiver<stream::StreamEvent> {
2858 match reservation {
2859 Some(reservation) => Self::hold_local_reservation_for_stream(source, reservation),
2860 None => source,
2861 }
2862 }
2863
2864 pub fn set_spend_limits(&self, limits: Option<SpendLimits>) {
2870 *self.spend_limits.write().unwrap() = limits;
2871 }
2872
2873 fn preferred_model_for_capability(&self, capability: ModelCapability) -> Option<&str> {
2874 match capability {
2875 ModelCapability::Generate => self.config.preferred_generation_model.as_deref(),
2876 ModelCapability::Embed => self.config.preferred_embedding_model.as_deref(),
2877 ModelCapability::Classify => self.config.preferred_classification_model.as_deref(),
2878 _ => None,
2879 }
2880 }
2881
2882 fn request_has_tools(req: &GenerateRequest) -> bool {
2888 req.tools.as_ref().is_some_and(|t| !t.is_empty())
2889 }
2890
2891 fn request_needs_vision(req: &GenerateRequest) -> bool {
2892 req.images.as_ref().is_some_and(|images| !images.is_empty())
2893 || req.messages.as_ref().is_some_and(|messages| {
2894 messages
2895 .iter()
2896 .any(|msg| matches!(msg, Message::UserMultimodal { .. }))
2897 })
2898 }
2899
2900 #[allow(dead_code)] fn request_has_video(req: &GenerateRequest) -> bool {
2906 let images_have_video = req
2907 .images
2908 .as_ref()
2909 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_video));
2910 let messages_have_video = req.messages.as_ref().is_some_and(|messages| {
2911 messages.iter().any(|msg| match msg {
2912 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_video),
2913 _ => false,
2914 })
2915 });
2916 images_have_video || messages_have_video
2917 }
2918
2919 #[allow(dead_code)] fn request_has_audio(req: &GenerateRequest) -> bool {
2924 let images_have_audio = req
2925 .images
2926 .as_ref()
2927 .is_some_and(|blocks| blocks.iter().any(ContentBlock::is_audio));
2928 let messages_have_audio = req.messages.as_ref().is_some_and(|messages| {
2929 messages.iter().any(|msg| match msg {
2930 Message::UserMultimodal { content } => content.iter().any(ContentBlock::is_audio),
2931 _ => false,
2932 })
2933 });
2934 images_have_audio || messages_have_audio
2935 }
2936
2937 pub fn new(config: InferenceConfig) -> Self {
2938 let registry = models::ModelRegistry::new(config.models_dir.clone());
2939 let hw = HardwareInfo::detect();
2940 let policy_evidence =
2941 resource_policy::FileResourcePolicyRepository::new(config.state_root.clone())
2942 .load_with_evidence()
2943 .unwrap_or_else(|error| {
2944 tracing::warn!(%error, "failed to read local model resource policy; using Everyday");
2945 resource_policy::ResourcePolicyLoadEvidence {
2946 policy: resource_policy::ResourcePolicy::everyday(),
2947 source: resource_policy::ResourcePolicyLoadSource::CorruptDefault,
2948 warning: Some(format!(
2949 "The local-model resource policy could not be read ({error}); CAR used Everyday."
2950 )),
2951 }
2952 });
2953 let policy = policy_evidence.policy.clone();
2954 let effective_budget = policy.effective_budget(hw.total_ram_mb);
2955 let local_admission = resource_policy::scoped_local_admission_with_models_dir(
2956 &config.state_root,
2957 policy,
2958 hw.clone(),
2959 Some(config.models_dir.clone()),
2960 );
2961 let runtime_scope = scoped_inference_runtime(
2962 &config.state_root,
2963 effective_budget.configured_model_ceiling_mb,
2964 local_admission.clone(),
2965 );
2966 let router = ModelRouter::new(hw.clone());
2967 let unified_registry = UnifiedRegistry::new_with_state_root(
2968 config.state_root.clone(),
2969 config.models_dir.clone(),
2970 );
2971 let adaptive_router = AdaptiveRouter::with_default_config(hw);
2972 let mut tracker = OutcomeTracker::new();
2973 let profiles_path = config.state_models_dir().join("outcome_profiles.json");
2975 let outcome_profiles_loaded = match tracker.load_from_file(&profiles_path) {
2978 Ok(n) => {
2979 if n > 0 {
2980 tracing::info!(loaded = n, "loaded persisted model profiles");
2981 }
2982 true
2983 }
2984 Err(error) => error.kind() == std::io::ErrorKind::NotFound,
2985 };
2986 let mut benchmark_models_loaded = 0usize;
2987 for path in benchmark_priors_paths(&config.state_models_dir()) {
2988 match routing_ext::load_benchmark_priors(&path) {
2989 Ok(priors) if !priors.is_empty() => {
2990 benchmark_models_loaded += priors.len();
2991 routing_ext::apply_benchmark_priors(&mut tracker, &priors);
2992 tracing::info!(
2993 path = %path.display(),
2994 loaded = priors.len(),
2995 "loaded benchmark quality priors"
2996 );
2997 }
2998 Ok(_) => {}
2999 Err(error) => {
3000 tracing::warn!(path = %path.display(), %error, "failed to load benchmark priors");
3001 }
3002 }
3003 }
3004 if benchmark_models_loaded > 0 {
3005 tracing::info!(
3006 loaded = benchmark_models_loaded,
3007 "applied benchmark priors to cold-start routing"
3008 );
3009 }
3010 let outcome_tracker = Arc::new(RwLock::new(tracker));
3011
3012 let remote_backend = RemoteBackend::new();
3013 let model_management = model_management::ModelManagementStore::new(
3014 config.state_root.clone(),
3015 config.models_dir.clone(),
3016 );
3017
3018 let lane_defaults_path = config
3019 .state_root
3020 .join(crate::lane_defaults::LANE_DEFAULTS_FILE);
3021 let next_action_seq = crate::action_ledger::read_actions(
3022 &config
3023 .state_root
3024 .join(crate::action_ledger::ACTION_LEDGER_FILE),
3025 0,
3026 )
3027 .iter()
3028 .map(|a| a.seq)
3029 .max()
3030 .map(|m| m + 1)
3031 .unwrap_or(1);
3032 Self {
3033 config,
3034 unified: std::sync::RwLock::new(Arc::new(unified_registry)),
3035 registry_write: std::sync::Mutex::new(()),
3036 adaptive_router,
3037 outcome_tracker,
3038 last_outcome_flush: Arc::new(std::sync::Mutex::new(None)),
3039 outcome_profiles_loaded,
3040 ledger_io_lock: Arc::new(tokio::sync::Mutex::new(())),
3041 spend_limits: Arc::new(std::sync::RwLock::new(None)),
3042 lane_defaults_cache: Arc::new(std::sync::RwLock::new(crate::lane_defaults::load_from(
3043 &lane_defaults_path,
3044 ))),
3045 concierge_action_lock: Arc::new(tokio::sync::Mutex::new(())),
3046 external_pins: Default::default(),
3047 disk_pressured: Default::default(),
3048 failed_upgrades: Default::default(),
3049 concierge_action_seq: Arc::new(std::sync::atomic::AtomicU64::new(next_action_seq)),
3050 remote_backend,
3051 model_management,
3052 local_admission: local_admission.clone(),
3053 _runtime_scope: runtime_scope.clone(),
3054 resource_policy_generation: runtime_scope.resource_policy_generation.clone(),
3055 resource_policy_evidence: Arc::new(std::sync::RwLock::new(policy_evidence)),
3056 model_budget: runtime_scope.model_budget.clone(),
3057 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3058 text_backends: runtime_scope.text_backends.clone(),
3059 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3060 local_backends: runtime_scope.local_backends.clone(),
3061 registry,
3062 router,
3063 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3064 backend: runtime_scope.backend.clone(),
3065 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3066 embedding_backend: runtime_scope.embedding_backend.clone(),
3067 speech_runtime: runtime_scope.speech_runtime.clone(),
3068 speech_policy: SpeechPolicy {
3069 prefer_local: cfg!(all(
3070 target_os = "macos",
3071 target_arch = "aarch64",
3072 not(car_skip_mlx)
3073 )),
3074 allow_remote_fallback: true,
3075 preferred_local_stt: None,
3076 preferred_local_tts: None,
3077 preferred_remote_stt: None,
3078 preferred_remote_tts: None,
3079 },
3080 vllm_pool: runtime_scope.vllm_pool.clone(),
3081 }
3082 }
3083
3084 pub async fn init_key_pool(&self) {
3087 for schema in self.unified_registry().list() {
3089 if schema.is_remote() {
3090 self.remote_backend.register_model_keys(schema).await;
3091 }
3092 }
3093
3094 let stats_path = self.config.state_models_dir().join("key_pool_stats.json");
3096 if let Ok(n) = self.remote_backend.key_pool.load_stats(&stats_path).await {
3097 if n > 0 {
3098 tracing::info!(loaded = n, "loaded persisted key pool stats");
3099 }
3100 }
3101
3102 let total = self.remote_backend.key_pool.total_keys().await;
3103 if total > 0 {
3104 tracing::info!(keys = total, "key pool initialized");
3105 }
3106 }
3107
3108 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3111 async fn ensure_backend(
3112 &self,
3113 schema: &ModelSchema,
3114 reservation: &mut resource_policy::LocalLoadReservation,
3115 ) -> Result<(), InferenceError> {
3116 let read = self.backend.read().await;
3117 if read.contains_key(&schema.id) {
3118 return Ok(());
3119 }
3120 drop(read);
3121
3122 let mut write = self.backend.write().await;
3123 if write.contains_key(&schema.id) {
3124 return Ok(());
3125 }
3126
3127 let model_path = self.registry.ensure_model(&schema.name).await?;
3128 let mut measured = backend_cache::estimate_model_size(&model_path);
3129 reservation
3130 .reconcile_measured_weights(measured)
3131 .map_err(InferenceError::from)?;
3132 let device = self.config.device.unwrap_or_else(Device::auto);
3133 let backend = match CandleBackend::load(&model_path, device) {
3135 Ok(b) => b,
3136 Err(load_err) => {
3137 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3141 return Err(load_err);
3142 }
3143 tracing::warn!(
3144 model = %schema.id,
3145 error = %load_err,
3146 "candle backend load failed; purged corrupt cache files and re-pulling once"
3147 );
3148 let model_path = self.registry.ensure_model(&schema.name).await?;
3149 measured = backend_cache::estimate_model_size(&model_path);
3150 reservation
3151 .reconcile_measured_weights(measured)
3152 .map_err(InferenceError::from)?;
3153 CandleBackend::load(&model_path, device)?
3154 }
3155 };
3156 write.insert(schema.id.clone(), backend);
3157 reservation.publish_resident_weights(measured);
3158 Ok(())
3159 }
3160
3161 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3164 async fn ensure_embedding_backend(
3165 &self,
3166 reservation: &mut resource_policy::LocalLoadReservation,
3167 ) -> Result<(), InferenceError> {
3168 let read = self.embedding_backend.read().await;
3169 if read.is_some() {
3170 return Ok(());
3171 }
3172 drop(read);
3173
3174 let mut write = self.embedding_backend.write().await;
3175 if write.is_some() {
3176 return Ok(());
3177 }
3178
3179 let embedding_model = self
3180 .preferred_model_for_capability(ModelCapability::Embed)
3181 .unwrap_or(&self.config.embedding_model);
3182 let model_path = self.registry.ensure_model(embedding_model).await?;
3183 let mut measured = backend_cache::estimate_model_size(&model_path);
3184 reservation
3185 .reconcile_measured_weights(measured)
3186 .map_err(InferenceError::from)?;
3187 let device = self.config.device.unwrap_or_else(Device::auto);
3188 let backend = match EmbeddingBackend::load(&model_path, device) {
3190 Ok(b) => b,
3191 Err(load_err) => {
3192 if crate::download::purge_corrupt_cache_files(&model_path) == 0 {
3193 return Err(load_err);
3194 }
3195 tracing::warn!(
3196 model = embedding_model,
3197 error = %load_err,
3198 "embedding backend load failed; purged corrupt cache files and re-pulling once"
3199 );
3200 let model_path = self.registry.ensure_model(embedding_model).await?;
3201 measured = backend_cache::estimate_model_size(&model_path);
3202 reservation
3203 .reconcile_measured_weights(measured)
3204 .map_err(InferenceError::from)?;
3205 EmbeddingBackend::load(&model_path, device)?
3206 }
3207 };
3208 *write = Some(backend);
3209 reservation.publish_resident_weights(measured);
3210 Ok(())
3211 }
3212
3213 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3216 async fn ensure_mlx_embedding_backend(&self) -> Result<String, InferenceError> {
3217 let embedding_model_name = self
3218 .preferred_model_for_capability(ModelCapability::Embed)
3219 .unwrap_or(&self.config.embedding_model)
3220 .to_string();
3221 let schema = self
3222 .resolve_schema(&embedding_model_name)
3223 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model_name.clone()))?
3224 .clone();
3225 Ok(schema.id)
3226 }
3227
3228 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3244 async fn load_backend_healing<T, F, RF, R>(
3245 schema_id: &str,
3246 model_dir: std::path::PathBuf,
3247 cache: &backend_cache::BackendCache<T>,
3248 size: u64,
3249 reservation: &mut resource_policy::LocalLoadReservation,
3250 loader: F,
3251 repull: RF,
3252 ) -> Result<
3253 (
3254 backend_cache::CachedBackend<T>,
3255 backend_cache::BackendRetention,
3256 ),
3257 InferenceError,
3258 >
3259 where
3260 T: Send + 'static,
3261 F: Fn(&Path) -> Result<T, InferenceError>,
3262 RF: FnOnce() -> R,
3263 R: std::future::Future<Output = Result<std::path::PathBuf, InferenceError>>,
3264 {
3265 match cache.get_or_load_admitted(schema_id, size, reservation, || loader(&model_dir)) {
3266 Ok(admitted) => Ok(admitted),
3267 Err(load_err) => {
3268 let purged = crate::download::purge_corrupt_cache_files(&model_dir);
3276 if purged == 0 {
3277 return Err(load_err);
3279 }
3280 tracing::warn!(
3281 model = schema_id,
3282 purged,
3283 error = %load_err,
3284 "backend load failed; purged corrupt cache files and re-pulling once"
3285 );
3286 let fresh_dir = repull().await?;
3287 let fresh_size = backend_cache::estimate_model_size(&fresh_dir);
3288 cache
3289 .get_or_load_admitted(schema_id, fresh_size, reservation, || loader(&fresh_dir))
3290 }
3291 }
3292 }
3293
3294 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3298 async fn ensure_text_backend(
3299 &self,
3300 schema: &ModelSchema,
3301 reservation: &mut resource_policy::LocalLoadReservation,
3302 ) -> Result<
3303 (
3304 backend_cache::CachedBackend<backend::SwiftLmBackend>,
3305 backend_cache::BackendRetention,
3306 ),
3307 InferenceError,
3308 > {
3309 if !Self::supports_native_mlx(schema) {
3310 return Err(InferenceError::InferenceFailed(format!(
3311 "native MLX backend does not support {} ({}) yet; use vLLM-MLX or add a family-specific MLX backend",
3312 schema.name, schema.family
3313 )));
3314 }
3315
3316 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3317 let size = backend_cache::estimate_model_size(&model_dir);
3318 if !reservation.authorizes_model(&schema.id) {
3319 return Err(InferenceError::InferenceFailed(format!(
3320 "local admission reservation does not authorize {}",
3321 schema.id
3322 )));
3323 }
3324 let role = if schema.has_capability(ModelCapability::Embed)
3334 && !schema.has_capability(ModelCapability::Generate)
3335 {
3336 backend::swift_lm::BackendRole::Embedding
3337 } else {
3338 backend::swift_lm::BackendRole::Text
3339 };
3340 let loader = move |dir: &Path| {
3341 let dir = dir.to_path_buf();
3342 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3343 backend::SwiftLmBackend::load_with_role(&dir, role)
3345 }))
3346 .map_err(|e| {
3347 InferenceError::InferenceFailed(format!(
3348 "MLX backend loading panicked (possible Metal/accelerate exception): {:?}",
3349 e
3350 ))
3351 })?
3352 };
3353 let registry = self.unified_registry();
3354 Self::load_backend_healing(
3355 &schema.id,
3356 model_dir,
3357 &self.text_backends,
3358 size,
3359 reservation,
3360 loader,
3361 || registry.redownload_local(&schema.id),
3362 )
3363 .await
3364 }
3365
3366 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3371 async fn ensure_local_backend(
3372 &self,
3373 schema: &ModelSchema,
3374 reservation: &mut resource_policy::LocalLoadReservation,
3375 ) -> Result<
3376 (
3377 backend_cache::CachedBackend<Box<dyn backend::local::LocalInferenceBackend>>,
3378 backend_cache::BackendRetention,
3379 ),
3380 InferenceError,
3381 > {
3382 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3383 let size = backend_cache::estimate_model_size(&model_dir);
3384 if !reservation.authorizes_model(&schema.id) {
3385 return Err(InferenceError::InferenceFailed(format!(
3386 "local admission reservation does not authorize {}",
3387 schema.id
3388 )));
3389 }
3390 let loader = |dir: &Path| {
3391 let dir = dir.to_path_buf();
3392 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3393 backend::SwiftLmBackend::load(&dir)
3403 .map(|b| Box::new(b) as Box<dyn backend::local::LocalInferenceBackend>)
3404 .map_err(|e| {
3405 InferenceError::InferenceFailed(format!(
3406 "mlx-swift-lm could not load model_type '{}' ({}): {e}",
3407 backend::local::describe_model_type(&dir),
3408 dir.display()
3409 ))
3410 })
3411 }))
3412 .map_err(|e| {
3413 InferenceError::InferenceFailed(format!(
3414 "local backend loading panicked (possible Metal/accelerate exception): {:?}",
3415 e
3416 ))
3417 })?
3418 };
3419 let registry = self.unified_registry();
3420 Self::load_backend_healing(
3421 &schema.id,
3422 model_dir,
3423 &self.local_backends,
3424 size,
3425 reservation,
3426 loader,
3427 || registry.redownload_local(&schema.id),
3428 )
3429 .await
3430 }
3431
3432 pub async fn reset_local_kv_cache(&self, model_id: &str) {
3443 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3447 {
3448 let Some(schema) = self.unified_registry().get(model_id).cloned() else {
3449 return;
3450 };
3451 if !matches!(
3455 schema.source,
3456 ModelSource::Mlx { .. } | ModelSource::Local { .. }
3457 ) {
3458 return;
3459 }
3460 if !self.local_backends.contains(&schema.id) {
3463 return;
3464 }
3465 let Ok(mut reservation) = self.reserve_local_request(&schema, 0) else {
3466 return;
3467 };
3468 if let Ok((handle, _retention)) =
3469 self.ensure_local_backend(&schema, &mut reservation).await
3470 {
3471 if let Ok(mut guard) = handle.lock() {
3472 guard.clear_kv_cache();
3473 }
3474 }
3475 }
3476 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3477 let _ = model_id;
3478 }
3479
3480 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3487 pub async fn warm_up<S: AsRef<str>>(
3488 &self,
3489 schema_ids: &[S],
3490 ) -> Vec<Result<(), InferenceError>> {
3491 let mut results = Vec::with_capacity(schema_ids.len());
3492 for id in schema_ids {
3493 let id = id.as_ref();
3494 let outcome: Result<(), InferenceError> = async {
3495 let schema = self.unified_registry().get(id).cloned().ok_or_else(|| {
3496 InferenceError::InferenceFailed(format!("warm_up: unknown schema id {id}"))
3497 })?;
3498 let mut reservation = self.reserve_local_request(&schema, 0)?;
3500 match schema.capabilities.first().copied() {
3501 Some(ModelCapability::ImageGeneration) => {
3502 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3507 }
3508 Some(ModelCapability::VideoGeneration) => {
3509 let _ = self.unified_registry().ensure_local(&schema.id).await?;
3514 }
3515 Some(ModelCapability::TextToSpeech) => {
3516 }
3520 _ => {
3521 let _ = self.ensure_text_backend(&schema, &mut reservation).await?;
3522 }
3523 }
3524 Ok(())
3525 }
3526 .await;
3527 results.push(outcome);
3528 }
3529 results
3530 }
3531
3532 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3534 pub async fn warm_up<S: AsRef<str>>(
3535 &self,
3536 _schema_ids: &[S],
3537 ) -> Vec<Result<(), InferenceError>> {
3538 Vec::new()
3539 }
3540
3541 async fn vllm_live_schema(
3547 &self,
3548 schema: ModelSchema,
3549 reservation: Option<resource_policy::LocalLoadReservation>,
3550 context_tokens: usize,
3551 ) -> Result<(ModelSchema, Option<resource_policy::LocalLoadReservation>), InferenceError> {
3552 match &schema.source {
3553 ModelSource::ManagedVllmMlx { .. } => {}
3554 ModelSource::VllmMlx { .. } => return Ok((schema, None)),
3555 _ => return Ok((schema, None)),
3556 }
3557 let _dispatch = self.vllm_pool.acquire_dispatch(&schema.id).await;
3562 self.vllm_pool
3563 .wait_for_teardown(&schema.id, std::time::Duration::from_secs(5))
3564 .await
3565 .map_err(InferenceError::InferenceFailed)?;
3566 self.vllm_pool
3571 .reap_dead(&schema.id)
3572 .await
3573 .map_err(InferenceError::InferenceFailed)?;
3574 let mut reservation = match reservation {
3575 Some(reservation) if reservation.model_id() == schema.id => reservation,
3576 _ => self.reserve_local_request(&schema, context_tokens)?,
3577 };
3578 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
3579 let model_name = model_dir.display().to_string();
3580 let measured = backend_cache::estimate_model_size(&model_dir)
3581 .max(reservation.reconciled_weights_bytes());
3582 reservation
3583 .reconcile_measured_weights(measured)
3584 .map_err(InferenceError::from)?;
3585 let endpoint = self
3587 .vllm_pool
3588 .ensure(&schema.id, &model_name, &reservation, &schema.family)
3591 .await
3592 .map_err(InferenceError::InferenceFailed)?;
3593 let allocation_id = resource_policy::vllm_process_allocation_id(&schema.id);
3594 reservation.publish_resident_weights_as(&allocation_id, measured);
3595 let mut schema = schema;
3596 schema.source = ModelSource::VllmMlx {
3597 endpoint,
3598 model_name,
3599 };
3600 Ok((schema, Some(reservation)))
3601 }
3602
3603 pub async fn evict_idle_vllm_servers(&self) -> usize {
3606 self.vllm_pool.evict_idle().await
3607 }
3608
3609 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3620 pub fn evict_idle_backends(&self) -> (usize, u64) {
3621 let mut entries = 0usize;
3622 let mut bytes = 0u64;
3623 for (n, b) in [
3624 self.text_backends.evict_idle(),
3625 self.local_backends.evict_idle(),
3626 ] {
3627 entries += n;
3628 bytes = bytes.saturating_add(b);
3629 }
3630 (entries, bytes)
3631 }
3632
3633 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
3635 pub fn evict_idle_backends(&self) -> (usize, u64) {
3636 (0, 0)
3637 }
3638
3639 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
3640 fn supports_native_mlx(schema: &ModelSchema) -> bool {
3641 matches!(schema.family.as_str(), "qwen3" | "qwen2.5-vl" | "qwen2-vl")
3642 }
3643
3644 fn resolve_schema(&self, name: &str) -> Option<ModelSchema> {
3648 let registry = self.unified_registry();
3649 registry
3650 .get(name)
3651 .or_else(|| registry.find_by_name(name))
3652 .cloned()
3653 }
3654
3655 fn resolve_schema_name_first(&self, name: &str) -> Option<ModelSchema> {
3658 let registry = self.unified_registry();
3659 registry
3660 .find_by_name(name)
3661 .or_else(|| registry.get(name))
3662 .cloned()
3663 }
3664
3665 pub fn unified_registry(&self) -> Arc<UnifiedRegistry> {
3670 let me = self as *const Self as usize;
3671 PINNED_REGISTRY
3672 .try_with(|(engine, pinned)| (*engine == me).then(|| pinned.clone()))
3673 .ok()
3674 .flatten()
3675 .unwrap_or_else(|| self.current_registry())
3676 }
3677
3678 fn current_registry(&self) -> Arc<UnifiedRegistry> {
3680 self.unified
3681 .read()
3682 .unwrap_or_else(std::sync::PoisonError::into_inner)
3683 .clone()
3684 }
3685
3686 async fn with_pinned_registry<F: std::future::Future>(&self, operation: F) -> F::Output {
3695 let me = self as *const Self as usize;
3696 let pinned_here = PINNED_REGISTRY
3697 .try_with(|(engine, _)| *engine == me)
3698 .unwrap_or(false);
3699 if pinned_here {
3700 operation.await
3701 } else {
3702 PINNED_REGISTRY
3703 .scope((me, self.current_registry()), operation)
3704 .await
3705 }
3706 }
3707
3708 pub fn unified_registry_mut(&mut self) -> &mut UnifiedRegistry {
3712 Arc::make_mut(
3713 self.unified
3714 .get_mut()
3715 .unwrap_or_else(std::sync::PoisonError::into_inner),
3716 )
3717 }
3718
3719 fn update_registry<R>(&self, edit: impl FnOnce(&mut UnifiedRegistry) -> R) -> R {
3725 let _writer = self
3726 .registry_write
3727 .lock()
3728 .unwrap_or_else(std::sync::PoisonError::into_inner);
3729 let mut next = (*self.current_registry()).clone();
3730 let out = edit(&mut next);
3731 *self
3732 .unified
3733 .write()
3734 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(next);
3735 out
3736 }
3737
3738 pub fn swap_signed_catalog(&self, catalog: catalog::VerifiedCatalog) -> usize {
3747 let origin = Arc::new(catalog.envelope());
3748 let revoked: std::collections::HashSet<String> =
3749 catalog.revoked().iter().cloned().collect();
3750 self.update_registry(|registry| {
3751 registry.refresh_availability();
3754 let mut keep: std::collections::HashSet<String> =
3756 self.protected_models(registry, true).into_keys().collect();
3757 keep.extend(
3758 registry
3759 .list()
3760 .into_iter()
3761 .filter(|m| {
3762 (m.is_local() && m.downloads_weights() && m.weights_ready)
3765 || matches!(self.model_management.load_receipt(&m.id), Ok(Some(_)))
3766 })
3767 .map(|m| m.id.clone()),
3768 );
3769 let loaded = registry.replace_signed_catalog(
3770 catalog.into_models(),
3771 Some(origin),
3772 &keep,
3773 &revoked,
3774 );
3775 registry.refresh_availability();
3776 self.persist_retained(registry);
3777 loaded
3778 })
3779 }
3780
3781 fn persist_retained(&self, registry: &UnifiedRegistry) {
3784 let path = catalog::retained_path(&self.config.state_root);
3785 if let Err(error) = catalog::save_retained(&path, ®istry.retained_rows()) {
3786 tracing::warn!(path = %path.display(), %error, "could not record retained catalog rows");
3787 }
3788 }
3789
3790 fn keep_pulled_row_if_withdrawn(&self, name: &str) {
3794 let Some(schema) = self.resolve_schema_name_first(name) else {
3795 return;
3796 };
3797 if self.current_registry().get(&schema.id).is_some() {
3798 return;
3799 }
3800 let Some(origin) = self.unified_registry().signed_origin_of(&schema.id) else {
3801 return;
3802 };
3803 self.update_registry(|registry| {
3804 registry.restore_retained(vec![(schema, origin)]);
3805 registry.refresh_availability();
3806 self.persist_retained(registry);
3807 });
3808 }
3809
3810 fn catalog_registry_snapshot(&self) -> UnifiedRegistry {
3811 self.catalog_registry_snapshot_with(false)
3812 }
3813
3814 fn catalog_registry_snapshot_with(&self, authoritative_credentials: bool) -> UnifiedRegistry {
3823 let mut registry = (*self.unified_registry()).clone();
3824 registry.prune_missing_on_disk_models();
3825 if authoritative_credentials {
3830 registry.refresh_availability_for_explicit_status();
3831 } else {
3832 registry.refresh_availability();
3833 }
3834 self.filter_disabled_local_models(&mut registry);
3835 registry
3836 }
3837
3838 fn filter_disabled_local_models(&self, registry: &mut UnifiedRegistry) {
3839 let disabled = registry
3840 .all()
3841 .filter(|schema| {
3842 schema.downloads_weights()
3843 && !self
3844 .model_management
3845 .car_enabled(&schema.id)
3846 .unwrap_or(false)
3847 })
3848 .map(|schema| schema.id.clone())
3849 .collect::<Vec<_>>();
3850 for model_id in disabled {
3851 registry.unregister(&model_id);
3852 }
3853 }
3854
3855 async fn routing_registry_snapshot_with_credential_failure(
3856 &self,
3857 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
3858 let parslee = car_auth::resolve_credential(car_auth::CredentialReadMode::Use).await;
3864 let (parslee_api_base, parslee_signed_out, credential_failure) = match &parslee {
3865 Ok(Some(credential)) => (Some(credential.api_base.as_str()), false, None),
3866 Ok(None) => (None, true, Some(parslee_signed_out_route_failure())),
3867 Err(error) => {
3868 let source_error = error.to_string();
3869 (
3870 None,
3871 false,
3872 Some(RouteCredentialFailure {
3873 summary: format!(
3874 "Parslee {AUTH_STORE_UNREADABLE_MARKER} — unlock or grant access to the credential store, then retry"
3875 ),
3876 source_error,
3877 }),
3878 )
3879 }
3880 };
3881 let mut registry = (*self.unified_registry()).clone();
3882 registry.prune_missing_on_disk_models();
3883 registry.refresh_routing_availability(parslee_api_base, parslee_signed_out);
3884 self.filter_disabled_local_models(&mut registry);
3885 (registry, credential_failure)
3886 }
3887
3888 async fn work_context_routing_snapshot(
3889 &self,
3890 context: &car_auth::context::CredentialContext,
3891 requested_model: Option<&str>,
3892 ) -> (UnifiedRegistry, Option<RouteCredentialFailure>) {
3893 let credential = if requested_model.is_some() {
3896 None
3897 } else {
3898 Some(car_auth::context::resolve_context_credential(context).await)
3899 };
3900 let available = context.organization_id.is_some()
3901 && credential.as_ref().is_none_or(|result| result.is_ok());
3902 let failure =
3903 credential
3904 .and_then(|result| result.err())
3905 .map(|error| RouteCredentialFailure {
3906 summary: format!("Work-context credentials unavailable: {error}"),
3907 source_error: error.to_string(),
3908 });
3909 let mut registry = (*self.unified_registry()).clone();
3910 registry.prune_missing_on_disk_models();
3911 registry.refresh_work_context_availability(&context.api_base, available);
3912 self.filter_disabled_local_models(&mut registry);
3913 (registry, failure)
3914 }
3915
3916 async fn routing_registry_snapshot(&self) -> UnifiedRegistry {
3917 self.routing_registry_snapshot_with_credential_failure()
3918 .await
3919 .0
3920 }
3921
3922 async fn request_routing_registry_snapshot(
3927 &self,
3928 requested_model: Option<&str>,
3929 ) -> UnifiedRegistry {
3930 if requested_model.is_some() {
3931 self.catalog_registry_snapshot()
3932 } else {
3933 self.routing_registry_snapshot().await
3934 }
3935 }
3936
3937 pub async fn route_adaptive(&self, prompt: &str) -> AdaptiveRoutingDecision {
3939 self.route_adaptive_with_intent(prompt, None).await
3940 }
3941
3942 pub async fn route_adaptive_with_intent(
3952 &self,
3953 prompt: &str,
3954 intent: Option<crate::intent::IntentHint>,
3955 ) -> AdaptiveRoutingDecision {
3956 if let Some(model) = self.preferred_model_for_capability(ModelCapability::Generate) {
3957 let exclude_set = self
3958 .adaptive_router
3959 .build_exclude_set(intent.as_ref(), &self.unified_registry());
3960 let overridden_is_excluded =
3961 Self::model_is_excluded(&exclude_set, &self.unified_registry(), model);
3962 if !overridden_is_excluded {
3963 let ctx_len = self
3969 .resolve_schema(model)
3970 .map(|s| s.context_length)
3971 .unwrap_or(0);
3972 return AdaptiveRoutingDecision {
3973 model_id: model.to_string(),
3974 model_name: model.to_string(),
3975 task: InferenceTask::Generate,
3976 complexity: TaskComplexity::assess(prompt),
3977 reason: "preferred generation model override".into(),
3978 strategy: RoutingStrategy::Explicit,
3979 predicted_quality: 0.5,
3980 fallbacks: vec![],
3981 context_length: ctx_len,
3982 needs_compaction: false,
3983 candidates: vec![],
3984 };
3985 }
3986 }
3987 let routing_registry = self.routing_registry_snapshot().await;
3988 let tracker = self.outcome_tracker.read().await;
3989 match intent {
3990 Some(hint) => self
3991 .adaptive_router
3992 .route_with(crate::adaptive_router::RouteRequest {
3993 intent: Some(&hint),
3994 ..crate::adaptive_router::RouteRequest::new(prompt, &routing_registry, &tracker)
3995 }),
3996 None => self
3997 .adaptive_router
3998 .route(prompt, &routing_registry, &tracker),
3999 }
4000 }
4001
4002 pub async fn route_adaptive_with_work_context(
4005 &self,
4006 prompt: &str,
4007 intent: Option<crate::intent::IntentHint>,
4008 context: &car_auth::context::CredentialContext,
4009 ) -> Result<AdaptiveRoutingDecision, String> {
4010 context.validate()?;
4011 let (registry, failure) = self.work_context_routing_snapshot(context, None).await;
4012 if let Some(failure) = failure {
4013 return Err(failure.summary);
4014 }
4015 let tracker = self.outcome_tracker.read().await;
4016 Ok(self
4017 .adaptive_router
4018 .route_with(crate::adaptive_router::RouteRequest {
4019 intent: intent.as_ref(),
4020 ..crate::adaptive_router::RouteRequest::new(prompt, ®istry, &tracker)
4021 }))
4022 }
4023
4024 pub fn route(&self, prompt: &str) -> RoutingDecision {
4026 self.router.route_generate(prompt, &self.registry)
4027 }
4028
4029 pub fn estimated_tokens(
4040 &self,
4041 req: &GenerateRequest,
4042 model_id: Option<&str>,
4043 ) -> (usize, usize, bool) {
4044 let prompt_tokens = remote::estimate_tokens(&req.prompt);
4045 let context_tokens = req
4046 .context
4047 .as_ref()
4048 .map(|c| remote::estimate_tokens(c))
4049 .unwrap_or(0);
4050 let tools_tokens = req
4051 .tools
4052 .as_ref()
4053 .map(|t| remote::estimate_tokens(&serde_json::to_string(t).unwrap_or_default()))
4054 .unwrap_or(0);
4055 let media_tokens = media_tokens::request_media_and_history_tokens(
4056 req.images.as_deref(),
4057 req.messages.as_deref(),
4058 );
4059 let total_input = prompt_tokens + context_tokens + tools_tokens + media_tokens;
4060
4061 let context_window = match model_id {
4070 Some(id) => {
4071 let routing_registry = self.catalog_registry_snapshot();
4072 routing_registry
4073 .get(id)
4074 .or_else(|| routing_registry.find_by_name(id))
4075 .map(|s| s.context_length)
4076 .unwrap_or(0)
4077 }
4078 None => 0,
4079 };
4080
4081 let fits = context_window == 0 || (total_input + req.params.max_tokens) <= context_window;
4082 (total_input, context_window, fits)
4083 }
4084
4085 fn routing_cache_estimates(req: &GenerateRequest, estimated_input: usize) -> (usize, usize) {
4091 let read = req
4092 .params
4093 .estimated_cache_read_input_tokens
4094 .min(estimated_input);
4095 let write = req
4096 .params
4097 .estimated_cache_write_input_tokens
4098 .min(estimated_input.saturating_sub(read));
4099 (read, write)
4100 }
4101
4102 pub fn model_context_window(&self, model_id: &str) -> usize {
4108 let routing_registry = self.catalog_registry_snapshot();
4109 routing_registry
4110 .get(model_id)
4111 .or_else(|| routing_registry.find_by_name(model_id))
4112 .map(|s| s.context_length)
4113 .unwrap_or(0)
4114 }
4115
4116 pub async fn generate_tracked(
4131 &self,
4132 req: GenerateRequest,
4133 ) -> Result<InferenceResult, InferenceError> {
4134 let mut ignore_retry = |_: InferenceRetryProgress| {};
4135 self.generate_tracked_with_retry_observer(req, &mut ignore_retry)
4136 .await
4137 }
4138
4139 pub async fn generate_tracked_with_retry_observer(
4143 &self,
4144 req: GenerateRequest,
4145 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4146 ) -> Result<InferenceResult, InferenceError> {
4147 let mut ignore_exclusion = |_: RouterExclusion| {};
4148 self.generate_tracked_with_observers(req, retry_observer, &mut ignore_exclusion)
4149 .await
4150 }
4151
4152 pub async fn generate_tracked_with_observers(
4156 &self,
4157 req: GenerateRequest,
4158 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4159 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4160 ) -> Result<InferenceResult, InferenceError> {
4161 self.with_pinned_registry(self.generate_tracked_with_observers_unpinned(
4163 req,
4164 retry_observer,
4165 exclusion_observer,
4166 ))
4167 .await
4168 }
4169
4170 async fn generate_tracked_with_observers_unpinned(
4171 &self,
4172 req: GenerateRequest,
4173 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4174 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4175 ) -> Result<InferenceResult, InferenceError> {
4176 crate::offload::ensure_not_controlled_terminated()?;
4177 let catalog_snapshot = self
4178 .catalog_snapshot()
4179 .map_err(InferenceError::InferenceFailed)?;
4180 let recover = matches!(req.params.thinking, ThinkingMode::Auto);
4181 let mut result = self
4182 .generate_tracked_inner(
4183 req.clone(),
4184 &catalog_snapshot,
4185 retry_observer,
4186 exclusion_observer,
4187 )
4188 .await?;
4189
4190 let action = classify_empty_pass(
4191 recover,
4192 result.stop_reason.as_deref(),
4193 &result.text,
4194 result.tool_calls.is_empty(),
4195 );
4196 let hit_decode_ceiling = action == EmptyPassAction::FailDecodeCeiling;
4197
4198 if action == EmptyPassAction::RetryWithoutThinking {
4199 result.stop_reason = Some("thinking_truncated".to_string());
4200 let mut retry = req;
4201 retry.params.thinking = ThinkingMode::Off;
4202 crate::offload::ensure_not_controlled_terminated()?;
4203 match self
4204 .generate_tracked_inner(
4205 retry,
4206 &catalog_snapshot,
4207 retry_observer,
4208 exclusion_observer,
4209 )
4210 .await
4211 {
4212 Ok(mut recovered) => {
4213 if !recovered.text.trim().is_empty() || !recovered.tool_calls.is_empty() {
4214 recovered.stop_reason = Some("thinking_recovered".to_string());
4215 return Ok(recovered);
4216 }
4217 }
4218 Err(error @ InferenceError::ControlledTermination) => return Err(error),
4219 Err(_) => {}
4220 }
4221 }
4222
4223 if hit_decode_ceiling && result.text.trim().is_empty() && result.tool_calls.is_empty() {
4229 return Err(InferenceError::InferenceFailed(format!(
4230 "local generation hit its {}s wall-clock ceiling before producing any output. \
4231 The `local prefill starting` / `local decode in progress` log lines show where \
4232 the time went — a large prompt can spend most of it on prefill. Try a smaller \
4233 model, shorten the prompt, check for another process contending for the GPU, \
4234 or raise the ceiling with CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables it).",
4235 local_decode_timeout().map_or(0, |t| t.as_secs())
4236 )));
4237 }
4238
4239 Ok(result)
4240 }
4241
4242 #[instrument(
4257 name = "inference.generate",
4258 skip_all,
4259 fields(
4260 model = tracing::field::Empty,
4261 max_tokens = req.params.max_tokens,
4262 prompt_tokens = tracing::field::Empty,
4263 completion_tokens = tracing::field::Empty,
4264 latency_ms = tracing::field::Empty,
4265 ttft_ms = tracing::field::Empty,
4269 gen_ai.operation.name = "chat",
4270 gen_ai.request.model = tracing::field::Empty,
4271 gen_ai.response.model = tracing::field::Empty,
4272 gen_ai.request.max_tokens = req.params.max_tokens,
4273 gen_ai.request.temperature = req.params.temperature,
4274 gen_ai.usage.input_tokens = tracing::field::Empty,
4275 gen_ai.usage.output_tokens = tracing::field::Empty,
4276 )
4277 )]
4278 async fn generate_tracked_inner(
4279 &self,
4280 mut req: GenerateRequest,
4281 catalog_snapshot: &CatalogSnapshot,
4282 retry_observer: &mut (dyn FnMut(InferenceRetryProgress) + Send),
4283 exclusion_observer: &mut (dyn FnMut(RouterExclusion) + Send),
4284 ) -> Result<InferenceResult, InferenceError> {
4285 self.refuse_system_one_generate(&req)?;
4286 crate::offload::ensure_not_controlled_terminated()?;
4287 validate_expected_catalog_revision(&req, catalog_snapshot)?;
4288 if let Some(requested) = req.model.as_deref() {
4292 tracing::Span::current().record("gen_ai.request.model", requested);
4293 }
4294 let requested_model_id = exact_pinned_model_id(&req).map(str::to_string);
4295 if let Some(model_id) = requested_model_id.as_ref() {
4296 req.model = Some(model_id.clone());
4297 req.params.strict_model = true;
4298 }
4299 let start = Instant::now();
4300 let has_requested_route = req.model.is_some();
4301 let (routing_registry, initial_route_credential_failure) =
4302 if let Some(context) = &req.work_context {
4303 self.work_context_routing_snapshot(context, req.model.as_deref())
4304 .await
4305 } else if has_requested_route {
4306 (self.catalog_registry_snapshot(), None)
4307 } else {
4308 self.routing_registry_snapshot_with_credential_failure()
4309 .await
4310 };
4311 if let Some(requested) = requested_model_id.as_deref() {
4312 if routing_registry.get(requested).is_none() {
4313 return Err(InferenceError::ModelNotFound(requested.to_string()));
4314 }
4315 } else if let Some(requested) = req.model.as_deref() {
4316 if routing_registry
4317 .get(requested)
4318 .or_else(|| routing_registry.find_by_name(requested))
4319 .is_none()
4320 {
4321 return Err(InferenceError::ModelNotFound(requested.to_string()));
4322 }
4323 }
4324
4325 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
4327 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
4335 let (estimated_cache_read, estimated_cache_write) =
4336 Self::routing_cache_estimates(&req, estimated_input);
4337 let tracker_read = self.outcome_tracker.read().await;
4338 let has_tools = Self::request_has_tools(&req);
4339 let has_vision = Self::request_needs_vision(&req);
4340 let preferred_model = self
4341 .preferred_model_for_capability(ModelCapability::Generate)
4342 .map(str::to_string);
4343 let exclude_set = self
4344 .adaptive_router
4345 .build_exclude_set(req.intent.as_ref(), &routing_registry);
4346 let unpinned_override = self
4347 .lane_pin_for(&req, &routing_registry)
4348 .or(preferred_model)
4349 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model));
4350 let decision = match req.model.clone().or(unpinned_override) {
4351 Some(m) => {
4352 let ctx_len = routing_registry
4353 .get(&m)
4354 .or_else(|| routing_registry.find_by_name(&m))
4355 .map(|s| s.context_length)
4356 .unwrap_or(0);
4357 AdaptiveRoutingDecision {
4358 model_id: m.clone(),
4359 model_name: m.clone(),
4360 task: InferenceTask::Generate,
4361 complexity: TaskComplexity::assess(&req.prompt),
4362 reason: "explicit model".into(),
4363 strategy: RoutingStrategy::Explicit,
4364 predicted_quality: 0.5,
4365 fallbacks: vec![],
4366 context_length: ctx_len,
4367 needs_compaction: ctx_len > 0 && estimated_footprint > ctx_len,
4368 candidates: vec![],
4369 }
4370 }
4371 None => self.adaptive_router.route_with_exclusion_observer(
4372 crate::adaptive_router::RouteRequest {
4373 estimated_total_tokens: estimated_footprint,
4374 estimated_input_tokens: estimated_input,
4375 estimated_output_tokens: req.params.max_tokens,
4376 estimated_cache_read_tokens: estimated_cache_read,
4377 estimated_cache_write_tokens: estimated_cache_write,
4378 has_tools,
4379 has_vision,
4380 workload: req.params.workload,
4381 intent: req.intent.as_ref(),
4382 ..crate::adaptive_router::RouteRequest::new(
4383 &req.prompt,
4384 &routing_registry,
4385 &tracker_read,
4386 )
4387 },
4388 exclusion_observer,
4389 ),
4390 };
4391 drop(tracker_read);
4392
4393 if decision.model_id.is_empty() {
4394 let excluded_models = req
4395 .intent
4396 .as_ref()
4397 .map(|hint| hint.exclude_models.join(", "))
4398 .unwrap_or_default();
4399 return Err(InferenceError::NoEligibleModel { excluded_models });
4400 }
4401
4402 if decision.needs_compaction {
4403 tracing::info!(
4404 model = %decision.model_name,
4405 prompt_tokens = estimated_input,
4406 context_window = decision.context_length,
4407 "prompt exceeds model context window — compaction or truncation needed"
4408 );
4409 }
4410
4411 debug!(
4416 model = %decision.model_name,
4417 strategy = ?decision.strategy,
4418 reason = %decision.reason,
4419 "adaptive-routed generate request"
4420 );
4421
4422 let mut req = req;
4425
4426 if let Some(schema) = routing_registry
4432 .get(&decision.model_id)
4433 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4434 {
4435 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
4436 }
4437
4438 let is_code_intent = is_explicit_code_intent(req.intent.as_ref());
4447 let is_complex = matches!(decision.complexity, TaskComplexity::Complex);
4448 if req.params.budget_tokens == 0 && (is_code_intent || is_complex) {
4449 let supports_thinking = routing_registry
4453 .get(&decision.model_id)
4454 .or_else(|| routing_registry.find_by_name(&decision.model_id))
4455 .map(|s| {
4456 s.supported_params
4457 .contains(&schema::GenerateParam::ExtendedThinking)
4458 })
4459 .unwrap_or(false);
4460 if let Some(budget) =
4461 auto_thinking_budget(is_code_intent, is_complex, supports_thinking)
4462 {
4463 req.params.budget_tokens = budget;
4464 tracing::info!(
4465 model = %decision.model_name,
4466 budget,
4467 code_intent = is_code_intent,
4468 "auto-enabled extended thinking"
4469 );
4470 }
4471 }
4472
4473 let mut models_to_try = vec![decision.model_id.clone()];
4475 models_to_try.extend(decision.fallbacks.iter().cloned());
4476
4477 let chain_has_local = models_to_try.iter().any(|m| {
4492 routing_registry
4493 .get(m)
4494 .or_else(|| routing_registry.find_by_name(m))
4495 .map(|s| s.is_local())
4496 .unwrap_or(false)
4497 });
4498 let mut local_last_resort_id = None;
4499 if should_append_local_last_resort(chain_has_local, req.params.strict_model) {
4500 if let Some(local) = self.first_installed_local_model(has_tools) {
4504 tracing::info!(
4505 local_model = %local,
4506 needs_tools = has_tools,
4507 "appended on-device model as last-resort fallback (chain was remote-only)"
4508 );
4509 models_to_try.push(local.clone());
4510 local_last_resort_id = Some(local);
4511 }
4512 } else if !chain_has_local && req.params.strict_model {
4513 tracing::info!(
4514 model = %decision.model_id,
4515 "strict_model set — not degrading to on-device; a remote failure will surface as an error"
4516 );
4517 }
4518
4519 let mut last_error = None;
4520 let mut auth_dead_lane: Option<String> = None;
4526 let mut route_credential_failure = if chain_includes_parslee_route(
4536 |m| {
4537 routing_registry
4538 .get(m)
4539 .or_else(|| routing_registry.find_by_name(m))
4540 },
4541 &models_to_try,
4542 ) {
4543 initial_route_credential_failure
4544 } else {
4545 None
4546 };
4547 let mut fallback_hops: Vec<FallbackFrom> = Vec::new();
4551
4552 let mut candidate_queue: std::collections::VecDeque<String> =
4559 models_to_try.iter().cloned().collect();
4560 let mut is_primary_attempt = true;
4561 while let Some(candidate_owned) = candidate_queue.pop_front() {
4562 crate::offload::ensure_not_controlled_terminated()?;
4563 let was_primary = is_primary_attempt;
4564 is_primary_attempt = false;
4565 let candidate_id = &candidate_owned;
4566 #[allow(unused_mut)]
4569 let mut schema = routing_registry
4570 .get(candidate_id)
4571 .or_else(|| routing_registry.find_by_name(candidate_id))
4572 .cloned();
4573
4574 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4582 if requested_model_id.is_none() {
4583 if let Some(ref s) = schema {
4584 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
4585 tracing::info!(
4586 from = %s.id, to = %mlx_equiv.id,
4587 "redirecting GGUF model to MLX equivalent on Apple Silicon"
4588 );
4589 schema = Some(mlx_equiv.clone());
4590 }
4591 }
4592 }
4593
4594 let candidate_name = schema
4599 .as_ref()
4600 .map(|s| s.name.clone())
4601 .unwrap_or_else(|| candidate_id.clone());
4602
4603 if has_tools
4614 && schema
4615 .as_ref()
4616 .map(|s| !s.has_capability(ModelCapability::ToolUse))
4617 .unwrap_or(false)
4618 {
4619 let backend = schema
4620 .as_ref()
4621 .map(|s| if s.is_local() { "local" } else { "remote" })
4622 .unwrap_or("unknown");
4623 tracing::warn!(
4624 model = %candidate_id,
4625 backend,
4626 "tools requested but resolved model lacks ToolUse capability — skipping candidate"
4627 );
4628 let unsupported = InferenceError::UnsupportedMode {
4629 mode: "tool_use",
4630 backend,
4631 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
4632 };
4633 record_fallback_from(&mut fallback_hops, &candidate_name, &unsupported);
4639 last_error = Some(unsupported);
4640 continue;
4641 }
4642
4643 let resolved_id = schema
4654 .as_ref()
4655 .map(|s| s.id.clone())
4656 .unwrap_or_else(|| candidate_id.clone());
4657 let reported_model_used = if requested_model_id.is_some() {
4658 resolved_id.clone()
4659 } else {
4660 candidate_name.clone()
4661 };
4662 validate_expected_catalog_row(&req, catalog_snapshot, &resolved_id)?;
4663
4664 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
4670 if requested_model_id.is_some()
4671 && schema
4672 .as_ref()
4673 .is_some_and(|schema| matches!(schema.source, ModelSource::Local { .. }))
4674 {
4675 let pinned = InferenceError::InferenceFailed(format!(
4676 "exact model_id `{resolved_id}` names a GGUF/Candle row, but Candle is disabled on Apple Silicon; MLX-equivalent substitution is disabled for exact pins"
4677 ));
4678 record_fallback_from(&mut fallback_hops, &candidate_name, &pinned);
4679 last_error = Some(pinned);
4680 continue;
4681 }
4682
4683 let is_remote = schema
4684 .as_ref()
4685 .map(|s| s.is_remote() || s.is_vllm_mlx())
4686 .unwrap_or(false);
4687 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
4688 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
4689
4690 let attempt_trace = {
4695 let mut tracker = self.outcome_tracker.write().await;
4696 tracker.record_start(&resolved_id, decision.task, &decision.reason)
4697 };
4698
4699 if is_delegated {
4705 let runner = match runner::current_inference_runner() {
4706 Some(r) => r,
4707 None => {
4708 let msg = "model declares ModelSource::Delegated but no inference runner is registered";
4709 self.outcome_tracker.write().await.record_not_model_fault(
4712 &attempt_trace,
4713 msg,
4714 outcome::UnattributedReason::Capability,
4715 );
4716 let unregistered = InferenceError::InferenceFailed(msg.into());
4717 record_fallback_from(&mut fallback_hops, &candidate_name, &unregistered);
4718 last_error = Some(unregistered);
4719 continue;
4720 }
4721 };
4722 let (tx, mut rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
4723 let emitter = runner::EventEmitter::new(tx);
4724 let runner_req = req.clone();
4725 let runner_handle = AbortOnDropTask(Some(tokio::spawn(async move {
4726 runner.run(runner_req, emitter).await
4727 })));
4728 let mut accumulator = stream::StreamAccumulator::default();
4729 while let Some(evt) = rx.recv().await {
4730 accumulator.push(&evt);
4731 }
4732 let (acc_text, acc_tool_calls, acc_usage, acc_stop_reason) =
4761 accumulator.finish_with_usage();
4762 match runner_handle.join().await {
4763 Ok(Ok(runner_result)) => {
4764 let elapsed = start.elapsed().as_millis() as u64;
4765 let acc_text = if acc_text.trim().is_empty() {
4766 runner_result.text
4767 } else {
4768 acc_text
4769 };
4770 let acc_tool_calls = if acc_tool_calls.is_empty() {
4771 runner_result.tool_calls
4772 } else {
4773 acc_tool_calls
4774 };
4775 let est_out = acc_text.split_whitespace().count();
4782 {
4783 let mut tracker = self.outcome_tracker.write().await;
4784 tracker.record_complete(
4785 &attempt_trace,
4786 elapsed,
4787 estimated_input,
4788 est_out,
4789 );
4790 }
4791 let local_last_resort = report_local_last_resort_served(
4792 local_last_resort_id.as_deref(),
4793 candidate_id,
4794 &resolved_id,
4795 );
4796 record_generation_span(candidate_name.as_str(), elapsed, None, None);
4803 return Ok(InferenceResult {
4804 text: acc_text,
4805 tool_calls: acc_tool_calls,
4806 bounding_boxes: vec![],
4807 trace_id: attempt_trace,
4808 model_used: reported_model_used,
4809 model_identity: bound_model_identity(
4810 catalog_snapshot,
4811 requested_model_id.as_deref(),
4812 &resolved_id,
4813 )?,
4814 latency_ms: elapsed,
4815 time_to_first_token_ms: None,
4816 usage: acc_usage,
4819 provider_output_items: vec![],
4820 thinking: vec![],
4823 stop_reason: acc_stop_reason,
4824 auth_fallback_from: auth_dead_lane.clone(),
4825 local_last_resort,
4826 fallback_from: fallback_hops.clone(),
4827 });
4828 }
4829 Ok(Err(e)) => {
4830 {
4831 let mut tracker = self.outcome_tracker.write().await;
4832 match &e {
4833 runner::RunnerError::Declined(_) => tracker.record_not_model_fault(
4836 &attempt_trace,
4837 &e.to_string(),
4838 outcome::UnattributedReason::Capability,
4839 ),
4840 runner::RunnerError::Failed(_) => {
4841 tracker.record_failure(&attempt_trace, &e.to_string())
4842 }
4843 }
4844 }
4845 let failed = InferenceError::InferenceFailed(e.to_string());
4852 record_auth_dead_lane(
4853 &mut auth_dead_lane,
4854 &candidate_name,
4855 &failed.to_string(),
4856 );
4857 record_route_credential_failure(
4858 &mut route_credential_failure,
4859 &candidate_name,
4860 &failed,
4861 was_primary && has_requested_route,
4862 );
4863 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
4864 last_error = Some(failed);
4865 continue;
4866 }
4867 Err(join_err) => {
4868 let msg = format!("runner task panicked: {join_err}");
4869 self.outcome_tracker
4870 .write()
4871 .await
4872 .record_failure(&attempt_trace, &msg);
4873 let failed = InferenceError::InferenceFailed(msg);
4879 record_fallback_from(&mut fallback_hops, &candidate_name, &failed);
4880 last_error = Some(failed);
4881 continue;
4882 }
4883 }
4884 }
4885
4886 let has_tools = Self::request_has_tools(&req);
4887
4888 let context = if has_tools
4890 && req.tools.as_ref().is_some_and(|t| {
4891 t.iter().any(|tool| {
4892 tool.get("function")
4893 .and_then(|f| f.get("name"))
4894 .and_then(|n| n.as_str())
4895 == Some("done")
4896 })
4897 }) {
4898 let base = req.context.as_deref().unwrap_or("");
4899 Some(format!(
4900 "{base}\n\nIMPORTANT: When calling the `done` tool, the `result` field MUST contain a DETAILED summary of everything you found and did. This is the ONLY output the user sees. Do NOT just say 'completed' — include specific findings, data, and conclusions."
4901 ))
4902 } else {
4903 req.context.clone()
4904 };
4905
4906 let mut captured_thinking: Vec<crate::tasks::generate::ThinkingBlock> = Vec::new();
4910 let mut captured_provider_output_items: Vec<serde_json::Value> = Vec::new();
4911 let mut local_reservation = if !is_delegated {
4917 match schema
4918 .as_ref()
4919 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
4920 {
4921 Some(local_schema) => {
4922 match self
4923 .reserve_local_request_with_worker_retry(
4924 local_schema,
4925 estimated_footprint,
4926 )
4927 .await
4928 {
4929 Ok(reservation) => Some(reservation),
4930 Err(error) if req.model.is_some() || req.params.strict_model => {
4931 self.outcome_tracker
4932 .write()
4933 .await
4934 .record_capability_rejection(
4935 &attempt_trace,
4936 &error.to_string(),
4937 );
4938 return Err(error);
4939 }
4940 Err(error) => {
4941 tracing::warn!(
4942 model = %local_schema.id,
4943 error = %error,
4944 "adaptive local candidate blocked by resource policy; trying next route"
4945 );
4946 self.outcome_tracker
4947 .write()
4948 .await
4949 .record_capability_rejection(
4950 &attempt_trace,
4951 &error.to_string(),
4952 );
4953 record_fallback_from(&mut fallback_hops, &candidate_name, &error);
4958 last_error = Some(error);
4959 continue;
4960 }
4961 }
4962 }
4963 None => None,
4964 }
4965 } else {
4966 None
4967 };
4968 let result = if is_codex_cli {
4969 let schema_ref = schema
4970 .as_ref()
4971 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
4972 if req.response_format.is_some() {
4973 Err(InferenceError::UnsupportedMode {
4974 mode: "response-format",
4975 backend: "codex-cli",
4976 reason: "the subscription-backed Codex source returns plain text and does not expose provider-enforced schemas",
4977 })
4978 } else {
4979 let model = match &schema_ref.source {
4980 ModelSource::CodexCli { model } => model,
4981 _ => unreachable!("is_codex_cli matched a different source"),
4982 };
4983 crate::backend::codex_cli::generate_request(
4984 model,
4985 &req,
4986 context.as_deref(),
4987 schema_ref.context_length,
4988 )
4989 .await
4990 .map(|(output, calls)| (output.text, calls, Some(output.usage), None, None))
4991 }
4992 } else if is_remote {
4993 let (schema_val, _remote_request_reservation) = match self
4997 .vllm_live_schema(
4998 schema.unwrap(),
4999 local_reservation.take(),
5000 estimated_footprint,
5001 )
5002 .await
5003 {
5004 Ok(pair) => pair,
5005 Err(e) => {
5006 record_dispatch_failure(
5010 &mut *self.outcome_tracker.write().await,
5011 &attempt_trace,
5012 &e,
5013 );
5014 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5015 last_error = Some(e);
5016 continue;
5017 }
5018 };
5019 let _ctx_len = schema_val.context_length;
5020 let temperature = if !schema_val.supported_params.is_empty()
5023 && !schema_val
5024 .supported_params
5025 .contains(&crate::schema::GenerateParam::Temperature)
5026 {
5027 -1.0
5028 } else {
5029 req.params.temperature
5030 };
5031
5032 self.remote_backend
5038 .with_work_context(req.work_context.clone())
5039 .generate_with_tools_multi_observed(
5040 &schema_val,
5041 &req.prompt,
5042 context.as_deref(),
5043 temperature,
5044 req.params.max_tokens,
5045 req.tools.as_deref(),
5046 req.images.as_deref(),
5047 req.messages.as_deref(),
5048 req.params.tool_choice.as_deref(),
5049 req.params.parallel_tool_calls,
5050 req.params.budget_tokens,
5051 req.cache_control,
5052 req.params.cache_ttl,
5053 req.context_stable_prefix.as_deref(),
5054 req.response_format.as_ref(),
5055 retry_observer,
5056 )
5057 .await
5058 .map(|(t, c, thinking, provider_items, u, stop)| {
5064 captured_thinking = thinking;
5065 captured_provider_output_items = provider_items;
5066 (t, c, u, None::<u64>, stop)
5067 })
5068 } else if let Some(offload) = schema
5069 .as_ref()
5070 .filter(|schema| Self::supports_worker_offload(schema))
5071 .and_then(|_| crate::offload::current_local_offload())
5072 {
5073 let mut offload_req = req.clone();
5084 offload_req.model = Some(resolved_id.clone());
5085 let schema_ref = schema
5086 .as_ref()
5087 .ok_or_else(|| InferenceError::ModelNotFound(resolved_id.clone()))?;
5088 let reservation = local_reservation.as_mut().ok_or_else(|| {
5089 InferenceError::InferenceFailed(
5090 "local worker dispatch missing admission reservation".into(),
5091 )
5092 })?;
5093 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_id) {
5094 reservation.bind_allocation_id(&allocation_id);
5100 }
5101 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
5102 match offload.generate_admitted(offload_req, admission).await {
5103 Ok(outcome) => {
5104 Self::reconcile_worker_residency(
5105 offload.as_ref(),
5106 &resolved_id,
5107 &outcome.residency,
5108 outcome.retention,
5109 reservation,
5110 )
5111 .await?;
5112 let ir = outcome.result;
5113 captured_thinking = ir.thinking;
5120 captured_provider_output_items = ir.provider_output_items;
5121 Ok((
5122 ir.text,
5123 ir.tool_calls,
5124 ir.usage,
5125 ir.time_to_first_token_ms,
5126 ir.stop_reason,
5127 ))
5128 }
5129 Err(e) => Err(e),
5130 }
5131 } else {
5132 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5133 {
5134 let schema_ref = schema
5138 .as_ref()
5139 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5140
5141 if schema_ref.is_foundation_models() {
5149 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5150 if Self::request_has_video(&req) || Self::request_has_audio(&req) {
5151 Err(InferenceError::UnsupportedMode {
5152 mode: "multimodal-content",
5153 backend: "foundation-models",
5154 reason: "the FoundationModels bridge exposes text and image \
5155 input — route audio/video to a remote VL model",
5156 })
5157 } else if has_images {
5158 let images: Vec<String> = req
5164 .images
5165 .as_ref()
5166 .map(|blocks| {
5167 blocks
5168 .iter()
5169 .filter_map(|b| match b {
5170 crate::ContentBlock::ImageBase64 { data, .. } => {
5171 Some(data.clone())
5172 }
5173 _ => None,
5177 })
5178 .collect()
5179 })
5180 .unwrap_or_default();
5181 if images.is_empty() {
5182 return Err(InferenceError::UnsupportedMode {
5183 mode: "multimodal-content",
5184 backend: "foundation-models",
5185 reason: "FoundationModels takes inline image bytes; \
5186 image URLs must be fetched by the caller first",
5187 });
5188 }
5189 let prompt = req.prompt.clone();
5190 let instructions = context.clone();
5191 let fm_prompt = prompt.clone();
5192 let fm_instructions = instructions.clone();
5193 let fm_ctx_window = schema_ref.context_length as u64;
5194 let max_tokens = req.params.max_tokens as u32;
5195 let temperature = req.params.temperature;
5196 tokio::task::spawn_blocking(move || {
5197 crate::backend::foundation_models::generate_with_images(
5198 &prompt,
5199 instructions.as_deref(),
5200 &images,
5201 max_tokens,
5202 temperature as f32,
5203 )
5204 })
5205 .await
5206 .map_err(|e| {
5207 InferenceError::InferenceFailed(format!(
5208 "FoundationModels task panicked: {e}"
5209 ))
5210 })
5211 .and_then(|r| r)
5212 .map(|text| {
5213 let usage = foundation_models_usage(
5214 fm_instructions.as_deref(),
5215 &fm_prompt,
5216 &text,
5217 fm_ctx_window,
5218 );
5219 (text, vec![], usage, None, None)
5220 })
5221 } else if has_tools {
5222 if req.response_format.is_some() {
5228 tracing::warn!(
5229 "FoundationModels: response_format is ignored when tools \
5230 are present — one turn is either tool-enabled or \
5231 schema-constrained"
5232 );
5233 }
5234 let prompt = req.prompt.clone();
5235 let instructions = context.clone();
5236 let fm_prompt = prompt.clone();
5239 let fm_instructions = instructions.clone();
5240 let fm_ctx_window = schema_ref.context_length as u64;
5241 let tools_defs = req.tools.clone().unwrap_or_default();
5242 let fm_tool_choice = req.params.tool_choice.clone();
5243 let max_tokens = req.params.max_tokens as u32;
5244 let temperature = req.params.temperature;
5245 tokio::task::spawn_blocking(move || {
5246 crate::backend::foundation_models::generate_with_tools(
5247 &prompt,
5248 instructions.as_deref(),
5249 &tools_defs,
5250 fm_tool_choice.as_deref(),
5251 max_tokens,
5252 temperature as f32,
5253 )
5254 })
5255 .await
5256 .map_err(|e| {
5257 InferenceError::InferenceFailed(format!(
5258 "FoundationModels task panicked: {e}"
5259 ))
5260 })
5261 .and_then(|r| r)
5262 .map(|(text, calls)| {
5263 let usage = foundation_models_usage(
5264 fm_instructions.as_deref(),
5265 &fm_prompt,
5266 &text,
5267 fm_ctx_window,
5268 );
5269 (text, calls, usage, None, None)
5270 })
5271 } else if let Some(crate::tasks::generate::ResponseFormat::JsonSchema {
5272 schema,
5273 ..
5274 }) = &req.response_format
5275 {
5276 let prompt = req.prompt.clone();
5280 let instructions = context.clone();
5281 let fm_prompt = prompt.clone();
5284 let fm_instructions = instructions.clone();
5285 let fm_ctx_window = schema_ref.context_length as u64;
5286 let schema_val = schema.clone();
5287 let max_tokens = req.params.max_tokens as u32;
5288 let temperature = req.params.temperature;
5289 tokio::task::spawn_blocking(move || {
5290 crate::backend::foundation_models::generate_structured(
5291 &prompt,
5292 instructions.as_deref(),
5293 &schema_val,
5294 max_tokens,
5295 temperature as f32,
5296 )
5297 })
5298 .await
5299 .map_err(|e| {
5300 InferenceError::InferenceFailed(format!(
5301 "FoundationModels task panicked: {e}"
5302 ))
5303 })
5304 .and_then(|r| r)
5305 .map(|text| {
5306 let usage = foundation_models_usage(
5307 fm_instructions.as_deref(),
5308 &fm_prompt,
5309 &text,
5310 fm_ctx_window,
5311 );
5312 (text, vec![], usage, None, None)
5313 })
5314 } else {
5315 let instructions = if matches!(
5319 req.response_format,
5320 Some(crate::tasks::generate::ResponseFormat::JsonObject)
5321 ) {
5322 tracing::warn!(
5323 "FoundationModels: JsonObject response_format has no native \
5324 constrained mode — enforcing via instruction injection"
5325 );
5326 let base = context.clone().unwrap_or_default();
5327 Some(format!(
5328 "{base}\n\nRespond with a single valid JSON object and \
5329 nothing else."
5330 ))
5331 } else {
5332 context.clone()
5333 };
5334 let prompt = req.prompt.clone();
5335 let fm_prompt = prompt.clone();
5338 let fm_instructions = instructions.clone();
5339 let fm_ctx_window = schema_ref.context_length as u64;
5340 let max_tokens = req.params.max_tokens as u32;
5341 let temperature = req.params.temperature;
5342 tokio::task::spawn_blocking(move || {
5343 crate::backend::foundation_models::generate(
5344 &prompt,
5345 instructions.as_deref(),
5346 max_tokens,
5347 temperature as f32,
5348 )
5349 })
5350 .await
5351 .map_err(|e| {
5352 InferenceError::InferenceFailed(format!(
5353 "FoundationModels task panicked: {e}"
5354 ))
5355 })
5356 .and_then(|r| r)
5357 .map(|text| {
5358 let usage = foundation_models_usage(
5359 fm_instructions.as_deref(),
5360 &fm_prompt,
5361 &text,
5362 fm_ctx_window,
5363 );
5364 (text, vec![], usage, None, None)
5365 })
5366 }
5367 } else if !schema_ref.is_mlx() {
5368 Err(InferenceError::InferenceFailed(format!(
5369 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
5370 schema_ref.id
5371 )))
5372 } else if schema_ref.tags.iter().any(|t| t == "mlx-vlm-cli") {
5373 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5385 if !has_images {
5386 return Err(InferenceError::UnsupportedMode {
5387 mode: "text-only-on-mlx-vlm-id",
5388 backend: "mlx-vlm-cli",
5389 reason: "the `mlx-vlm/...` model IDs route exclusively \
5390 through the mlx-vlm CLI for image inference. \
5391 For text-only generation, route to a Qwen3 \
5392 text model (`mlx/qwen3-4b:4bit` etc.) — the \
5393 CLI shell-out has higher latency than the \
5394 in-process MLX text tower.",
5395 });
5396 }
5397 let vlm_status = crate::backend::mlx_vlm_cli::runtime_status();
5398 if !vlm_status.is_available() {
5399 return Err(InferenceError::InferenceFailed(vlm_status.user_message()));
5400 }
5401 let model_dir =
5402 self.unified_registry().ensure_local(&schema_ref.id).await?;
5403 let reservation = local_reservation
5404 .as_mut()
5405 .expect("local VLM branch has admission reservation");
5406 Self::reconcile_transient_local_allocation(
5407 reservation,
5408 backend_cache::estimate_model_size(&model_dir),
5409 )?;
5410 let detached_lease = reservation.detached_lease();
5411 let repo = match &schema_ref.source {
5412 crate::schema::ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
5413 _ => {
5414 return Err(InferenceError::InferenceFailed(format!(
5415 "model '{}' is tagged mlx-vlm-cli but its \
5416 source isn't ModelSource::Mlx — registry bug",
5417 schema_ref.id
5418 )));
5419 }
5420 };
5421 let imgs = req.images.clone().unwrap_or_default();
5422 let temp = req.params.temperature;
5423 let max_t = req.params.max_tokens;
5424 let prompt = req.prompt.clone();
5425 let (text, cli_usage) = run_admitted_blocking(detached_lease, move || {
5426 crate::backend::mlx_vlm_cli::generate(
5427 &repo, &prompt, &imgs, temp, max_t,
5428 )
5429 })
5430 .await
5431 .map_err(|e| {
5432 InferenceError::InferenceFailed(format!(
5433 "mlx_vlm CLI task panicked: {e}"
5434 ))
5435 })??;
5436 let bounding_boxes = parse_boxes(&text);
5437 let latency_ms = start.elapsed().as_millis() as u64;
5438 let usage = cli_usage.map(|u| TokenUsage {
5448 prompt_tokens: u.prompt_tokens,
5449 completion_tokens: u.completion_tokens,
5450 total_tokens: u.prompt_tokens + u.completion_tokens,
5451 context_window: schema_ref.context_length as u64,
5452 ..Default::default()
5454 });
5455 {
5456 let (in_tokens, out_tokens) = match &usage {
5462 Some(u) => (u.prompt_tokens as usize, u.completion_tokens as usize),
5463 None => (estimated_input, text.split_whitespace().count()),
5464 };
5465 let mut tracker = self.outcome_tracker.write().await;
5466 tracker.record_complete(
5467 &attempt_trace,
5468 latency_ms,
5469 in_tokens,
5470 out_tokens,
5471 );
5472 }
5473 let local_last_resort = report_local_last_resort_served(
5474 local_last_resort_id.as_deref(),
5475 candidate_id,
5476 &resolved_id,
5477 );
5478 record_generation_span(
5483 candidate_name.as_str(),
5484 latency_ms,
5485 None,
5486 usage.as_ref(),
5487 );
5488 return Ok(InferenceResult {
5489 text,
5490 tool_calls: vec![],
5491 bounding_boxes,
5492 trace_id: attempt_trace,
5493 model_used: schema_ref.id.clone(),
5494 model_identity: bound_model_identity(
5495 catalog_snapshot,
5496 requested_model_id.as_deref(),
5497 &resolved_id,
5498 )?,
5499 latency_ms,
5500 time_to_first_token_ms: None,
5501 usage,
5502 provider_output_items: Vec::new(),
5503 thinking: Vec::new(), stop_reason: None,
5505 auth_fallback_from: auth_dead_lane.clone(),
5506 local_last_resort,
5507 fallback_from: fallback_hops.clone(),
5508 });
5509 } else if !Self::supports_native_mlx(schema_ref) {
5510 if req.images.as_ref().is_some_and(|i| !i.is_empty())
5520 || Self::request_has_video(&req)
5521 || Self::request_has_audio(&req)
5522 {
5523 return Err(InferenceError::UnsupportedMode {
5524 mode: "multimodal-content-block",
5525 backend: "native-mlx-local",
5526 reason: "this in-process MLX backend is text-only; route \
5527 image/video/audio understanding to a vLLM-MLX or remote \
5528 multimodal model",
5529 });
5530 }
5531 self.generate_local(
5542 req.clone(),
5543 &schema_ref.id,
5544 local_reservation
5545 .as_mut()
5546 .expect("local branch has admission reservation"),
5547 )
5548 .await
5549 } else {
5550 if Self::request_has_video(&req) {
5556 return Err(InferenceError::UnsupportedMode {
5557 mode: "video-content-block",
5558 backend: "native-mlx-qwen25vl",
5559 reason: "Qwen2.5-VL video understanding is on the request surface \
5560 but the video-tokenization path (frame sampling + merger) \
5561 is not yet wired; route to a remote VL provider for now",
5562 });
5563 }
5564 if Self::request_has_audio(&req) {
5565 return Err(InferenceError::UnsupportedMode {
5566 mode: "audio-content-block",
5567 backend: "native-mlx-qwen25vl",
5568 reason: "audio understanding is on the request surface (Gemma 4 \
5569 E2B/E4B and Gemini accept it) but the native MLX path \
5570 for this model does not — route to Gemini or Gemma-4",
5571 });
5572 }
5573 let has_images = req.images.as_ref().is_some_and(|imgs| !imgs.is_empty());
5574 if has_images {
5575 let (handle, _retention) = self
5595 .ensure_text_backend(
5596 schema_ref,
5597 local_reservation
5598 .as_mut()
5599 .expect("local branch has admission reservation"),
5600 )
5601 .await?;
5602 let can_do_vision = {
5603 let guard = handle.lock().map_err(|_| {
5604 InferenceError::InferenceFailed(
5605 "MLX backend mutex poisoned".into(),
5606 )
5607 })?;
5608 guard.supports_capability(crate::schema::ModelCapability::Vision)
5609 };
5610 if !can_do_vision {
5611 return Err(InferenceError::UnsupportedMode {
5621 mode: "image-content-block",
5622 backend: "native-mlx-text",
5623 reason: "this MLX backend is a plain Qwen3 text tower. \
5624 For local image inference, route to \
5625 `mlx-vlm/qwen3-vl-2b:bf16` or another `mlx-vlm/...` \
5626 catalog ID so CAR shells out to `mlx_vlm.generate`. \
5627 Alternatives: a local vLLM-MLX VLM server, or a \
5628 remote VL model. (#115)",
5629 });
5630 }
5631 }
5632 self.generate_mlx(
5633 render_for_local_backend(req.clone()),
5634 &schema_ref.id,
5635 local_reservation
5636 .as_mut()
5637 .expect("local branch has admission reservation"),
5638 )
5639 .await
5640 .map(|(text, usage, ttft, stop)| (text, vec![], usage, ttft, stop))
5641 }
5642 }
5643
5644 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
5645 {
5646 let schema_ref = schema
5647 .as_ref()
5648 .ok_or_else(|| InferenceError::ModelNotFound(candidate_id.clone()))?;
5649 match self
5650 .ensure_backend(
5651 schema_ref,
5652 local_reservation
5653 .as_mut()
5654 .expect("local branch has admission reservation"),
5655 )
5656 .await
5657 {
5658 Ok(()) => {
5659 let mut write = self.backend.write().await;
5660 let backend = write.get_mut(&schema_ref.id).ok_or_else(|| {
5661 InferenceError::InferenceFailed(format!(
5662 "candle backend missing after ensure_backend for {}",
5663 schema_ref.id
5664 ))
5665 })?;
5666 let ctx_window = backend.context_length().unwrap_or(0) as u64;
5672 tasks::generate::generate(
5673 backend,
5674 render_for_local_backend(req.clone()),
5675 )
5676 .await
5677 .map(
5678 |(text, ttft, prompt_tokens, completion_tokens)| {
5679 let usage = TokenUsage {
5680 prompt_tokens: prompt_tokens as u64,
5681 completion_tokens: completion_tokens as u64,
5682 total_tokens: (prompt_tokens + completion_tokens) as u64,
5683 context_window: ctx_window,
5684 ..Default::default()
5687 };
5688 (text, vec![], Some(usage), ttft, None)
5689 },
5690 )
5691 }
5692 Err(e) => Err(e),
5693 }
5694 }
5695 };
5696
5697 match result {
5698 Ok((text, mut tool_calls, usage, time_to_first_token_ms, stop_reason)) => {
5699 let text = if !is_remote
5706 && !is_codex_cli
5707 && !is_delegated
5708 && req.tools.is_some()
5709 && tool_calls.is_empty()
5710 {
5711 let (clean, parsed) = tasks::generate::parse_tool_calls(&text);
5712 tool_calls = parsed;
5713 clean
5714 } else {
5715 text
5716 };
5717 let latency_ms = start.elapsed().as_millis() as u64;
5718 let estimated_tokens = usage
5719 .as_ref()
5720 .map(|u| u.completion_tokens as usize)
5721 .unwrap_or_else(|| text.split_whitespace().count());
5722 let input_tokens = usage
5727 .as_ref()
5728 .map(|u| u.prompt_tokens as usize)
5729 .unwrap_or(estimated_input);
5730 let (cache_read, cache_creation) = usage
5735 .as_ref()
5736 .map(|u| {
5737 (
5738 u.cache_read_input_tokens as usize,
5739 u.cache_creation_input_tokens as usize,
5740 )
5741 })
5742 .unwrap_or((0, 0));
5743 {
5744 let mut tracker = self.outcome_tracker.write().await;
5745 tracker.record_complete_cached(
5746 &attempt_trace,
5747 latency_ms,
5748 input_tokens,
5749 estimated_tokens,
5750 cache_read,
5751 cache_creation,
5752 );
5753 }
5754 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
5760 cb.record_success(&resolved_id);
5761 }
5762 self.auto_save_outcomes().await;
5764
5765 record_generation_span(
5767 candidate_name.as_str(),
5768 latency_ms,
5769 time_to_first_token_ms,
5770 usage.as_ref(),
5771 );
5772
5773 let bounding_boxes = tasks::grounding::parse_boxes(&text);
5776 let local_last_resort = report_local_last_resort_served(
5777 local_last_resort_id.as_deref(),
5778 candidate_id,
5779 &resolved_id,
5780 );
5781 return Ok(InferenceResult {
5782 text,
5783 tool_calls,
5784 bounding_boxes,
5785 trace_id: attempt_trace,
5786 model_used: reported_model_used,
5787 model_identity: bound_model_identity(
5788 catalog_snapshot,
5789 requested_model_id.as_deref(),
5790 &resolved_id,
5791 )?,
5792 latency_ms,
5793 time_to_first_token_ms,
5794 usage,
5795 provider_output_items: captured_provider_output_items,
5796 thinking: captured_thinking,
5797 stop_reason,
5798 auth_fallback_from: auth_dead_lane.clone(),
5799 local_last_resort,
5800 fallback_from: fallback_hops.clone(),
5801 });
5802 }
5803 Err(e) => {
5804 if matches!(e, InferenceError::ControlledTermination) {
5805 self.outcome_tracker.write().await.record_not_model_fault(
5806 &attempt_trace,
5807 &e.to_string(),
5808 outcome::UnattributedReason::Cancelled,
5809 );
5810 return Err(e);
5811 }
5812 tracing::warn!(
5813 model = %candidate_name,
5814 error = %e,
5815 remaining = candidate_queue.len(),
5816 "model failed, trying next fallback immediately"
5817 );
5818 record_auth_dead_lane(&mut auth_dead_lane, &candidate_name, &e.to_string());
5825 record_route_credential_failure(
5826 &mut route_credential_failure,
5827 &candidate_name,
5828 &e,
5829 was_primary && has_requested_route,
5830 );
5831 record_fallback_from(&mut fallback_hops, &candidate_name, &e);
5832 {
5837 let mut tracker = self.outcome_tracker.write().await;
5838 record_dispatch_failure(&mut tracker, &attempt_trace, &e);
5839 }
5840 if error_counts_against_circuit_breaker(&e) {
5850 let err_str = e.to_string();
5851 let is_client_error =
5852 err_str.contains("API returned 4") && !err_str.contains("429");
5853 if let Ok(mut cb) = self.adaptive_router.circuit_breakers.lock() {
5854 cb.record_failure(&resolved_id);
5856 if is_client_error {
5859 cb.record_failure(&resolved_id);
5860 }
5861 }
5862 }
5863 #[cfg(not(all(
5865 target_os = "macos",
5866 target_arch = "aarch64",
5867 not(car_skip_mlx)
5868 )))]
5869 {
5870 let mut write = self.backend.write().await;
5871 if write.remove(&resolved_id).is_some() {
5872 self.local_admission.mark_evicted(&resolved_id);
5873 }
5874 }
5875 if let InferenceError::GatewayUnconfigured { namespace, .. } = &e {
5903 let before = candidate_queue.len();
5904 candidate_queue.retain(|id| !id.starts_with(namespace.as_str()));
5905 let dropped = before - candidate_queue.len();
5906 if dropped > 0 {
5907 tracing::info!(
5908 %namespace,
5909 dropped,
5910 remaining = candidate_queue.len(),
5911 "gateway has no upstream for this namespace; dropping its \
5912 remaining candidates instead of replaying the same rejection"
5913 );
5914 }
5915 }
5916 if error_ends_fallback_chain(&e) {
5917 let dropped = candidate_queue.len();
5918 candidate_queue.clear();
5919 if dropped > 0 {
5920 tracing::info!(
5921 dropped,
5922 "content refused for this request; ending the fallback chain \
5923 rather than answering with a different model"
5924 );
5925 }
5926 }
5927 if let InferenceError::ProviderAccount { provider, .. } = &e {
5928 let mut rest: Vec<String> = candidate_queue.iter().cloned().collect();
5932 let demoted = routing_ext::demote_provider(provider, &mut rest, |id| {
5933 routing_registry
5934 .get(id)
5935 .or_else(|| routing_registry.find_by_name(id))
5936 .map(|s| s.provider.clone())
5937 });
5938 candidate_queue = rest.into();
5939 if demoted > 0 {
5940 tracing::info!(
5941 %provider,
5942 demoted,
5943 remaining = candidate_queue.len(),
5944 "account-level rejection; deferring this provider's \
5945 remaining candidates to the end of the chain"
5946 );
5947 }
5948 }
5949 if was_primary && !candidate_queue.is_empty() {
5950 let err_str = e.to_string();
5951 let status = parse_api_returned_status(&err_str);
5958 if routing_ext::is_provider_transient(status, &err_str) {
5959 let provider_of = |id: &str| {
5964 routing_registry
5965 .get(id)
5966 .or_else(|| routing_registry.find_by_name(id))
5967 .map(|s| s.provider.clone())
5968 };
5969 let primary = provider_of(candidate_id).unwrap_or_default();
5970 let queue_vec: Vec<String> = candidate_queue.iter().cloned().collect();
5971 if let Some(cross) =
5972 routing_ext::first_cross_provider(&primary, &queue_vec, provider_of)
5973 {
5974 let cross = cross.to_string();
5975 if let Some(pos) = candidate_queue.iter().position(|m| *m == cross)
5976 {
5977 if pos > 0 {
5978 if let Some(m) = candidate_queue.remove(pos) {
5979 tracing::info!(
5980 promoted = %m,
5981 "transient provider error on primary; promoting cross-provider fallback"
5982 );
5983 candidate_queue.push_front(m);
5984 }
5985 }
5986 }
5987 }
5988 }
5989 }
5990 last_error = Some(e);
5991 }
5992 }
5993 }
5994
5995 let underlying = last_error.unwrap_or(InferenceError::InferenceFailed(
5997 "no models available".into(),
5998 ));
5999
6000 let e = apply_route_failure_context(underlying, route_credential_failure.as_ref());
6014 self.auto_save_outcomes().await;
6018 Err(e)
6019 }
6020
6021 async fn generate_stream_raw(
6026 &self,
6027 req: GenerateRequest,
6028 ) -> Result<(String, tokio::sync::mpsc::Receiver<stream::StreamEvent>), InferenceError> {
6029 let routing_registry = if let Some(context) = &req.work_context {
6030 self.work_context_routing_snapshot(context, req.model.as_deref())
6031 .await
6032 .0
6033 } else {
6034 self.request_routing_registry_snapshot(req.model.as_deref())
6035 .await
6036 };
6037 if let Some(requested) = req.model.as_deref() {
6038 if routing_registry
6039 .get(requested)
6040 .or_else(|| routing_registry.find_by_name(requested))
6041 .is_none()
6042 {
6043 return Err(InferenceError::ModelNotFound(requested.to_string()));
6044 }
6045 }
6046 let has_tools = Self::request_has_tools(&req);
6047 let has_vision = Self::request_needs_vision(&req);
6048 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6049 let estimated_footprint = estimated_input.saturating_add(req.params.max_tokens);
6050 let (estimated_cache_read, estimated_cache_write) =
6051 Self::routing_cache_estimates(&req, estimated_input);
6052 let preferred_model = self
6053 .preferred_model_for_capability(ModelCapability::Generate)
6054 .map(str::to_string);
6055 let exclude_set = self
6056 .adaptive_router
6057 .build_exclude_set(req.intent.as_ref(), &routing_registry);
6058 let unpinned_override = self
6059 .lane_pin_for(&req, &routing_registry)
6060 .or(preferred_model)
6061 .filter(|model| !Self::model_is_excluded(&exclude_set, &routing_registry, model));
6062 let decision = match req.model.clone().or(unpinned_override) {
6063 Some(m) => {
6064 let ctx_len = routing_registry
6065 .get(&m)
6066 .or_else(|| routing_registry.find_by_name(&m))
6067 .map(|s| s.context_length)
6068 .unwrap_or(0);
6069 AdaptiveRoutingDecision {
6070 model_id: m.clone(),
6071 model_name: m,
6072 task: InferenceTask::Generate,
6073 complexity: TaskComplexity::assess(&req.prompt),
6074 reason: "explicit model".into(),
6075 strategy: RoutingStrategy::Explicit,
6076 predicted_quality: 0.5,
6077 fallbacks: vec![],
6078 context_length: ctx_len,
6079 needs_compaction: false,
6080 candidates: vec![],
6081 }
6082 }
6083 None => {
6084 let tracker_read = self.outcome_tracker.read().await;
6085 self.adaptive_router
6086 .route_with(crate::adaptive_router::RouteRequest {
6087 estimated_total_tokens: estimated_footprint,
6088 estimated_input_tokens: estimated_input,
6089 estimated_output_tokens: req.params.max_tokens,
6090 estimated_cache_read_tokens: estimated_cache_read,
6091 estimated_cache_write_tokens: estimated_cache_write,
6092 has_tools,
6093 has_vision,
6094 workload: req.params.workload,
6095 intent: req.intent.as_ref(),
6096 ..crate::adaptive_router::RouteRequest::new(
6097 &req.prompt,
6098 &routing_registry,
6099 &tracker_read,
6100 )
6101 })
6102 }
6103 };
6104
6105 if decision.model_id.is_empty() {
6106 let excluded_models = req
6107 .intent
6108 .as_ref()
6109 .map(|hint| hint.exclude_models.join(", "))
6110 .unwrap_or_default();
6111 return Err(InferenceError::NoEligibleModel { excluded_models });
6112 }
6113
6114 #[allow(unused_mut)]
6117 let mut schema = routing_registry
6118 .get(&decision.model_id)
6119 .or_else(|| routing_registry.find_by_name(&decision.model_id))
6120 .cloned();
6121
6122 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6124 if let Some(ref s) = schema {
6125 if let Some(mlx_equiv) = routing_registry.resolve_mlx_equivalent(s) {
6126 tracing::info!(
6127 from = %s.id, to = %mlx_equiv.id,
6128 "redirecting GGUF model to MLX equivalent on Apple Silicon (stream)"
6129 );
6130 schema = Some(mlx_equiv.clone());
6131 }
6132 }
6133
6134 if has_tools
6140 && schema
6141 .as_ref()
6142 .map(|s| !s.has_capability(ModelCapability::ToolUse))
6143 .unwrap_or(false)
6144 {
6145 let backend = schema
6146 .as_ref()
6147 .map(|s| if s.is_local() { "local" } else { "remote" })
6148 .unwrap_or("unknown");
6149 return Err(InferenceError::UnsupportedMode {
6150 mode: "tool_use",
6151 backend,
6152 reason: "resolved model does not support structured tool calls; configure a tool-capable model (a remote API model, or run the vllm-mlx OpenAI-compatible server)",
6153 });
6154 }
6155
6156 let resolved_model_id = schema
6158 .as_ref()
6159 .map(|s| s.id.clone())
6160 .unwrap_or_else(|| decision.model_id.clone());
6161
6162 let mut req = req;
6169 if let Some(schema) = schema.as_ref() {
6170 req.params.max_tokens = resolved_max_tokens(req.params.max_tokens, schema);
6171 }
6172
6173 let is_remote = schema
6174 .as_ref()
6175 .map(|s| s.is_remote() || s.is_vllm_mlx())
6176 .unwrap_or(false);
6177
6178 let is_codex_cli = schema.as_ref().map(|s| s.is_codex_cli()).unwrap_or(false);
6179 let is_delegated = schema.as_ref().map(|s| s.is_delegated()).unwrap_or(false);
6180
6181 if is_codex_cli {
6182 return Err(InferenceError::UnsupportedMode {
6183 mode: "streaming",
6184 backend: "codex-cli",
6185 reason: "codex exec reports final answer items rather than token deltas; use non-streaming infer/generate_tracked",
6186 });
6187 }
6188
6189 if is_delegated {
6190 let runner = runner::current_inference_runner().ok_or_else(|| {
6194 InferenceError::InferenceFailed(
6195 "model declares ModelSource::Delegated but no inference runner is registered \
6196 (call set_inference_runner / registerInferenceRunner / register_inference_runner)"
6197 .into(),
6198 )
6199 })?;
6200 let (tx, rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6201 let emitter = runner::EventEmitter::new(tx);
6202 let request = req.clone();
6203 tokio::spawn(async move {
6204 if let Err(e) = runner.run(request, emitter).await {
6205 tracing::warn!(error = %e, "delegated inference runner failed");
6206 }
6207 });
6208 return Ok((resolved_model_id, rx));
6209 }
6210
6211 let mut local_reservation = if !is_remote
6213 || schema.as_ref().is_some_and(ModelSchema::is_vllm_mlx)
6214 {
6215 if let Some(local_schema) = schema
6216 .as_ref()
6217 .filter(|schema| Self::reserve_in_outer_dispatch(schema))
6218 {
6219 Some(
6220 self.reserve_local_request_with_worker_retry(local_schema, estimated_footprint)
6221 .await?,
6222 )
6223 } else {
6224 None
6225 }
6226 } else {
6227 None
6228 };
6229
6230 if !is_remote && schema.as_ref().is_some_and(Self::supports_worker_offload) {
6237 if let Some(offload) = crate::offload::current_local_offload() {
6238 let mut offload_req = req.clone();
6239 offload_req.model = Some(resolved_model_id.clone());
6240 let schema_ref = schema
6241 .as_ref()
6242 .ok_or_else(|| InferenceError::ModelNotFound(resolved_model_id.clone()))?;
6243 let reservation = local_reservation.as_mut().ok_or_else(|| {
6244 InferenceError::InferenceFailed(
6245 "local worker stream missing admission reservation".into(),
6246 )
6247 })?;
6248 if let Some(allocation_id) = offload.resident_allocation_id(&resolved_model_id) {
6249 reservation.bind_allocation_id(&allocation_id);
6250 }
6251 let admission = self.prepare_worker_admission(schema_ref, reservation)?;
6252 let offload_stream = offload.stream_admitted(offload_req, admission).await?;
6253 Self::reconcile_worker_residency(
6254 offload.as_ref(),
6255 &resolved_model_id,
6256 &offload_stream.residency,
6257 offload_stream.retention,
6258 reservation,
6259 )
6260 .await?;
6261 let rx = offload_stream.events;
6262 let rx = match local_reservation {
6263 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6264 None => rx,
6265 };
6266 return Ok((resolved_model_id, rx));
6267 }
6268 }
6269
6270 if is_remote {
6271 let mut candidates = vec![schema.unwrap()];
6272 let mut local_fallback_ids = Vec::new();
6273 for fallback_id in &decision.fallbacks {
6274 if let Some(fallback) = routing_registry
6275 .get(fallback_id)
6276 .or_else(|| routing_registry.find_by_name(fallback_id))
6277 {
6278 if (fallback.is_remote() || fallback.is_vllm_mlx())
6279 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6280 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6281 && !candidates
6282 .iter()
6283 .any(|candidate| candidate.id == fallback.id)
6284 {
6285 candidates.push(fallback.clone());
6286 } else if fallback.is_local()
6287 && !fallback.is_vllm_mlx()
6288 && (!has_tools || fallback.has_capability(ModelCapability::ToolUse))
6289 && (!has_vision || fallback.has_capability(ModelCapability::Vision))
6290 {
6291 local_fallback_ids.push(fallback.id.clone());
6292 }
6293 }
6294 }
6295 let mut last_error = None;
6296 for candidate in candidates {
6297 let (candidate, mut candidate_reservation) = match self
6300 .vllm_live_schema(candidate, local_reservation.take(), estimated_footprint)
6301 .await
6302 {
6303 Ok(candidate) => candidate,
6304 Err(error) => {
6305 last_error = Some(error);
6306 continue;
6307 }
6308 };
6309 self.remote_backend.register_model_keys(&candidate).await;
6310
6311 let spend_guard = self
6312 .spend_limits
6313 .read()
6314 .unwrap()
6315 .as_ref()
6316 .and_then(|limits| limits.per_request_usd)
6317 .map(|budget| {
6318 let mut prompt_tokens =
6319 routing_ext::MidStreamSpendGuard::estimate_tokens(&req.prompt)
6320 + req
6321 .context
6322 .as_deref()
6323 .map(routing_ext::MidStreamSpendGuard::estimate_tokens)
6324 .unwrap_or(0);
6325 prompt_tokens += media_tokens::request_media_and_history_tokens(
6326 req.images.as_deref(),
6327 req.messages.as_deref(),
6328 ) as u64;
6329 if let Some(tools) = &req.tools {
6330 prompt_tokens += routing_ext::MidStreamSpendGuard::estimate_tokens(
6331 &serde_json::to_string(tools).unwrap_or_default(),
6332 );
6333 }
6334 let prices = candidate.cost.prices_for(prompt_tokens as usize);
6335 let input_price = prices
6336 .input_per_mtok
6337 .map(|c| c / 1_000_000.0)
6338 .unwrap_or(0.0);
6339 let output_price = prices
6340 .output_per_mtok
6341 .map(|c| c / 1_000_000.0)
6342 .unwrap_or(0.0);
6343 routing_ext::MidStreamSpendGuard::new(
6344 Some(budget),
6345 prompt_tokens as f64 * input_price,
6346 input_price,
6347 output_price,
6348 )
6349 });
6350
6351 match self
6352 .remote_backend
6353 .with_work_context(req.work_context.clone())
6354 .generate_stream(
6355 &candidate,
6356 &req.prompt,
6357 req.messages.as_deref(),
6358 req.context.as_deref(),
6359 req.params.temperature,
6360 req.params.max_tokens,
6361 req.tools.as_deref(),
6362 req.images.as_deref(),
6363 req.params.tool_choice.as_deref(),
6364 req.params.parallel_tool_calls,
6365 req.response_format.as_ref(),
6366 spend_guard,
6367 )
6368 .await
6369 {
6370 Ok(receiver) => {
6371 let receiver = Self::hold_optional_reservation_for_stream(
6372 receiver,
6373 candidate_reservation.take(),
6374 );
6375 return Ok((candidate.id, receiver));
6376 }
6377 Err(error) => {
6378 tracing::warn!(
6379 model = %candidate.id,
6380 %error,
6381 "remote stream setup failed; trying routed fallback"
6382 );
6383 last_error = Some(error);
6384 }
6385 }
6386 }
6387
6388 if !req.params.strict_model {
6393 if let Some(local) = self.first_installed_local_model(has_tools) {
6394 let schema = routing_registry
6395 .get(&local)
6396 .or_else(|| routing_registry.find_by_name(&local));
6397 let supports_request = schema.is_some_and(|schema| {
6398 !has_vision || schema.has_capability(ModelCapability::Vision)
6399 });
6400 if supports_request
6401 && !local_fallback_ids.iter().any(|candidate| {
6402 routing_registry
6403 .get(candidate)
6404 .or_else(|| routing_registry.find_by_name(candidate))
6405 .is_some_and(|schema| schema.id == local || schema.name == local)
6406 })
6407 {
6408 local_fallback_ids.push(local);
6409 }
6410 }
6411 for local_id in local_fallback_ids {
6412 let mut fallback_req = req.clone();
6413 fallback_req.model = Some(local_id);
6414 fallback_req.params.strict_model = true;
6415 match Box::pin(self.generate_stream_raw(fallback_req)).await {
6416 Ok(stream) => return Ok(stream),
6417 Err(error) => {
6418 tracing::warn!(%error, "local streaming fallback failed");
6419 last_error = Some(error);
6420 }
6421 }
6422 }
6423 }
6424
6425 Err(last_error.unwrap_or_else(|| {
6426 InferenceError::InferenceFailed(
6427 "no compatible remote streaming model available".to_string(),
6428 )
6429 }))
6430 } else {
6431 let schema =
6432 schema.ok_or_else(|| InferenceError::ModelNotFound(decision.model_id.clone()))?;
6433 let (tx, rx) = tokio::sync::mpsc::channel(64);
6434
6435 #[cfg(any(
6441 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
6442 all(target_os = "ios", target_arch = "aarch64")
6443 ))]
6444 {
6445 if schema.is_foundation_models() {
6446 if Self::request_has_video(&req)
6450 || Self::request_has_audio(&req)
6451 || req.images.as_ref().is_some_and(|imgs| !imgs.is_empty())
6452 {
6453 return Err(InferenceError::UnsupportedMode {
6454 mode: "multimodal-content",
6455 backend: "foundation-models",
6456 reason: "the FoundationModels bridge currently exposes text-only \
6457 generation — route image/audio/video to a remote VL model",
6458 });
6459 }
6460 if let Some(tools_defs) = req.tools.clone().filter(|t| !t.is_empty()) {
6466 let prompt = req.prompt.clone();
6467 let instructions = req.context.clone();
6468 let fm_tool_choice = req.params.tool_choice.clone();
6469 let max_tokens = req.params.max_tokens as u32;
6470 let temperature = req.params.temperature;
6471 tokio::task::spawn_blocking(move || {
6472 match crate::backend::foundation_models::generate_with_tools(
6473 &prompt,
6474 instructions.as_deref(),
6475 &tools_defs,
6476 fm_tool_choice.as_deref(),
6477 max_tokens,
6478 temperature as f32,
6479 ) {
6480 Ok((text, tool_calls)) => {
6481 if !text.is_empty() {
6482 let _ = tx.blocking_send(stream::StreamEvent::TextDelta(
6483 text.clone(),
6484 ));
6485 }
6486 let _ = tx.blocking_send(stream::StreamEvent::Done {
6487 text,
6488 tool_calls,
6489 });
6490 }
6491 Err(e) => {
6492 tracing::warn!(
6493 error = %e,
6494 "FoundationModels tool-enabled stream failed"
6495 );
6496 send_stream_failure_blocking(&tx, &e);
6502 }
6503 }
6504 });
6505 return Ok((resolved_model_id, rx));
6506 }
6507 let prompt = req.prompt.clone();
6508 let instructions = req.context.clone();
6509 let max_tokens = req.params.max_tokens as u32;
6510 let temperature = req.params.temperature;
6511 let tx_clone = tx.clone();
6512 tokio::task::spawn_blocking(move || {
6513 let accum = std::sync::Arc::new(std::sync::Mutex::new(String::new()));
6520 let accum_cb = accum.clone();
6521 let cb = crate::backend::foundation_models::StreamCallback::new(
6522 move |delta: &str| {
6523 if let Ok(mut g) = accum_cb.lock() {
6524 g.push_str(delta);
6525 }
6526 tx_clone
6527 .blocking_send(stream::StreamEvent::TextDelta(
6528 delta.to_string(),
6529 ))
6530 .is_ok()
6531 },
6532 );
6533 let result = crate::backend::foundation_models::stream(
6534 &prompt,
6535 instructions.as_deref(),
6536 max_tokens,
6537 temperature as f32,
6538 cb,
6539 );
6540 let final_text = accum.lock().map(|g| g.clone()).unwrap_or_default();
6541 match &result {
6545 Ok(()) => {
6546 let _ = tx.blocking_send(stream::StreamEvent::Done {
6547 text: final_text,
6548 tool_calls: vec![],
6549 });
6550 }
6551 Err(e) => send_stream_failure_blocking(&tx, e),
6552 }
6553 result
6554 });
6555 return Ok((resolved_model_id, rx));
6556 }
6557 }
6558
6559 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6561 {
6562 if !schema.is_mlx() {
6565 return Err(InferenceError::InferenceFailed(format!(
6566 "model '{}' has no MLX equivalent; Candle backend disabled on Apple Silicon",
6567 schema.id
6568 )));
6569 }
6570 let (backend, _retention) = self
6571 .ensure_text_backend(
6572 &schema,
6573 local_reservation
6574 .as_mut()
6575 .expect("local stream has admission reservation"),
6576 )
6577 .await?;
6578 let model_id = schema.id.clone();
6579 let cache = Arc::clone(&self.text_backends);
6580 let device_guard = Self::mlx_device_lock().lock_owned().await;
6587 tokio::task::spawn_blocking(move || {
6593 let _device_guard = device_guard;
6594 let _ = Self::stream_local_mlx(backend, cache, model_id, req, tx);
6595 });
6596 let rx = match local_reservation {
6597 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6598 None => rx,
6599 };
6600 Ok((resolved_model_id, rx))
6601 }
6602
6603 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6604 {
6605 self.ensure_backend(
6606 &schema,
6607 local_reservation
6608 .as_mut()
6609 .expect("local stream has admission reservation"),
6610 )
6611 .await?;
6612 let backend = self.backend.clone();
6613 let model_id = schema.id.clone();
6614 tokio::spawn(async move {
6615 let _ = Self::stream_local_candle(backend, model_id, req, tx).await;
6616 });
6617 let rx = match local_reservation {
6618 Some(reservation) => Self::hold_local_reservation_for_stream(rx, reservation),
6619 None => rx,
6620 };
6621 Ok((resolved_model_id, rx))
6622 }
6623 }
6624 }
6625
6626 pub async fn generate_tracked_stream(
6671 &self,
6672 req: GenerateRequest,
6673 ) -> Result<TrackedStream, InferenceError> {
6674 self.refuse_system_one_generate(&req)?;
6675 let (estimated_input, _, _) = self.estimated_tokens(&req, None);
6679 let start = Instant::now();
6680
6681 let (model_id, mut producer_rx) = self.generate_stream_raw(req).await?;
6685
6686 let trace = {
6687 let mut t = self.outcome_tracker.write().await;
6688 t.record_start(&model_id, InferenceTask::Generate, "stream")
6689 };
6690 let trace_for_return = trace.clone();
6694
6695 let (out_tx, out_rx) = tokio::sync::mpsc::channel::<stream::StreamEvent>(64);
6696 let tracker = Arc::clone(&self.outcome_tracker);
6697 tokio::spawn(async move {
6698 let mut acc = stream::StreamAccumulator::default();
6699 let mut stream_error: Option<String> = None;
6700 let mut attribution: Option<outcome::UnattributedReason> = None;
6701 let mut saw_done = false;
6702 let mut receiver_abandoned = false;
6703 while let Some(evt) = producer_rx.recv().await {
6704 if let stream::StreamEvent::Attribution(reason) = &evt {
6705 attribution = Some(*reason);
6707 continue;
6708 }
6709 if let stream::StreamEvent::Error(message) = &evt {
6710 stream_error = Some(message.clone());
6711 }
6712 if matches!(evt, stream::StreamEvent::Done { .. }) {
6713 saw_done = true;
6714 }
6715 acc.push(&evt);
6716 if out_tx.send(evt).await.is_err() {
6717 receiver_abandoned = true;
6718 break;
6719 }
6720 }
6721 if stream_error.is_none() && !saw_done && !receiver_abandoned {
6722 let error = "stream ended without positive provider completion".to_string();
6723 stream_error = Some(error.clone());
6724 let _ = out_tx.send(stream::StreamEvent::Error(error)).await;
6725 } else if stream_error.is_none() && receiver_abandoned {
6726 stream_error = Some("stream receiver was abandoned before completion".to_string());
6730 attribution = Some(outcome::UnattributedReason::Cancelled);
6731 }
6732 let (text, tool_calls, usage, _stop) = acc.finish_with_usage();
6733 let latency_ms = start.elapsed().as_millis() as u64;
6734 let input_tokens = usage
6735 .as_ref()
6736 .map(|u| u.prompt_tokens as usize)
6737 .unwrap_or(estimated_input);
6738 let mut output_tokens = usage
6743 .as_ref()
6744 .map(|u| u.completion_tokens as usize)
6745 .unwrap_or_else(|| text.split_whitespace().count());
6746 if output_tokens == 0 && !tool_calls.is_empty() {
6747 output_tokens = 1;
6748 }
6749 let (cache_read, cache_creation) = usage
6754 .as_ref()
6755 .map(|u| {
6756 (
6757 u.cache_read_input_tokens as usize,
6758 u.cache_creation_input_tokens as usize,
6759 )
6760 })
6761 .unwrap_or((0, 0));
6762 let mut t = tracker.write().await;
6763 if let Some(error) = stream_error {
6764 let reason = attribution.or_else(|| {
6772 stream::content_refusal_tags(&error)
6773 .map(|_| outcome::UnattributedReason::ContentRefused)
6774 });
6775 t.note_elapsed(&trace, latency_ms);
6776 match reason {
6777 Some(reason) => t.record_not_model_fault(&trace, &error, reason),
6778 None => t.record_failure(&trace, &error),
6779 }
6780 } else {
6781 t.record_complete_cached(
6782 &trace,
6783 latency_ms,
6784 input_tokens,
6785 output_tokens,
6786 cache_read,
6787 cache_creation,
6788 );
6789 }
6790 });
6791
6792 Ok(TrackedStream {
6793 model_used: model_id,
6794 trace_id: trace_for_return,
6795 events: out_rx,
6796 })
6797 }
6798
6799 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
6800 async fn stream_local_candle(
6801 backend_lock: Arc<RwLock<std::collections::HashMap<String, CandleBackend>>>,
6802 model_id: String,
6803 req: GenerateRequest,
6804 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
6805 ) -> Result<(), InferenceError> {
6806 let mut write = backend_lock.write().await;
6807 let backend = write.get_mut(&model_id).ok_or_else(|| {
6808 InferenceError::InferenceFailed(format!("backend not initialized for {model_id}"))
6809 })?;
6810 backend.clear_kv_cache();
6811
6812 let formatted = tasks::generate::render_chat_prompt(&req);
6813 let tokens = backend.encode(&formatted)?;
6814 let eos = backend.eos_token_id();
6815 let eos_alt = backend.token_id("<|im_end|>");
6816 let params = &req.params;
6817
6818 if tokens.is_empty() {
6819 let _ = tx
6820 .send(stream::StreamEvent::Done {
6821 text: String::new(),
6822 tool_calls: vec![],
6823 })
6824 .await;
6825 return Ok(());
6826 }
6827
6828 let max_ctx = backend.context_length().unwrap_or(32768);
6829 let headroom = params.max_tokens.min(max_ctx / 4);
6830 let max_prompt = max_ctx.saturating_sub(headroom);
6831 let tokens = if tokens.len() > max_prompt {
6832 tokens[tokens.len() - max_prompt..].to_vec()
6833 } else {
6834 tokens
6835 };
6836
6837 let mut generated = Vec::new();
6838 let logits = backend.forward(&tokens, 0)?;
6839 let mut next_token = tasks::generate::sample_token(&logits, params)?;
6840
6841 for _ in 0..params.max_tokens {
6842 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
6843 break;
6844 }
6845
6846 generated.push(next_token);
6847 let delta = backend.decode(&[next_token])?;
6848 if !delta.is_empty()
6849 && tx
6850 .send(stream::StreamEvent::TextDelta(delta))
6851 .await
6852 .is_err()
6853 {
6854 return Ok(());
6855 }
6856
6857 if !params.stop.is_empty() {
6858 let text_so_far = backend.decode(&generated)?;
6859 if params.stop.iter().any(|s| text_so_far.contains(s)) {
6860 break;
6861 }
6862 }
6863
6864 let pos = tokens.len() + generated.len() - 1;
6865 let logits = backend.forward(&[next_token], pos)?;
6866 next_token = tasks::generate::sample_token(&logits, params)?;
6867 }
6868
6869 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
6870 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
6871 let _ = tx
6875 .send(stream::StreamEvent::Usage {
6876 input_tokens: tokens.len() as u64,
6877 output_tokens: generated.len() as u64,
6878 cache_read_input_tokens: 0,
6879 cache_creation_input_tokens: 0,
6880 })
6881 .await;
6882 let _ = tx
6883 .send(stream::StreamEvent::Done {
6884 text,
6885 tool_calls: vec![],
6886 })
6887 .await;
6888 Ok(())
6889 }
6890
6891 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6895 fn stream_local_mlx(
6896 handle: backend_cache::CachedBackend<backend::SwiftLmBackend>,
6897 cache: Arc<backend_cache::BackendCache<backend::SwiftLmBackend>>,
6898 model_id: String,
6899 req: GenerateRequest,
6900 tx: tokio::sync::mpsc::Sender<stream::StreamEvent>,
6901 ) -> Result<(), InferenceError> {
6902 let mut guard = handle.lock().map_err(|_| {
6903 InferenceError::InferenceFailed(format!("MLX backend mutex poisoned for {model_id}"))
6904 })?;
6905 let backend: &mut backend::SwiftLmBackend = &mut guard;
6906 backend.clear_kv_cache();
6907
6908 let formatted = tasks::generate::render_chat_prompt(&req);
6909 let tokens = backend.encode(&formatted)?;
6910 let eos = backend.eos_token_id();
6911 let eos_alt = backend.token_id("<|im_end|>");
6912 let params = &req.params;
6913
6914 if tokens.is_empty() {
6915 let _ = tx.blocking_send(stream::StreamEvent::Done {
6916 text: String::new(),
6917 tool_calls: vec![],
6918 });
6919 return Ok(());
6920 }
6921
6922 let max_ctx = backend.context_length();
6923 let headroom = params.max_tokens.min(max_ctx / 4);
6924 let max_prompt = max_ctx.saturating_sub(headroom);
6925 let tokens = if tokens.len() > max_prompt {
6926 tokens[tokens.len() - max_prompt..].to_vec()
6927 } else {
6928 tokens
6929 };
6930
6931 let mut generated = Vec::new();
6932
6933 let started = std::time::Instant::now();
6939 let timeout = local_decode_timeout();
6940 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
6941 let mut last_heartbeat = std::time::Duration::ZERO;
6942 tracing::info!(
6943 prompt_tokens = tokens.len(),
6944 max_tokens = params.max_tokens,
6945 timeout_secs = timeout.map(|t| t.as_secs()),
6946 "local stream prefill starting"
6947 );
6948
6949 let logits = match Self::catch_mlx("stream prefill", || backend.forward(&tokens, 0)) {
6954 Ok(v) => v,
6955 Err(e) => {
6956 cache.invalidate(&model_id);
6957 return Err(e);
6958 }
6959 };
6960 let mut next_token = Self::sample_from_logits(&logits, params)?;
6961
6962 for _ in 0..params.max_tokens {
6963 if (eos == Some(next_token)) || (eos_alt == Some(next_token)) {
6964 break;
6965 }
6966
6967 generated.push(next_token);
6968 let delta = backend.decode(&[next_token])?;
6969 if !delta.is_empty()
6970 && tx
6971 .blocking_send(stream::StreamEvent::TextDelta(delta))
6972 .is_err()
6973 {
6974 return Ok(());
6975 }
6976
6977 if !params.stop.is_empty() {
6978 let text_so_far = backend.decode(&generated)?;
6979 if params.stop.iter().any(|s| text_so_far.contains(s)) {
6980 break;
6981 }
6982 }
6983
6984 let elapsed = started.elapsed();
6989 if deadline_exceeded(elapsed, timeout) {
6990 tracing::warn!(
6991 elapsed_secs = elapsed.as_secs_f64(),
6992 timeout_secs = timeout.map(|t| t.as_secs()),
6993 completion_tokens = generated.len(),
6994 max_tokens = params.max_tokens,
6995 "local stream hit its wall-clock ceiling and was cut short; \
6996 returning what was streamed so far. Raise or disable it with \
6997 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
6998 );
6999 let _ = tx.blocking_send(stream::StreamEvent::StopReason(
7004 LOCAL_DECODE_TIMEOUT_STOP_REASON.to_string(),
7005 ));
7006 break;
7007 }
7008 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7009 last_heartbeat = elapsed;
7010 tracing::info!(
7011 completion_tokens = generated.len(),
7012 max_tokens = params.max_tokens,
7013 elapsed_secs = elapsed.as_secs_f64(),
7014 tokens_per_sec =
7015 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7016 "local stream in progress"
7017 );
7018 }
7019
7020 let pos = tokens.len() + generated.len() - 1;
7021 let logits =
7022 match Self::catch_mlx("stream forward", || backend.forward(&[next_token], pos)) {
7023 Ok(v) => v,
7024 Err(e) => {
7025 cache.invalidate(&model_id);
7026 return Err(e);
7027 }
7028 };
7029 next_token = Self::sample_from_logits(&logits, params)?;
7030 }
7031
7032 let trimmed = tasks::generate::truncate_at_stop(&backend.decode(&generated)?, ¶ms.stop);
7033 let text = tasks::generate::strip_thinking(&trimmed, params.thinking);
7034 let _ = tx.blocking_send(stream::StreamEvent::Usage {
7042 input_tokens: tokens.len() as u64,
7043 output_tokens: generated.len() as u64,
7044 cache_read_input_tokens: 0,
7046 cache_creation_input_tokens: 0,
7047 });
7048 let _ = tx.blocking_send(stream::StreamEvent::Done {
7049 text,
7050 tool_calls: vec![],
7051 });
7052 Ok(())
7053 }
7054
7055 pub async fn route_context_snapshot(
7057 &self,
7058 prompt: &str,
7059 workload: RoutingWorkload,
7060 has_tools: bool,
7061 has_vision: bool,
7062 ) -> AdaptiveRoutingDecision {
7063 let routing_registry = self.catalog_registry_snapshot();
7064 let tracker = self.outcome_tracker.read().await;
7065 self.adaptive_router.route_context_aware(
7066 prompt,
7067 0,
7068 &routing_registry,
7069 &tracker,
7070 has_tools,
7071 has_vision,
7072 workload,
7073 )
7074 }
7075
7076 pub async fn generate(&self, req: GenerateRequest) -> Result<String, InferenceError> {
7079 Ok(self.generate_tracked(req).await?.text)
7080 }
7081
7082 pub async fn tokenize(&self, model: &str, text: &str) -> Result<Vec<u32>, InferenceError> {
7094 self.assert_local_for_tokenize(model)?;
7095 let admission_schema = self
7096 .resolve_schema(model)
7097 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7098 .clone();
7099 let mut reservation =
7101 self.reserve_local_request(&admission_schema, text.len().div_ceil(4))?;
7102
7103 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7104 {
7105 let schema = admission_schema;
7106 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7107 let guard = handle.lock().map_err(|_| {
7108 InferenceError::InferenceFailed(format!(
7109 "MLX backend mutex poisoned for {}",
7110 schema.id
7111 ))
7112 })?;
7113 let tokens = guard.tokenize_raw(text)?;
7114 Ok(tokens)
7115 }
7116
7117 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7118 {
7119 self.ensure_backend(&admission_schema, &mut reservation)
7120 .await?;
7121 let read = self.backend.read().await;
7122 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7123 InferenceError::InferenceFailed(
7124 "candle backend missing after ensure_backend".to_string(),
7125 )
7126 })?;
7127 let tokens = backend.tokenize_raw(text)?;
7128 Ok(tokens)
7129 }
7130 }
7131
7132 pub async fn detokenize(&self, model: &str, tokens: &[u32]) -> Result<String, InferenceError> {
7134 self.assert_local_for_tokenize(model)?;
7135 let admission_schema = self
7136 .resolve_schema(model)
7137 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?
7138 .clone();
7139 let mut reservation = self.reserve_local_request(&admission_schema, tokens.len())?;
7141
7142 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7143 {
7144 let schema = admission_schema;
7145 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7146 let guard = handle.lock().map_err(|_| {
7147 InferenceError::InferenceFailed(format!(
7148 "MLX backend mutex poisoned for {}",
7149 schema.id
7150 ))
7151 })?;
7152 let text = guard.detokenize_raw(tokens)?;
7153 Ok(text)
7154 }
7155
7156 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7157 {
7158 self.ensure_backend(&admission_schema, &mut reservation)
7159 .await?;
7160 let read = self.backend.read().await;
7161 let backend = read.get(&admission_schema.id).ok_or_else(|| {
7162 InferenceError::InferenceFailed(
7163 "candle backend missing after ensure_backend".to_string(),
7164 )
7165 })?;
7166 let text = backend.detokenize_raw(tokens)?;
7167 Ok(text)
7168 }
7169 }
7170
7171 fn assert_local_for_tokenize(&self, model: &str) -> Result<(), InferenceError> {
7175 if let Some(schema) = self.resolve_schema(model) {
7176 if !schema.is_local() {
7177 return Err(InferenceError::UnsupportedMode {
7178 mode: "tokenize/detokenize",
7179 backend: "remote",
7180 reason: "remote provider tokenizer is not exposed by the runtime; \
7181 use a local model (Qwen3 GGUF / MLX) for tokenizer-correctness checks",
7182 });
7183 }
7184 }
7185 Ok(())
7187 }
7188
7189 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7195 fn catch_mlx<F, T>(context: &str, f: F) -> Result<T, InferenceError>
7196 where
7197 F: FnOnce() -> Result<T, InferenceError>,
7198 {
7199 std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|e| {
7200 InferenceError::InferenceFailed(format!("MLX panicked during {context}: {e:?}"))
7201 })?
7202 }
7203
7204 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7214 fn drive_generation(
7215 backend: &mut dyn backend::local::TextDecoder,
7216 prompt: &str,
7217 params: &GenerateParams,
7218 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7219 Self::drive_generation_with_timeout(backend, prompt, params, local_decode_timeout())
7220 }
7221
7222 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7226 fn drive_generation_with_timeout(
7227 backend: &mut dyn backend::local::TextDecoder,
7228 prompt: &str,
7229 params: &GenerateParams,
7230 timeout: Option<std::time::Duration>,
7231 ) -> Result<backend::local::LocalGeneration, backend::local::DriveError> {
7232 use backend::local::{DriveError, LocalGeneration};
7233
7234 let start = std::time::Instant::now();
7235
7236 let tokens = backend.encode(prompt).map_err(DriveError::Recoverable)?;
7237 let eos_ids = backend.eos_ids();
7238
7239 if tokens.is_empty() {
7240 backend.clear_kv_cache();
7241 return Ok(LocalGeneration {
7242 text: String::new(),
7243 ttft_ms: None,
7244 stop_reason: None,
7245 prompt_tokens: 0,
7246 completion_tokens: 0,
7247 });
7248 }
7249
7250 let max_ctx = backend.context_length();
7252 let headroom = params.max_tokens.min(max_ctx / 4);
7253 let max_prompt = max_ctx.saturating_sub(headroom);
7254 let tokens = if tokens.len() > max_prompt {
7255 tokens[tokens.len() - max_prompt..].to_vec()
7256 } else {
7257 tokens
7258 };
7259
7260 let mut generated = Vec::new();
7261
7262 tracing::info!(
7269 prompt_tokens = tokens.len(),
7270 max_tokens = params.max_tokens,
7271 timeout_secs = timeout.map(|t| t.as_secs()),
7272 "local prefill starting"
7273 );
7274
7275 let offset = backend.begin_prompt(&tokens);
7280 let logits = Self::catch_mlx("prefill", || backend.forward(&tokens[offset..], offset))
7281 .map_err(DriveError::BackendCorrupted)?;
7282 let mut next_token =
7283 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7284 let ttft_ms = Some(start.elapsed().as_millis() as u64);
7285
7286 tracing::info!(
7291 prompt_tokens = tokens.len(),
7292 max_tokens = params.max_tokens,
7293 prefill_ms = ttft_ms,
7294 timeout_secs = timeout.map(|t| t.as_secs()),
7295 "local decode starting"
7296 );
7297
7298 let mut natural_stop = false;
7301 let mut timed_out = false;
7302 let heartbeat = std::time::Duration::from_secs(LOCAL_DECODE_HEARTBEAT_SECS);
7303 let mut last_heartbeat = std::time::Duration::ZERO;
7304 for _ in 0..params.max_tokens {
7305 if eos_ids.contains(&next_token) {
7306 natural_stop = true;
7307 break;
7308 }
7309
7310 generated.push(next_token);
7311
7312 if !params.stop.is_empty() {
7313 let text_so_far = backend
7314 .decode(&generated)
7315 .map_err(DriveError::Recoverable)?;
7316 if params.stop.iter().any(|s| text_so_far.contains(s)) {
7317 natural_stop = true;
7318 break;
7319 }
7320 }
7321
7322 let elapsed = start.elapsed();
7323 if deadline_exceeded(elapsed, timeout) {
7324 timed_out = true;
7325 break;
7326 }
7327 if heartbeat_due(elapsed, last_heartbeat, heartbeat) {
7328 last_heartbeat = elapsed;
7329 tracing::info!(
7330 completion_tokens = generated.len(),
7331 max_tokens = params.max_tokens,
7332 elapsed_secs = elapsed.as_secs_f64(),
7333 tokens_per_sec =
7334 generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON),
7335 "local decode in progress"
7336 );
7337 }
7338
7339 let pos = tokens.len() + generated.len() - 1;
7340 let logits = Self::catch_mlx("forward", || backend.forward(&[next_token], pos))
7341 .map_err(DriveError::BackendCorrupted)?;
7342 next_token =
7343 Self::sample_from_logits(&logits, params).map_err(DriveError::Recoverable)?;
7344 }
7345
7346 let elapsed = start.elapsed();
7347 let tokens_per_sec = generated.len() as f64 / elapsed.as_secs_f64().max(f64::EPSILON);
7348 if timed_out {
7349 tracing::warn!(
7350 elapsed_secs = elapsed.as_secs_f64(),
7351 timeout_secs = timeout.map(|t| t.as_secs()),
7352 completion_tokens = generated.len(),
7353 max_tokens = params.max_tokens,
7354 tokens_per_sec,
7355 "local decode hit its wall-clock ceiling and was cut short; \
7356 returning the partial response. Raise or disable it with \
7357 CAR_LOCAL_DECODE_TIMEOUT_SECS (0 disables)."
7358 );
7359 } else {
7360 tracing::info!(
7361 elapsed_secs = elapsed.as_secs_f64(),
7362 completion_tokens = generated.len(),
7363 max_tokens = params.max_tokens,
7364 tokens_per_sec,
7365 natural_stop,
7366 "local decode finished"
7367 );
7368 }
7369
7370 let decoded = backend
7371 .decode(&generated)
7372 .map_err(DriveError::Recoverable)?;
7373 let text = tasks::generate::truncate_at_stop(&decoded, ¶ms.stop);
7374 let stop_reason = Some(
7375 if timed_out {
7376 LOCAL_DECODE_TIMEOUT_STOP_REASON
7377 } else if natural_stop {
7378 "stop"
7379 } else {
7380 "length"
7381 }
7382 .to_string(),
7383 );
7384 Ok(LocalGeneration {
7385 text: tasks::generate::strip_thinking(&text, params.thinking),
7386 ttft_ms,
7387 stop_reason,
7388 prompt_tokens: tokens.len(),
7389 completion_tokens: generated.len(),
7390 })
7391 }
7392
7393 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7396 async fn generate_mlx(
7406 &self,
7407 req: GenerateRequest,
7408 model_id: &str,
7409 reservation: &mut resource_policy::LocalLoadReservation,
7410 ) -> Result<(String, Option<TokenUsage>, Option<u64>, Option<String>), InferenceError> {
7411 let schema = self
7412 .unified_registry()
7413 .get(model_id)
7414 .cloned()
7415 .ok_or_else(|| {
7416 InferenceError::InferenceFailed(format!(
7417 "generate_mlx: unknown schema id {model_id}"
7418 ))
7419 })?;
7420 let (handle, _retention) = self.ensure_text_backend(&schema, reservation).await?;
7421 let formatted = tasks::generate::render_chat_prompt(&req);
7428 let params = req.params.clone();
7429 let model_id = model_id.to_string();
7430 let cache = self.text_backends.clone();
7431 let device_guard = Self::mlx_device_lock().lock_owned().await;
7432 let detached_lease = reservation.detached_lease();
7433
7434 run_admitted_blocking(detached_lease, move || {
7440 let _device_guard = device_guard;
7441 let mut guard = handle.lock().map_err(|_| {
7442 InferenceError::InferenceFailed(format!(
7443 "MLX backend mutex poisoned for {model_id}"
7444 ))
7445 })?;
7446 let backend: &mut backend::SwiftLmBackend = &mut guard;
7447 let ctx_window = backend.context_length() as u64;
7448 match Self::drive_generation(backend, &formatted, ¶ms) {
7449 Ok(gen) => {
7450 let usage = TokenUsage {
7451 prompt_tokens: gen.prompt_tokens as u64,
7452 completion_tokens: gen.completion_tokens as u64,
7453 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7454 context_window: ctx_window,
7455 ..Default::default()
7456 };
7457 Ok((gen.text, Some(usage), gen.ttft_ms, gen.stop_reason))
7458 }
7459 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
7460 Err(backend::local::DriveError::BackendCorrupted(error)) => {
7461 drop(guard);
7462 cache.invalidate(&model_id);
7463 Err(error)
7464 }
7465 }
7466 })
7467 .await
7468 .map_err(|error| {
7469 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
7470 })?
7471 }
7472
7473 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7478 async fn generate_local(
7479 &self,
7480 req: GenerateRequest,
7481 model_id: &str,
7482 reservation: &mut resource_policy::LocalLoadReservation,
7483 ) -> Result<
7484 (
7485 String,
7486 Vec<tasks::generate::ToolCall>,
7487 Option<TokenUsage>,
7488 Option<u64>,
7489 Option<String>,
7490 ),
7491 InferenceError,
7492 > {
7493 let schema = self
7494 .unified_registry()
7495 .get(model_id)
7496 .cloned()
7497 .ok_or_else(|| {
7498 InferenceError::InferenceFailed(format!(
7499 "generate_local: unknown schema id {model_id}"
7500 ))
7501 })?;
7502 let (handle, _retention) = self.ensure_local_backend(&schema, reservation).await?;
7503 let params = req.params.clone();
7504 let model_id = model_id.to_string();
7505 let cache = self.local_backends.clone();
7506 let device_guard = Self::mlx_device_lock().lock_owned().await;
7507 let detached_lease = reservation.detached_lease();
7508
7509 run_admitted_blocking(detached_lease, move || {
7510 let _device_guard = device_guard;
7511 let mut guard = handle.lock().map_err(|_| {
7512 InferenceError::InferenceFailed(format!(
7513 "local backend mutex poisoned for {model_id}"
7514 ))
7515 })?;
7516 let formatted = guard.render_prompt(&req)?;
7517 let backend: &mut dyn backend::local::TextDecoder = &mut **guard;
7518 let outcome = Self::drive_generation(backend, &formatted, ¶ms);
7519 match outcome {
7520 Ok(gen) => {
7521 let (clean, tool_calls) = guard.parse_tool_calls(&gen.text);
7522 let usage = TokenUsage {
7523 prompt_tokens: gen.prompt_tokens as u64,
7524 completion_tokens: gen.completion_tokens as u64,
7525 total_tokens: (gen.prompt_tokens + gen.completion_tokens) as u64,
7526 context_window: guard.context_length() as u64,
7527 ..Default::default()
7528 };
7529 Ok((clean, tool_calls, Some(usage), gen.ttft_ms, gen.stop_reason))
7530 }
7531 Err(backend::local::DriveError::Recoverable(error)) => Err(error),
7532 Err(backend::local::DriveError::BackendCorrupted(error)) => {
7533 drop(guard);
7534 cache.invalidate(&model_id);
7535 Err(error)
7536 }
7537 }
7538 })
7539 .await
7540 .map_err(|error| {
7541 InferenceError::InferenceFailed(format!("native local task panicked: {error}"))
7542 })?
7543 }
7544
7545 #[allow(dead_code)] fn apply_top_k_top_p(probs: &mut [f32], top_k: usize, top_p: f64) {
7553 if top_k > 0 && top_k < probs.len() {
7555 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
7556 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
7557 let allowed: std::collections::HashSet<usize> =
7558 indexed[..top_k].iter().map(|(i, _)| *i).collect();
7559 for (i, p) in probs.iter_mut().enumerate() {
7560 if !allowed.contains(&i) {
7561 *p = 0.0;
7562 }
7563 }
7564 let sum: f32 = probs.iter().sum();
7565 if sum > 0.0 {
7566 for p in probs.iter_mut() {
7567 *p /= sum;
7568 }
7569 }
7570 }
7571
7572 if top_p < 1.0 {
7574 let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
7575 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
7576 let mut cumsum = 0.0f32;
7577 let mut cutoff_idx = indexed.len();
7578 for (i, &(_, p)) in indexed.iter().enumerate() {
7579 cumsum += p;
7580 if cumsum > top_p as f32 {
7581 cutoff_idx = i + 1;
7582 break;
7583 }
7584 }
7585 let allowed: std::collections::HashSet<usize> =
7586 indexed[..cutoff_idx].iter().map(|(i, _)| *i).collect();
7587 for (i, p) in probs.iter_mut().enumerate() {
7588 if !allowed.contains(&i) {
7589 *p = 0.0;
7590 }
7591 }
7592 let sum: f32 = probs.iter().sum();
7593 if sum > 0.0 {
7594 for p in probs.iter_mut() {
7595 *p /= sum;
7596 }
7597 }
7598 }
7599 }
7600
7601 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7603 fn sample_from_logits(logits: &[f32], params: &GenerateParams) -> Result<u32, InferenceError> {
7604 if params.temperature <= 0.0 {
7605 let (idx, _) = logits
7607 .iter()
7608 .enumerate()
7609 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
7610 .ok_or_else(|| InferenceError::InferenceFailed("empty logits".into()))?;
7611 return Ok(idx as u32);
7612 }
7613
7614 let temp = params.temperature as f32;
7616 let max_logit = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
7617 let mut probs: Vec<f32> = logits
7618 .iter()
7619 .map(|&l| ((l - max_logit) / temp).exp())
7620 .collect();
7621 let sum: f32 = probs.iter().sum();
7622 for p in &mut probs {
7623 *p /= sum;
7624 }
7625
7626 Self::apply_top_k_top_p(&mut probs, params.top_k, params.top_p);
7632
7633 use rand::Rng;
7635 let mut rng = rand::rng();
7636 let r: f32 = rng.random();
7637 let mut cumsum = 0.0;
7638 for (i, &p) in probs.iter().enumerate() {
7639 cumsum += p;
7640 if cumsum >= r {
7641 return Ok(i as u32);
7642 }
7643 }
7644 Ok((probs.len() - 1) as u32)
7645 }
7646
7647 pub async fn embed(&self, req: EmbedRequest) -> Result<Vec<Vec<f32>>, InferenceError> {
7650 let instruction = req
7651 .instruction
7652 .as_deref()
7653 .unwrap_or("Retrieve relevant memory facts");
7654 let embedding_model = self
7655 .preferred_model_for_capability(ModelCapability::Embed)
7656 .unwrap_or(&self.config.embedding_model);
7657 let admission_schema = self
7658 .resolve_schema(embedding_model)
7659 .ok_or_else(|| InferenceError::ModelNotFound(embedding_model.to_string()))?
7660 .clone();
7661 let estimated_tokens = req.texts.iter().map(|text| text.len().div_ceil(4)).sum();
7662 let mut reservation = self.reserve_local_request(&admission_schema, estimated_tokens)?;
7664
7665 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7666 {
7667 let model_id = self.ensure_mlx_embedding_backend().await?;
7668 let schema = self
7669 .unified_registry()
7670 .get(&model_id)
7671 .cloned()
7672 .ok_or_else(|| {
7673 InferenceError::InferenceFailed(format!("embed: unknown schema id {model_id}"))
7674 })?;
7675 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
7676 let _device_guard = Self::mlx_device_lock().lock_owned().await;
7680 let mut guard = handle.lock().map_err(|_| {
7681 InferenceError::InferenceFailed(format!(
7682 "MLX embedding backend mutex poisoned for {model_id}"
7683 ))
7684 })?;
7685 let backend: &mut backend::SwiftLmBackend = &mut guard;
7686
7687 let mut results = Vec::with_capacity(req.texts.len());
7688 for text in &req.texts {
7689 let embedding = if req.is_query {
7690 backend.embed_query(text, instruction)?
7691 } else {
7692 backend.embed_one(text)?
7693 };
7694 results.push(embedding);
7695 }
7696 Ok(results)
7697 }
7698
7699 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7700 {
7701 self.ensure_embedding_backend(&mut reservation).await?;
7702 let mut write = self.embedding_backend.write().await;
7703 let backend = write.as_mut().unwrap();
7704
7705 let mut results = Vec::with_capacity(req.texts.len());
7706 for text in &req.texts {
7707 let embedding = if req.is_query {
7708 backend.embed_query(text, instruction)?
7709 } else {
7710 backend.embed_one(text)?
7711 };
7712 results.push(embedding);
7713 }
7714 Ok(results)
7715 }
7716 }
7717
7718 pub async fn rerank(&self, req: RerankRequest) -> Result<RerankResult, InferenceError> {
7752 if req.documents.is_empty() {
7753 return Ok(RerankResult {
7754 ranked: Vec::new(),
7755 model_used: None,
7756 });
7757 }
7758
7759 let model_name = match req.model.clone() {
7760 Some(m) => m,
7761 None => self
7762 .preferred_model_for_capability(ModelCapability::Rerank)
7763 .map(str::to_string)
7764 .ok_or_else(|| {
7765 InferenceError::InferenceFailed(
7766 "no reranker model available — pull a Qwen3-Reranker model first".into(),
7767 )
7768 })?,
7769 };
7770
7771 let schema = self.resolve_schema_name_first(&model_name).ok_or_else(|| {
7772 InferenceError::InferenceFailed(format!("rerank: unknown reranker model {model_name}"))
7773 })?;
7774 if !schema.has_capability(ModelCapability::Rerank) {
7775 return Err(InferenceError::InferenceFailed(format!(
7776 "model {} does not declare the Rerank capability",
7777 schema.name
7778 )));
7779 }
7780
7781 let instruction = req.instruction.as_deref().unwrap_or(
7782 "Given a web search query, retrieve relevant passages that answer the query",
7783 );
7784
7785 let mut scored: Vec<RerankedDocument> = Vec::with_capacity(req.documents.len());
7786 for (idx, doc) in req.documents.iter().enumerate() {
7787 let prompt = rerank_prompt(instruction, &req.query, doc);
7788 let gen_req = GenerateRequest {
7789 prompt,
7790 model: Some(schema.id.clone()),
7791 params: tasks::generate::GenerateParams {
7792 temperature: 0.0,
7793 max_tokens: 3,
7797 thinking: tasks::generate::ThinkingMode::Off,
7798 ..Default::default()
7799 },
7800 context: None,
7801 context_stable_prefix: None,
7802 tools: None,
7803 images: None,
7804 messages: None,
7805 cache_control: false,
7806 response_format: None,
7807 intent: None,
7808 client_ref: None,
7809 expected_row_digest: None,
7810 expected_catalog_revision: None,
7811 caller: None,
7812 work_context: None,
7813 };
7814 let out = self.generate(gen_req).await?;
7815 let score = score_from_rerank_output(&out, &schema.name);
7816 scored.push(RerankedDocument {
7817 index: idx,
7818 score,
7819 document: doc.clone(),
7820 });
7821 }
7822
7823 scored.sort_by(|a, b| {
7826 b.score
7827 .partial_cmp(&a.score)
7828 .unwrap_or(std::cmp::Ordering::Equal)
7829 .then_with(|| a.index.cmp(&b.index))
7830 });
7831 if let Some(n) = req.top_n {
7832 scored.truncate(n);
7833 }
7834
7835 Ok(RerankResult {
7836 ranked: scored,
7837 model_used: Some(schema.name),
7838 })
7839 }
7840
7841 pub async fn ground(&self, req: GroundRequest) -> Result<GroundResult, InferenceError> {
7851 let model_name = match req.model.clone() {
7852 Some(m) => m,
7853 None => self
7854 .preferred_model_for_capability(ModelCapability::Grounding)
7855 .map(str::to_string)
7856 .ok_or_else(|| {
7857 InferenceError::InferenceFailed(
7858 "no grounding-capable model available — pull a Qwen2.5-VL model first"
7859 .into(),
7860 )
7861 })?,
7862 };
7863
7864 let gen_req = GenerateRequest {
7865 prompt: req.prompt.clone(),
7866 model: Some(model_name),
7867 params: GenerateParams::default(),
7868 context: None,
7869 context_stable_prefix: None,
7870 tools: None,
7871 images: Some(vec![req.image.clone()]),
7872 messages: None,
7873 cache_control: false,
7874 response_format: None,
7875 intent: None,
7876 client_ref: None,
7877 expected_row_digest: None,
7878 expected_catalog_revision: None,
7879 caller: None,
7880 work_context: None,
7881 };
7882 let result = self.generate_tracked(gen_req).await?;
7883 Ok(GroundResult {
7884 boxes: result.bounding_boxes,
7885 raw_text: result.text,
7886 model_used: Some(result.model_used),
7887 })
7888 }
7889
7890 pub async fn classify(
7893 &self,
7894 req: ClassifyRequest,
7895 ) -> Result<Vec<ClassifyResult>, InferenceError> {
7896 let registry = if let Some(context) = &req.work_context {
7897 Arc::new(
7898 self.work_context_routing_snapshot(context, req.model.as_deref())
7899 .await
7900 .0,
7901 )
7902 } else {
7903 self.unified_registry()
7904 };
7905 let model = match req.model.clone().or_else(|| {
7906 self.preferred_model_for_capability(ModelCapability::Classify)
7907 .map(str::to_string)
7908 }) {
7909 Some(m) => m,
7910 None => match default_system_one_classifier(registry.all()) {
7914 Some(m) => {
7915 debug!(model = %m, "classify defaulted to an available System One model");
7916 m
7917 }
7918 None => {
7919 let m = self.router.route_small(&self.registry);
7920 debug!(model = %m, "auto-routed classify request");
7921 m
7922 }
7923 },
7924 };
7925 let schema = self
7926 .resolve_schema(&model)
7927 .ok_or_else(|| InferenceError::ModelNotFound(model.clone()))?
7928 .clone();
7929
7930 if let ModelSource::Proprietary { protocol, .. } = &schema.source {
7931 if protocol.wire == ProprietaryWire::SystemOne {
7932 return self.classify_via_system_one(req, &schema).await;
7933 }
7934 }
7935 if !schema.is_local() {
7936 return self.classify_via_generate(req, &schema.id).await;
7937 }
7938
7939 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7942 {
7943 return self.classify_via_generate(req, &model).await;
7944 }
7945
7946 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
7947 {
7948 let mut reservation =
7950 self.reserve_local_request(&schema, req.text.len().div_ceil(4))?;
7951 self.ensure_backend(&schema, &mut reservation).await?;
7952 let mut write = self.backend.write().await;
7953 let backend = write.get_mut(&schema.id).ok_or_else(|| {
7954 InferenceError::InferenceFailed(format!(
7955 "candle backend missing after ensure_backend for {}",
7956 schema.id
7957 ))
7958 })?;
7959 let result = tasks::classify::classify(backend, req).await?;
7960 Ok(result)
7961 }
7962 }
7963
7964 fn refuse_system_one_generate(&self, req: &GenerateRequest) -> Result<(), InferenceError> {
7973 let Some(model) = req.model.as_deref() else {
7974 return Ok(());
7975 };
7976 let model = model.strip_prefix(EXACT_MODEL_ID_PREFIX).unwrap_or(model);
7977 let schema = self.resolve_schema(model);
7978 match schema.as_ref().map(|s| &s.source) {
7979 Some(ModelSource::Proprietary { protocol, .. })
7980 if protocol.wire == ProprietaryWire::SystemOne =>
7981 {
7982 Err(InferenceError::UnsupportedMode {
7983 mode: "generate",
7984 backend: "parslee-system-one",
7985 reason: "a System One model returns typed decisions, not text; use classify",
7986 })
7987 }
7988 _ => Ok(()),
7989 }
7990 }
7991
7992 pub async fn option_probabilities(
8011 &self,
8012 model: &str,
8013 prompt: &str,
8014 options: &[String],
8015 ) -> Result<OptionProbabilities, InferenceError> {
8016 if options.is_empty() {
8017 return Err(InferenceError::InvalidClassifyLabels(
8018 "no options given".into(),
8019 ));
8020 }
8021 let schema = self
8022 .resolve_schema(model)
8023 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
8024 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8025 {
8026 let _ = (schema, prompt);
8027 Err(InferenceError::UnsupportedMode {
8028 mode: "option-logits",
8029 backend: "local",
8030 reason: "option probabilities read logits from the native MLX text backend, \
8031 which is Apple Silicon only",
8032 })
8033 }
8034 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8035 {
8036 if !schema.is_local() || !schema.family.to_lowercase().starts_with("qwen") {
8037 return Err(InferenceError::UnsupportedMode {
8038 mode: "option-logits",
8039 backend: "local",
8040 reason: "option probabilities need a local Qwen-family model: the prompt is \
8041 rendered with Qwen's chat template",
8042 });
8043 }
8044 let req = GenerateRequest {
8045 prompt: prompt.to_string(),
8046 model: Some(schema.id.clone()),
8047 params: tasks::generate::GenerateParams {
8048 thinking: tasks::generate::ThinkingMode::Off,
8049 ..Default::default()
8050 },
8051 ..Default::default()
8052 };
8053 let formatted = tasks::generate::render_chat_prompt(&req);
8054 let mut reservation =
8055 self.reserve_local_request(&schema, formatted.len().div_ceil(4))?;
8056 let (handle, _retention) = self.ensure_text_backend(&schema, &mut reservation).await?;
8057 let options = options.to_vec();
8058 let model_id = schema.id.clone();
8059 let device_guard = Self::mlx_device_lock().lock_owned().await;
8060 let detached_lease = reservation.detached_lease();
8061 run_admitted_blocking(detached_lease, move || {
8062 let _device_guard = device_guard;
8063 let mut guard = handle.lock().map_err(|_| {
8064 InferenceError::InferenceFailed(format!(
8065 "MLX backend mutex poisoned for {model_id}"
8066 ))
8067 })?;
8068 let backend: &mut backend::SwiftLmBackend = &mut guard;
8069 let result = tasks::classify::score_options(backend, &formatted, &options);
8070 backend.clear_kv_cache();
8071 result
8072 })
8073 .await
8074 .map_err(|error| {
8075 InferenceError::InferenceFailed(format!("native MLX task panicked: {error}"))
8076 })?
8077 }
8078 }
8079
8080 async fn classify_via_system_one(
8085 &self,
8086 req: ClassifyRequest,
8087 schema: &ModelSchema,
8088 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8089 let body = tasks::classify::system_one_request_body(&schema.name, &req.text, &req.labels)?;
8090 let trace = {
8093 let mut tracker = self.outcome_tracker.write().await;
8094 tracker.record_start(
8095 &schema.id,
8096 InferenceTask::Classify,
8097 "explicit system-one model",
8098 )
8099 };
8100 let start = Instant::now();
8101 let result = match self
8102 .remote_backend
8103 .with_work_context(req.work_context.clone())
8104 .parslee_system_one(schema, &body)
8105 .await
8106 {
8107 Ok(response) => tasks::classify::system_one_results(&response, &req.labels)
8108 .map(|results| (results, response)),
8109 Err(error) => Err(error),
8110 };
8111 let mut tracker = self.outcome_tracker.write().await;
8112 match result {
8113 Ok((results, response)) => {
8114 let usage = |key: &str| {
8115 response
8116 .pointer(&format!("/usage/{key}"))
8117 .and_then(|t| t.as_u64())
8118 .unwrap_or(0) as usize
8119 };
8120 tracker.record_complete(
8123 &trace,
8124 start.elapsed().as_millis() as u64,
8125 usage("input_tokens"),
8126 usage("output_tokens").max(1),
8127 );
8128 Ok(results)
8129 }
8130 Err(error) => {
8131 record_dispatch_failure(&mut tracker, &trace, &error);
8132 Err(error)
8133 }
8134 }
8135 }
8136
8137 async fn classify_via_generate(
8138 &self,
8139 req: ClassifyRequest,
8140 model: &str,
8141 ) -> Result<Vec<ClassifyResult>, InferenceError> {
8142 let labels_str = req
8143 .labels
8144 .iter()
8145 .enumerate()
8146 .map(|(i, l)| format!("{}. {}", i + 1, l))
8147 .collect::<Vec<_>>()
8148 .join("\n");
8149
8150 let prompt = format!(
8151 "Classify the following text into one of these categories:\n\
8152 {labels_str}\n\n\
8153 Text: {}\n\n\
8154 Respond with ONLY the category name, nothing else.",
8155 req.text
8156 );
8157
8158 let gen_req = GenerateRequest {
8159 prompt,
8160 model: Some(model.to_string()),
8161 params: tasks::generate::GenerateParams {
8162 temperature: 0.0,
8163 max_tokens: 32,
8164 thinking: tasks::generate::ThinkingMode::Off,
8167 ..Default::default()
8168 },
8169 context: None,
8170 context_stable_prefix: None,
8171 tools: None,
8172 images: None,
8173 messages: None,
8174 cache_control: false,
8175 response_format: None,
8176 intent: None,
8177 client_ref: None,
8178 expected_row_digest: None,
8179 expected_catalog_revision: None,
8180 caller: None,
8181 work_context: req.work_context.clone(),
8182 };
8183
8184 let response = self.generate(gen_req).await?;
8185 tasks::classify::score_reply(&response, &req.labels)
8186 }
8187
8188 pub async fn transcribe(
8190 &self,
8191 req: TranscribeRequest,
8192 ) -> Result<TranscribeResult, InferenceError> {
8193 let candidates =
8194 self.speech_candidates(ModelCapability::SpeechToText, req.model.as_deref())?;
8195 let mut last_error = None;
8196 let mut local_resource_block = None;
8197
8198 for schema in candidates {
8199 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8201 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8202 SpeechCandidateAdmission::SkipBlocked(error) => {
8203 local_resource_block = Some(error.to_string());
8204 last_error = Some(error);
8205 continue;
8206 }
8207 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8208 };
8209 let result = match &schema.source {
8210 ModelSource::Mlx { .. } => {
8211 self.transcribe_local_mlx(&schema, &req, reservation.as_mut())
8212 .await
8213 }
8214 ModelSource::WhisperCpp { model } => {
8215 self.transcribe_whisper(&schema, model, &req, reservation.as_mut())
8216 .await
8217 }
8218 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8219 self.transcribe_elevenlabs(&schema, &req).await
8220 }
8221 _ => Err(InferenceError::InferenceFailed(format!(
8222 "speech-to-text not implemented for model source: {}",
8223 schema.id
8224 ))),
8225 };
8226
8227 match result {
8228 Ok(mut result) => {
8229 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8230 if let Some(reason) = local_resource_block.take() {
8231 result.routing_explanation = Some(format!(
8232 "Local speech recognition was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8233 ));
8234 }
8235 }
8236 return Ok(result);
8237 }
8238 Err(err) if req.model.is_some() => return Err(err),
8239 Err(err) => last_error = Some(err),
8240 }
8241 }
8242
8243 Err(last_error.unwrap_or_else(|| {
8244 InferenceError::InferenceFailed("no speech-to-text models available".into())
8245 }))
8246 }
8247
8248 pub async fn synthesize(
8250 &self,
8251 req: SynthesizeRequest,
8252 ) -> Result<SynthesizeResult, InferenceError> {
8253 let candidates =
8254 self.speech_candidates(ModelCapability::TextToSpeech, req.model.as_deref())?;
8255 let mut last_error = None;
8256 let mut local_resource_block = None;
8257
8258 for schema in candidates {
8259 let mut reservation = match self.admit_speech_candidate(&schema, req.model.is_some()) {
8261 SpeechCandidateAdmission::Proceed(reservation) => reservation,
8262 SpeechCandidateAdmission::SkipBlocked(error) => {
8263 local_resource_block = Some(error.to_string());
8264 last_error = Some(error);
8265 continue;
8266 }
8267 SpeechCandidateAdmission::FailBlocked(error) => return Err(error),
8268 };
8269 let result = match &schema.source {
8270 ModelSource::Mlx { .. } => {
8271 self.synthesize_local_mlx(&schema, &req, reservation.as_mut())
8272 .await
8273 }
8274 ModelSource::WindowsSpeech {} => {
8275 self.synthesize_windows_speech(&schema, &req).await
8276 }
8277 ModelSource::Proprietary { provider, .. } if provider == "elevenlabs" => {
8278 self.synthesize_elevenlabs(&schema, &req).await
8279 }
8280 _ => Err(InferenceError::InferenceFailed(format!(
8281 "text-to-speech not implemented for model source: {}",
8282 schema.id
8283 ))),
8284 };
8285
8286 match result {
8287 Ok(mut result) => {
8288 if matches!(schema.source, ModelSource::Proprietary { .. }) {
8289 if let Some(reason) = local_resource_block.take() {
8290 result.routing_explanation = Some(format!(
8291 "Local speech synthesis was blocked ({reason}); CAR routed to a remote provider, which may affect privacy and cost."
8292 ));
8293 }
8294 }
8295 return Ok(result);
8296 }
8297 Err(err) if req.model.is_some() => return Err(err),
8298 Err(err) => last_error = Some(err),
8299 }
8300 }
8301
8302 Err(last_error.unwrap_or_else(|| {
8303 InferenceError::InferenceFailed("no text-to-speech models available".into())
8304 }))
8305 }
8306
8307 async fn synthesize_windows_speech(
8313 &self,
8314 schema: &ModelSchema,
8315 req: &SynthesizeRequest,
8316 ) -> Result<SynthesizeResult, InferenceError> {
8317 #[cfg(target_os = "windows")]
8318 {
8319 let text = req.text.clone();
8320 let voice = req.voice.clone().unwrap_or_default();
8321 let rate = req.speed.unwrap_or(1.0) as f64;
8322 let bytes =
8323 tokio::task::spawn_blocking(move || winrt_synthesize_wav(&text, &voice, rate))
8324 .await
8325 .map_err(|e| {
8326 InferenceError::InferenceFailed(format!("winrt tts join: {e}"))
8327 })??;
8328 let dest = requested_or_temp_output(req.output_path.as_deref(), "wav")?;
8329 ensure_parent_dir(&dest)?;
8330 std::fs::write(&dest, &bytes)?;
8331 Ok(SynthesizeResult {
8332 audio_path: dest.to_string_lossy().to_string(),
8333 media_type: "audio/wav".to_string(),
8334 model_used: Some(schema.name.clone()),
8335 voice_used: req.voice.clone(),
8336 routing_explanation: None,
8337 })
8338 }
8339 #[cfg(not(target_os = "windows"))]
8340 {
8341 let _ = (schema, req);
8342 Err(InferenceError::InferenceFailed(
8343 "Windows OS TTS is only available on Windows".into(),
8344 ))
8345 }
8346 }
8347
8348 pub async fn generate_image(
8350 &self,
8351 req: GenerateImageRequest,
8352 ) -> Result<GenerateImageResult, InferenceError> {
8353 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8370 {
8371 use crate::backend::external_flux;
8372 tracing::info!(
8373 model = req.model.as_deref().unwrap_or("<default>"),
8374 "routing image generation to mflux"
8375 );
8376 let schema = self
8377 .media_generation_candidates(
8378 ModelCapability::ImageGeneration,
8379 req.model.as_deref(),
8380 )?
8381 .into_iter()
8382 .next()
8383 .ok_or_else(|| {
8384 InferenceError::InferenceFailed("no image generation models available".into())
8385 })?;
8386 let mut reservation = self.reserve_local_request(&schema, 0)?;
8388 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8389 reservation
8390 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8391 .map_err(InferenceError::from)?;
8392 let mut req = req;
8393 req.model = self
8394 .resolve_external_hf_repo(req.model.as_deref(), ModelCapability::ImageGeneration);
8395 external_flux::generate_image(&req)
8396 }
8397
8398 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
8403 {
8404 let _ = &req;
8405 Err(InferenceError::InferenceFailed(
8406 "local image generation requires Apple Silicon (mflux)".into(),
8407 ))
8408 }
8409 }
8410
8411 pub async fn generate_image_batch(
8427 &self,
8428 req: GenerateImageRequest,
8429 ) -> Result<Vec<GenerateImageResult>, InferenceError> {
8430 let count = req.variant_count.unwrap_or(1).max(1);
8431 if count == 1 {
8432 return self.generate_image(req).await.map(|r| vec![r]);
8433 }
8434 let base_seed = req.seed.unwrap_or(0);
8435 let mut results = Vec::with_capacity(count as usize);
8436 for i in 0..count {
8437 let mut variant_req = req.clone();
8442 variant_req.seed = Some(base_seed.wrapping_add(i as u64));
8443 variant_req.variant_count = Some(1);
8447 results.push(self.generate_image(variant_req).await?);
8448 }
8449 Ok(results)
8450 }
8451
8452 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8464 fn mlx_device_lock() -> Arc<tokio::sync::Mutex<()>> {
8465 static MLX_DEVICE_LOCK: std::sync::OnceLock<Arc<tokio::sync::Mutex<()>>> =
8466 std::sync::OnceLock::new();
8467 MLX_DEVICE_LOCK
8468 .get_or_init(|| Arc::new(tokio::sync::Mutex::new(())))
8469 .clone()
8470 }
8471
8472 pub async fn generate_video(
8474 &self,
8475 req: GenerateVideoRequest,
8476 ) -> Result<GenerateVideoResult, InferenceError> {
8477 if let Err(msg) = req.validate() {
8480 return Err(InferenceError::InferenceFailed(format!(
8481 "invalid GenerateVideoRequest: {}",
8482 msg
8483 )));
8484 }
8485 #[allow(unused_variables)]
8487 let requires_audio_conditioning = req.requires_audio_passthrough_opt_in();
8488 let candidates = self
8491 .media_generation_candidates(ModelCapability::VideoGeneration, req.model.as_deref())?;
8492 let mut last_error = None;
8493
8494 for schema in candidates {
8495 let mut reservation = self.reserve_local_request(&schema, 0)?;
8497 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
8498 reservation
8499 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
8500 .map_err(InferenceError::from)?;
8501 let result = match &schema.source {
8502 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
8503 ModelSource::Mlx { hf_repo, .. } => {
8504 if !crate::backend::external_ltx::is_available() {
8518 return Err(InferenceError::InferenceFailed(
8519 "video generation requires the external `ltx-2-mlx` CLI on PATH"
8520 .to_string(),
8521 ));
8522 }
8523 let mut req = req.clone();
8524 req.model = Some(hf_repo.clone());
8525 crate::backend::external_ltx::generate_video(&req)
8526 }
8527 _ => Err(InferenceError::InferenceFailed(format!(
8528 "video generation not implemented for model source: {}",
8529 schema.id
8530 ))),
8531 };
8532
8533 match result {
8534 Ok(result) => return Ok(result),
8535 Err(err) if req.model.is_some() => return Err(err),
8536 Err(err) => last_error = Some(err),
8537 }
8538 }
8539
8540 Err(last_error.unwrap_or_else(|| {
8541 InferenceError::InferenceFailed("no video generation models available".into())
8542 }))
8543 }
8544
8545 pub fn response_format_rejection_reason(
8553 &self,
8554 model: &str,
8555 rf: &crate::tasks::generate::ResponseFormat,
8556 ) -> Option<String> {
8557 let registry = (*self.unified_registry()).clone();
8558 let schema = registry
8559 .list()
8560 .into_iter()
8561 .find(|s| s.id == model || s.name == model)?;
8562 match &schema.source {
8563 ModelSource::Proprietary { .. } => {
8564 Some("the Parslee gateway does not accept response_format".to_string())
8565 }
8566 ModelSource::RemoteApi { protocol, .. } => {
8567 let handler = crate::protocol::handler_for(*protocol);
8568 (!handler.supports_response_format(rf)).then(|| {
8569 format!(
8570 "the {} protocol rejects response_format",
8571 handler.protocol_name()
8572 )
8573 })
8574 }
8575 _ => None,
8576 }
8577 }
8578
8579 pub fn list_models_unified(&self) -> Vec<ModelInfo> {
8584 let hardware = HardwareInfo::detect();
8585 let policy = self.active_local_resource_policy();
8586 self.list_models_unified_for(&hardware, &policy.policy)
8587 }
8588
8589 pub fn model_fit(&self, schema: &ModelSchema) -> recommend::ModelFit {
8593 recommend::model_fit(
8594 schema,
8595 &HardwareInfo::detect(),
8596 Some(&self.active_local_resource_policy().policy),
8597 )
8598 }
8599
8600 pub fn list_models_unified_for(
8604 &self,
8605 hardware: &HardwareInfo,
8606 policy: &resource_policy::ResourcePolicy,
8607 ) -> Vec<ModelInfo> {
8608 self.list_models_unified_for_inner(hardware, policy, false)
8609 }
8610
8611 pub fn list_models_unified_for_explicit_status(&self) -> Vec<ModelInfo> {
8632 let hardware = HardwareInfo::detect();
8633 let policy = self.active_local_resource_policy();
8634 self.list_models_unified_for_inner(&hardware, &policy.policy, true)
8635 }
8636
8637 fn list_models_unified_for_inner(
8638 &self,
8639 hardware: &HardwareInfo,
8640 policy: &resource_policy::ResourcePolicy,
8641 authoritative_credentials: bool,
8642 ) -> Vec<ModelInfo> {
8643 let mut registry = (*self.unified_registry()).clone();
8644 registry.prune_missing_on_disk_models();
8645 if authoritative_credentials {
8646 registry.refresh_availability_for_explicit_status();
8647 } else {
8648 registry.refresh_availability();
8649 }
8650 registry
8651 .list()
8652 .iter()
8653 .map(|schema| {
8654 let mut info = ModelInfo::from(*schema).with_fit(recommend::model_fit(
8655 schema,
8656 hardware,
8657 Some(policy),
8658 ));
8659 if !authoritative_credentials {
8660 info.credential_required = None;
8677 } else if let Some(env) = info.credential_required.as_deref() {
8678 if registry.credential_state_is_unknown(env) {
8689 info.credential_required = None;
8690 }
8691 }
8692 if schema.downloads_weights() {
8693 let enabled = self
8694 .model_management
8695 .car_enabled(&schema.id)
8696 .unwrap_or(false);
8697 if !enabled {
8698 info.available = false;
8702 }
8703 if enabled
8704 && self
8705 .model_management
8706 .load_receipt(&schema.id)
8707 .is_ok_and(|receipt| receipt.is_none())
8708 {
8709 if let Ok(_mutation) = self.model_management.begin_mutation(&schema.id) {
8716 if let Some(path) = registry.existing_local_artifact(&schema.id) {
8717 if std::fs::symlink_metadata(&path)
8718 .is_ok_and(|metadata| metadata.file_type().is_symlink())
8719 {
8720 let generation = self
8721 .resource_policy_generation
8722 .load(std::sync::atomic::Ordering::Acquire);
8723 let _ = self.model_management.record_managed_artifact(
8724 &schema.id,
8725 model_source_identity(schema),
8726 None,
8727 generation,
8728 true,
8729 path,
8730 );
8731 }
8732 }
8733 }
8734 }
8735 let receipt = self.model_management.load_receipt(&schema.id);
8736 let directory_cleanup_unsupported =
8737 !model_management::directory_removal_supported()
8738 && receipt.as_ref().is_ok_and(|receipt| {
8739 receipt.as_ref().is_some_and(|receipt| {
8740 receipt.artifact_kind
8741 == model_management::ManagedArtifactKind::Directory
8742 })
8743 });
8744 info.car_enabled = enabled;
8745 info.can_remove = enabled
8746 && self
8747 .model_management
8748 .can_remove(&schema.id)
8749 .unwrap_or(false);
8750 info.in_use = self.local_model_in_use(&schema.id);
8751 info.management_evidence = if !enabled {
8752 Some("disabled_tombstone".into())
8753 } else if directory_cleanup_unsupported {
8754 Some("install_receipt_directory_cleanup_unsupported".into())
8755 } else if receipt.is_ok_and(|receipt| receipt.is_some()) {
8756 Some("install_receipt".into())
8757 } else if info.weights_ready {
8758 Some("shared_or_hand_installed".into())
8759 } else {
8760 None
8761 };
8762 }
8763 info
8764 })
8765 .collect()
8766 }
8767
8768 fn local_model_in_use(&self, model_id: &str) -> bool {
8772 self.local_admission.active_request_count(model_id) > 0
8773 || self.local_admission.is_resident(model_id)
8774 || self.local_admission.teardown_pending(model_id)
8775 || self.model_management.model_in_use(model_id).unwrap_or(true)
8776 }
8777
8778 pub fn retire_plan(&self, model_id: &str) -> Result<retire::RetirePlan, InferenceError> {
8782 self.retire_plan_in(model_id, &hf_cache::hub_dir())
8783 }
8784
8785 pub fn retire_plan_in(
8787 &self,
8788 model_id: &str,
8789 hub: &Path,
8790 ) -> Result<retire::RetirePlan, InferenceError> {
8791 self.retire_plan_with(model_id, hub, None)
8792 .map(|(plan, _)| plan)
8793 }
8794
8795 fn retire_plan_with(
8800 &self,
8801 model_id: &str,
8802 hub: &Path,
8803 held: Option<&std::collections::BTreeSet<String>>,
8804 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
8805 if self.unified_registry().get(model_id).is_none() {
8806 return Err(InferenceError::ModelNotFound(model_id.to_string()));
8807 }
8808 let (refreshed, owned) = self.retire_snapshot(hub);
8809 self.retire_plan_on(&refreshed, &owned, model_id, hub, held)
8810 }
8811
8812 fn retire_snapshot(&self, hub: &Path) -> (UnifiedRegistry, retire::Ownership) {
8817 let mut refreshed = (*self.unified_registry()).clone();
8818 refreshed.refresh_availability();
8819 let owned = retire::ownership(
8820 retire::catalog_ownership(),
8821 retire::recorded_files(&self.config.state_root, hub),
8822 );
8823 (refreshed, owned)
8824 }
8825
8826 fn retire_plan_on(
8828 &self,
8829 refreshed: &UnifiedRegistry,
8830 owned: &retire::Ownership,
8831 model_id: &str,
8832 hub: &Path,
8833 held: Option<&std::collections::BTreeSet<String>>,
8834 ) -> Result<(retire::RetirePlan, std::collections::BTreeSet<String>), InferenceError> {
8835 let schema = refreshed
8836 .get(model_id)
8837 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
8838 let registry: Vec<&ModelSchema> = refreshed.all().collect();
8839 let installed = |m: &ModelSchema| {
8840 m.downloads_weights()
8841 && m.weights_ready
8842 && self.model_management.car_enabled(&m.id).unwrap_or(true)
8843 };
8844 let primary = retire::primary_whole_repo(schema);
8850 let expands = |repo: &str| {
8854 hf_cache::checked_repo_dir_in(hub, repo).is_some()
8855 && owned.get(repo) == Some(&retire::Owned::Whole)
8856 };
8857 let mut targets: Vec<&ModelSchema> = vec![schema];
8858 targets.extend(registry.iter().copied().filter(|m| {
8859 m.id != schema.id
8860 && primary.is_some_and(|repo| {
8861 retire::primary_whole_repo(m) == Some(repo) && expands(repo)
8862 })
8863 }));
8864 let mut plan = retire::plan_hub_retirement(&retire::RetireInputs {
8865 targets: &targets,
8866 registry: ®istry,
8867 installed: &installed,
8868 owned,
8869 hub,
8870 });
8871 if plan.refusals.iter().any(|r| {
8872 matches!(
8873 r,
8874 retire::RetireRefusal::NothingToRetire | retire::RetireRefusal::Unsupported { .. }
8875 )
8876 }) {
8877 return Ok((plan, std::collections::BTreeSet::new()));
8878 }
8879
8880 let models_dir = self
8884 .config
8885 .models_dir
8886 .canonicalize()
8887 .unwrap_or_else(|_| self.config.models_dir.clone());
8888 for target in &targets {
8889 let mut projections = Vec::new();
8890 if retire::single_component(&target.name) {
8891 projections.push(models_dir.join(&target.name));
8892 } else {
8893 plan.refusals.push(retire::RetireRefusal::UnsafePath {
8894 path: self.config.models_dir.join(&target.name),
8895 why: "the model's name is not a single path component".into(),
8896 });
8897 }
8898 if let Ok(adopted) = self.model_management.adopted_projection_path(&target.id) {
8899 projections.push(adopted);
8900 }
8901 let receipt_path = self
8906 .model_management
8907 .load_receipt(&target.id)
8908 .ok()
8909 .flatten()
8910 .map(|r| r.managed_path);
8911 for path in projections {
8912 if std::fs::symlink_metadata(&path).is_err()
8913 || plan.projections.iter().any(|p| p.path == path)
8914 || plan.user_content.contains(&path)
8915 {
8916 continue;
8917 }
8918 let receipted = receipt_path
8923 .as_ref()
8924 .is_some_and(|managed| retire::same_entry(managed, &path));
8925 if receipted || retire::links_only(&path) {
8926 plan.freed_bytes += retire::tree_bytes(&path);
8927 plan.projections.push(retire::Projection {
8928 path,
8929 receipt_for: receipted.then(|| target.id.clone()),
8930 });
8931 } else {
8932 plan.user_content.push(path);
8933 }
8934 }
8935 }
8936
8937 if targets.iter().any(|t| self.local_model_in_use(&t.id)) {
8938 plan.refusals.push(retire::RetireRefusal::InUse);
8939 }
8940 let repos: std::collections::HashSet<String> = targets
8944 .iter()
8945 .flat_map(|t| retire::hf_references(t))
8946 .map(|r| r.repo)
8947 .collect();
8948 let lock_keys: std::collections::BTreeSet<String> = registry
8949 .iter()
8950 .filter(|m| {
8951 targets.iter().any(|t| t.id == m.id)
8952 || retire::hf_references(m)
8953 .iter()
8954 .any(|r| repos.contains(&r.repo))
8955 })
8956 .flat_map(|m| [m.id.clone(), m.name.clone()])
8957 .collect();
8958 match held {
8959 None => {
8960 if lock_keys
8961 .iter()
8962 .any(|key| crate::download::model_lock_held(key))
8963 {
8964 plan.refusals.push(retire::RetireRefusal::Downloading);
8965 }
8966 }
8967 Some(held) => {
8968 if !lock_keys.is_subset(held) {
8969 plan.refusals.push(retire::RetireRefusal::StateChanged);
8970 }
8971 }
8972 }
8973 for default in self.lane_defaults().defaults {
8974 if targets.iter().any(|t| t.id == default.model_id) {
8975 let use_case = serde_json::to_value(default.use_case)
8976 .ok()
8977 .and_then(|v| v.as_str().map(str::to_string))
8978 .unwrap_or_default();
8979 plan.refusals
8980 .push(retire::RetireRefusal::LaneDefault { use_case });
8981 }
8982 }
8983 let protected = self.protected_models(refreshed, false);
8987 for target in &targets {
8988 if let Some(why) = protected.get(&target.id) {
8989 plan.refusals
8990 .push(retire::RetireRefusal::Protected { why: why.clone() });
8991 }
8992 }
8993 if plan.deletions.is_empty() && plan.projections.is_empty() && plan.refusals.is_empty() {
8995 plan.refusals.push(retire::RetireRefusal::NothingToRetire);
8996 }
8997 plan.seal();
8998 Ok((plan, lock_keys))
8999 }
9000
9001 pub async fn retire_model(
9007 &self,
9008 model_id: &str,
9009 dry_run: bool,
9010 expect: Option<&str>,
9011 ) -> Result<retire::RetireOutcome, InferenceError> {
9012 self.retire_model_in(model_id, &hf_cache::hub_dir(), dry_run, expect)
9013 .await
9014 }
9015
9016 pub async fn retire_model_in(
9032 &self,
9033 model_id: &str,
9034 hub: &Path,
9035 dry_run: bool,
9036 expect: Option<&str>,
9037 ) -> Result<retire::RetireOutcome, InferenceError> {
9038 let not_executed = |plan| retire::RetireOutcome {
9039 plan,
9040 executed: false,
9041 removed: Vec::new(),
9042 errors: Vec::new(),
9043 };
9044 let (first, keys) = self.retire_plan_with(model_id, hub, None)?;
9045 if dry_run || first.is_refused() {
9046 return Ok(not_executed(first));
9047 }
9048 let _concierge = self.concierge_action_lock.lock().await;
9052 let mut locks = Vec::with_capacity(keys.len());
9053 for key in &keys {
9054 match crate::download::try_acquire_model_lock(key) {
9055 Some(guard) => locks.push(guard),
9056 None => {
9057 let mut refused = first;
9058 refused.refusals.push(retire::RetireRefusal::Downloading);
9059 return Ok(not_executed(refused));
9060 }
9061 }
9062 }
9063 let (mut plan, _) = self.retire_plan_with(model_id, hub, Some(&keys))?;
9064 if expect.is_some_and(|digest| digest != plan.digest) {
9065 plan.refusals.push(retire::RetireRefusal::StateChanged);
9066 }
9067 if plan.is_refused() {
9068 return Ok(not_executed(plan));
9069 }
9070 let targets: Vec<String> = std::iter::once(plan.model_id.clone())
9071 .chain(plan.also_retires.iter().cloned())
9072 .collect();
9073 let mut maintenance = Vec::with_capacity(targets.len());
9074 for id in &targets {
9075 match self.prepare_local_model_removal(id).await {
9076 Ok(guard) => maintenance.push(guard),
9077 Err(error) => {
9078 tracing::info!(model = %id, %error, "retirement refused: the model could not be released");
9079 plan.refusals.push(retire::RetireRefusal::InUse);
9080 return Ok(not_executed(plan));
9081 }
9082 }
9083 }
9084
9085 let started_at = std::time::SystemTime::now()
9088 .duration_since(std::time::UNIX_EPOCH)
9089 .map(|d| d.as_secs())
9090 .unwrap_or(0);
9091 for id in &targets {
9092 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9093 seq: 0,
9094 kind: action_ledger::ConciergeActionKind::RetireStarted,
9095 model_id: id.clone(),
9096 use_case: None,
9097 project: None,
9098 prior_model_id: None,
9099 detail: format!(
9100 "retiring with {}; plan {}; {} bytes planned",
9101 targets.join(", "),
9102 plan.digest,
9103 plan.freed_bytes
9104 ),
9105 timestamp: started_at,
9106 });
9107 }
9108
9109 let mut removed = Vec::new();
9110 let mut errors = Vec::new();
9111
9112 for projection in &plan.projections {
9115 let result = match &projection.receipt_for {
9116 Some(id) => {
9117 let result = async {
9120 self.model_management
9121 .note_retiring(id)
9122 .map_err(|e| e.to_string())?;
9123 self.remove_model_from_car_inner(id, true)
9124 .await
9125 .map_err(|e| e.to_string())?;
9126 self.model_management
9127 .clear_tombstone(id)
9128 .map_err(|e| e.to_string())?;
9129 self.model_management.retired(id);
9130 Ok::<_, String>(vec![projection.path.clone()])
9131 }
9132 .await;
9133 result
9134 }
9135 None => retire::remove_links_only_tree(&projection.path, &self.config.models_dir),
9136 };
9137 match result {
9138 Ok(paths) => removed.extend(paths),
9139 Err(why) => {
9140 errors.push(format!("{}: {why}", projection.path.display()));
9141 break;
9142 }
9143 }
9144 }
9145
9146 let hub_canonical = hub.canonicalize().unwrap_or_else(|_| hub.to_path_buf());
9147 if errors.is_empty() {
9148 'hub: for deletion in &plan.deletions {
9149 match deletion {
9150 retire::HubDeletion::Repo { path, .. } => {
9151 let quarantine = path.with_file_name(format!(
9152 "{}{:016x}",
9153 model_management::RETIRE_QUARANTINE_PREFIX,
9154 rand::random::<u64>()
9155 ));
9156 let verify = || retire::verify_repo_dir(path, hub);
9157 match self.model_management.detach_and_remove_tree(
9158 path,
9159 &quarantine,
9160 model_id,
9161 verify,
9162 ) {
9163 Ok(()) => removed.push(path.clone()),
9164 Err(failure) => {
9165 if failure.detached {
9166 removed.push(path.clone());
9169 }
9170 errors.push(format!("{}: {}", path.display(), failure.error));
9171 break 'hub;
9172 }
9173 }
9174 }
9175 retire::HubDeletion::Files {
9176 repo,
9177 entries,
9178 blobs,
9179 ..
9180 } => {
9181 let Some(dir) = hf_cache::checked_repo_dir_in(&hub_canonical, repo) else {
9182 errors.push(format!("{repo}: not a Hub repo id"));
9183 break 'hub;
9184 };
9185 for blob in blobs {
9189 let checked =
9190 retire::verify_blob(blob, &dir.join("blobs")).and_then(|()| {
9191 model_management::unlink_checked(
9192 blob,
9193 model_management::UnlinkKind::File,
9194 )
9195 .map_err(|e| e.to_string())
9196 });
9197 match checked {
9198 Ok(()) => removed.push(blob.clone()),
9199 Err(why) => {
9200 errors.push(format!("{}: {why}", blob.display()));
9201 break 'hub;
9202 }
9203 }
9204 }
9205 let snapshots = dir.join("snapshots");
9206 for entry in entries {
9207 let checked = retire::verify_entry(entry, &snapshots).and_then(|()| {
9208 model_management::unlink_checked(
9209 entry,
9210 model_management::UnlinkKind::FileOrLink,
9211 )
9212 .map_err(|e| e.to_string())
9213 });
9214 match checked {
9215 Ok(()) => removed.push(entry.clone()),
9216 Err(why) => {
9217 errors.push(format!("{}: {why}", entry.display()));
9218 break 'hub;
9219 }
9220 }
9221 }
9222 for entry in entries {
9225 let mut at = entry.parent();
9226 while let Some(parent) = at {
9227 if !parent.starts_with(&snapshots) || parent == snapshots {
9228 break;
9229 }
9230 if std::fs::remove_dir(parent).is_err() {
9231 break;
9232 }
9233 at = parent.parent();
9234 }
9235 }
9236 retire::prune_empty_repo(&dir);
9237 }
9238 }
9239 }
9240 }
9241 drop(maintenance);
9242 drop(locks);
9243
9244 if !removed.is_empty() {
9245 let now = std::time::SystemTime::now()
9246 .duration_since(std::time::UNIX_EPOCH)
9247 .map(|d| d.as_secs())
9248 .unwrap_or(0);
9249 for id in &targets {
9250 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9251 seq: 0,
9252 kind: action_ledger::ConciergeActionKind::Retire,
9253 model_id: id.clone(),
9254 use_case: None,
9255 project: None,
9256 prior_model_id: None,
9257 detail: format!(
9258 "retired with {}; plan {}; {} bytes planned; {} paths removed{}",
9259 targets.join(", "),
9260 plan.digest,
9261 plan.freed_bytes,
9262 removed.len(),
9263 if errors.is_empty() {
9264 String::new()
9265 } else {
9266 format!("; stopped: {}", errors.join("; "))
9267 }
9268 ),
9269 timestamp: now,
9270 });
9271 }
9272 }
9273 Ok(retire::RetireOutcome {
9274 executed: !removed.is_empty(),
9275 plan,
9276 removed,
9277 errors,
9278 })
9279 }
9280
9281 pub fn set_external_pins(&self, pins: std::collections::BTreeMap<String, Vec<String>>) {
9289 *self
9290 .external_pins
9291 .write()
9292 .unwrap_or_else(std::sync::PoisonError::into_inner) = pins;
9293 }
9294
9295 pub fn start_usage_tracking(&self) {
9299 self.model_management.start_usage_tracking();
9300 self.model_management.beat();
9301 }
9302
9303 fn recent_concierge_actions(&self, now: u64) -> Vec<portfolio::RecentAction> {
9307 use action_ledger::ConciergeActionKind as Kind;
9308 let since = now.saturating_sub(30 * 24 * 60 * 60);
9309 let actions = crate::action_ledger::read_actions(&self.action_ledger_path(), 0);
9310 let mut out: Vec<portfolio::RecentAction> = Vec::new();
9311 for (i, a) in actions.iter().enumerate() {
9312 if a.timestamp < since {
9313 continue;
9314 }
9315 let (kind, undo, interrupted) = match a.kind {
9316 Kind::Retire => (
9317 "retire",
9318 (!a.model_id.starts_with("repo:"))
9319 .then(|| format!("car models pull {}", a.model_id)),
9320 false,
9321 ),
9322 Kind::RetireStarted => {
9323 let finished = actions[i + 1..]
9324 .iter()
9325 .any(|later| later.kind == Kind::Retire && later.model_id == a.model_id);
9326 if finished {
9327 continue;
9328 }
9329 ("retire", None, true)
9330 }
9331 Kind::DiscardPartials => ("discard_partials", None, false),
9332 Kind::ResourcePolicy => (
9333 "resource_policy",
9334 a.prior_model_id
9335 .as_ref()
9336 .map(|prior| format!("car models resource-policy set {prior}")),
9337 false,
9338 ),
9339 Kind::AutoUpgrade => (
9340 "upgrade",
9341 match (a.use_case, &a.prior_model_id) {
9342 (Some(use_case), _) => serde_json::to_value(use_case)
9343 .ok()
9344 .and_then(|v| v.as_str().map(str::to_owned))
9345 .map(|lane| format!("car concierge rollback {lane}")),
9346 (None, Some(prior)) => {
9347 Some(format!("keep using {prior}; nothing was switched"))
9348 }
9349 (None, None) => None,
9350 },
9351 false,
9352 ),
9353 Kind::RevokedLaneCleared => ("lane_cleared", None, false),
9356 Kind::Install | Kind::SetDefault | Kind::ClearDefault | Kind::Rollback => continue,
9357 };
9358 out.push(portfolio::RecentAction {
9359 at: a.timestamp,
9360 kind: kind.to_string(),
9361 subject: a.model_id.clone(),
9362 detail: a.detail.clone(),
9363 undo,
9364 use_case: a.use_case.and_then(|use_case| {
9365 serde_json::to_value(use_case)
9366 .ok()
9367 .and_then(|v| v.as_str().map(str::to_owned))
9368 }),
9369 prior: a.prior_model_id.clone(),
9370 interrupted,
9371 });
9372 }
9373 out.reverse();
9374 out
9375 }
9376
9377 pub async fn set_resource_policy_by_user(
9382 &self,
9383 policy: resource_policy::ResourcePolicy,
9384 ) -> Result<(), resource_policy::ResourcePolicyError> {
9385 let _concierge = self.concierge_action_lock.lock().await;
9386 let repository =
9387 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9388 repository.clear_concierge_marker()?;
9389 resource_policy::ResourcePolicyRepository::save(&repository, &policy)?;
9390 self.apply_local_resource_policy(policy);
9391 Ok(())
9392 }
9393
9394 async fn apply_one_curated_upgrade(
9403 &self,
9404 hub: &Path,
9405 nudge: &crate::nudge::NudgeState,
9406 now: u64,
9407 ) -> Result<Option<maintenance::Upgraded>, String> {
9408 let findings = self.detect_upgrades().await;
9409 self.apply_one_curated_upgrade_from(findings, hub, nudge, now)
9410 .await
9411 }
9412
9413 async fn apply_one_curated_upgrade_from(
9415 &self,
9416 findings: Vec<crate::upgrade::UpgradeFinding>,
9417 hub: &Path,
9418 nudge: &crate::nudge::NudgeState,
9419 now: u64,
9420 ) -> Result<Option<maintenance::Upgraded>, String> {
9421 use crate::upgrade::UpgradeSource;
9422 for finding in findings {
9423 let label = match finding.source {
9424 UpgradeSource::Benchmark => "benchmark upgrade",
9425 UpgradeSource::Curated | UpgradeSource::Upstream => "curated upgrade",
9426 };
9427 let key = format!("{}=>{}", finding.from_id, finding.to_id);
9428 let backing_off = self
9429 .failed_upgrades
9430 .lock()
9431 .unwrap_or_else(std::sync::PoisonError::into_inner)
9432 .get(&key)
9433 .is_some_and(|at| now.saturating_sub(*at) < 24 * 60 * 60);
9434 if finding.source == UpgradeSource::Upstream
9435 || finding.trust_tier != crate::schema::TrustTier::Curated
9436 || !finding.target_pullable
9437 || nudge.dismissed.contains(&key)
9438 || backing_off
9439 {
9440 continue;
9441 }
9442 let Some(target) = self.unified_registry().get(&finding.to_id).cloned() else {
9443 continue;
9444 };
9445 let fit = self.model_fit(&target);
9446 if !fit.platform_compatible || fit.fit == recommend::ModelFitStatus::TooBig {
9447 continue;
9448 }
9449 let lanes: Vec<crate::lane_defaults::LaneDefault> = self
9450 .lane_defaults()
9451 .defaults
9452 .into_iter()
9453 .filter(|d| d.model_id == finding.from_id)
9454 .collect();
9455 if lanes.is_empty() {
9456 continue;
9457 }
9458 let present = self
9460 .unified_registry()
9461 .existing_local_artifact(&finding.to_id)
9462 .is_some();
9463 if !present && crate::download::check_disk_space(hub, target.size_mb()).is_err() {
9464 continue;
9465 }
9466 let mut switched = Vec::new();
9467 for lane in &lanes {
9468 if let Err(e) = self
9469 .apply_concierge(lane.use_case, &finding.to_id, lane.project.clone())
9470 .await
9471 {
9472 self.failed_upgrades
9473 .lock()
9474 .unwrap_or_else(std::sync::PoisonError::into_inner)
9475 .insert(key, now);
9476 if let Some(first) = lanes.first().filter(|_| !switched.is_empty()) {
9478 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9479 seq: 0,
9480 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
9481 model_id: finding.to_id.clone(),
9482 use_case: Some(first.use_case),
9483 project: first.project.clone(),
9484 prior_model_id: Some(finding.from_id.clone()),
9485 detail: format!(
9486 "{label}, partial: switched {}; then {e}",
9487 switched.join(", ")
9488 ),
9489 timestamp: now,
9490 });
9491 }
9492 return Err(format!(
9493 "upgrading {} to {} (fetch, then switch the lane): {e}; retried in a day",
9494 finding.from_id, finding.to_id
9495 ));
9496 }
9497 switched.push(
9498 serde_json::to_value(lane.use_case)
9499 .ok()
9500 .and_then(|v| v.as_str().map(str::to_owned))
9501 .unwrap_or_default(),
9502 );
9503 }
9504 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9505 seq: 0,
9506 kind: action_ledger::ConciergeActionKind::AutoUpgrade,
9507 model_id: finding.to_id.clone(),
9508 use_case: lanes.first().map(|l| l.use_case),
9509 project: lanes.first().and_then(|l| l.project.clone()),
9510 prior_model_id: Some(finding.from_id.clone()),
9511 detail: format!("{label}: {}", finding.reason),
9512 timestamp: now,
9513 });
9514 return Ok(Some(maintenance::Upgraded {
9515 from: finding.from_id,
9516 to: finding.to_id,
9517 lanes: switched,
9518 }));
9519 }
9520 Ok(None)
9521 }
9522
9523 async fn size_memory_policy(
9532 &self,
9533 hw: &HardwareInfo,
9534 observable_since: Option<u64>,
9535 apply: bool,
9536 now: u64,
9537 ) -> Option<maintenance::PolicySizing> {
9538 use resource_policy::{ResourcePolicy, ResourcePolicyLoadSource};
9539 let since = observable_since?;
9540 let _concierge = self.concierge_action_lock.lock().await;
9541 let repository =
9542 resource_policy::FileResourcePolicyRepository::new(self.config.state_root.clone());
9543 let loaded = repository.load_with_evidence().ok()?;
9544 let concierge = repository.concierge_choice();
9545 let user_chose = match loaded.source {
9546 ResourcePolicyLoadSource::MissingDefault => false,
9547 ResourcePolicyLoadSource::CorruptDefault => true,
9548 ResourcePolicyLoadSource::Loaded => {
9549 concierge.as_ref().map(|c| &c.policy) != Some(&loaded.policy)
9550 }
9551 };
9552 let everyday = ResourcePolicy::everyday();
9553 let local_focused = ResourcePolicy::local_focused();
9554 let fit = |m: &ModelSchema, policy: &ResourcePolicy| {
9555 recommend::model_fit(m, hw, Some(policy)).fit
9556 };
9557 let raised_at = concierge
9558 .as_ref()
9559 .filter(|c| !c.reverted && c.policy == local_focused)
9560 .map(|c| c.at);
9561 let mut evidence = maintenance::PolicyEvidence::default();
9562 let mut days = std::collections::BTreeSet::new();
9563 for e in crate::outcome::read_ledger(
9564 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9565 0,
9566 ) {
9567 if e.timestamp < since
9568 || e.attribution != Some(crate::outcome::UnattributedReason::ResourceBlocked)
9569 {
9570 continue;
9571 }
9572 let registry = self.unified_registry();
9573 let Some(m) = registry.get(&e.model_id).filter(|m| m.is_local()) else {
9574 continue;
9575 };
9576 if fit(m, &loaded.policy) == recommend::ModelFitStatus::TooBig
9581 && fit(m, &local_focused) == recommend::ModelFitStatus::Fits
9582 {
9583 evidence.ceiling_refusals += 1;
9584 days.insert(e.timestamp / (24 * 60 * 60));
9585 }
9586 if raised_at.is_some_and(|at| e.timestamp >= at)
9587 && fit(m, &everyday) == recommend::ModelFitStatus::Fits
9588 {
9589 evidence.live_refusals_since_raise += 1;
9590 }
9591 }
9592 if let Some(at) = raised_at {
9594 for e in crate::outcome::read_ledger(
9595 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9596 0,
9597 ) {
9598 let admitted_by_raise = self.unified_registry().get(&e.model_id).is_some_and(|m| {
9599 m.is_local() && fit(m, &everyday) == recommend::ModelFitStatus::TooBig
9600 });
9601 if e.timestamp < at || !admitted_by_raise {
9602 continue;
9603 }
9604 match (e.success, e.attribution) {
9605 (Some(true), _) => evidence.admitted_successes_since_raise += 1,
9606 (Some(false), None) => evidence.admitted_failures_since_raise += 1,
9607 _ => {}
9608 }
9609 }
9610 }
9611 evidence.ceiling_refusal_days = days.len() as u64;
9612 let decision = maintenance::decide_resource_policy(
9613 &loaded.policy,
9614 user_chose,
9615 concierge.as_ref(),
9616 hw.total_ram_mb,
9617 &evidence,
9618 )?;
9619 let (target, reason, reverted) = match decision {
9620 maintenance::PolicyMove::Raise(reason) => (local_focused, reason, false),
9621 maintenance::PolicyMove::Revert(reason) => (everyday, reason, true),
9622 };
9623 let mut sizing = maintenance::PolicySizing {
9624 from: loaded.policy.profile.clone(),
9625 to: target.profile.clone(),
9626 reason,
9627 applied: false,
9628 };
9629 if apply {
9630 match repository.save_concierge_choice(&target, &sizing.reason, now, reverted) {
9631 Ok(()) => {
9632 self.apply_local_resource_policy(target);
9633 sizing.applied = true;
9634 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9635 seq: 0,
9636 kind: action_ledger::ConciergeActionKind::ResourcePolicy,
9637 model_id: String::new(),
9638 use_case: None,
9639 project: None,
9640 prior_model_id: serde_json::to_value(&sizing.from)
9641 .ok()
9642 .and_then(|v| v.as_str().map(str::to_owned)),
9643 detail: format!("{:?} -> {:?}: {}", sizing.from, sizing.to, sizing.reason),
9644 timestamp: now,
9645 });
9646 }
9647 Err(error) => tracing::warn!(%error, "could not change the memory policy"),
9648 }
9649 }
9650 Some(sizing)
9651 }
9652
9653 async fn retire_orphan_repo(&self, repo: &str, hub: &Path) -> Result<u64, String> {
9660 let _concierge = self.concierge_action_lock.lock().await;
9661 let (refreshed, owned) = self.retire_snapshot(hub);
9662 let referenced: std::collections::HashSet<String> = refreshed
9663 .all()
9664 .flat_map(retire::hf_references)
9665 .map(|r| r.repo)
9666 .collect();
9667 let orphan = retire::unreferenced_repos(hub, &referenced, &owned)
9668 .into_iter()
9669 .find(|o| o.repo == repo && o.owner == retire::OrphanOwner::Car)
9670 .ok_or("no longer an orphan CAR owns")?;
9671 if retire::transfer_in_progress(&orphan.path)? {
9672 return Err("a transfer is writing into it".into());
9673 }
9674 let repo_dir = orphan
9679 .path
9680 .canonicalize()
9681 .map_err(|e| format!("cannot resolve the repo: {e}"))?;
9682 if let Some(user) = retire::path_resolving_into(&self.config.models_dir, &repo_dir) {
9683 return Err(format!("{} still resolves into it", user.display()));
9684 }
9685 for row in refreshed.all().filter(|m| m.is_local()) {
9686 let inside = refreshed
9687 .existing_local_artifact(&row.id)
9688 .and_then(|path| path.canonicalize().ok())
9689 .is_some_and(|path| path.starts_with(&repo_dir));
9690 if inside {
9691 return Err(format!("the model {} still loads from it", row.id));
9692 }
9693 }
9694 let label = format!("repo:{repo}");
9695 let now = std::time::SystemTime::now()
9696 .duration_since(std::time::UNIX_EPOCH)
9697 .map(|d| d.as_secs())
9698 .unwrap_or(0);
9699 let entry = |kind, detail: String| action_ledger::ConciergeActionEntry {
9700 seq: 0,
9701 kind,
9702 model_id: label.clone(),
9703 use_case: None,
9704 project: None,
9705 prior_model_id: None,
9706 detail,
9707 timestamp: now,
9708 };
9709 self.record_concierge_action(entry(
9710 action_ledger::ConciergeActionKind::RetireStarted,
9711 format!("orphaned repo; {} bytes planned", orphan.bytes),
9712 ));
9713 let quarantine = orphan.path.with_file_name(format!(
9714 "{}{:016x}",
9715 model_management::RETIRE_QUARANTINE_PREFIX,
9716 rand::random::<u64>()
9717 ));
9718 let verify = || retire::verify_repo_dir(&orphan.path, hub);
9719 self.model_management
9720 .detach_and_remove_tree(&orphan.path, &quarantine, &label, verify)
9721 .map_err(|failure| failure.error.to_string())?;
9722 self.record_concierge_action(entry(
9723 action_ledger::ConciergeActionKind::Retire,
9724 format!("orphaned repo no model uses; {} bytes", orphan.bytes),
9725 ));
9726 Ok(orphan.bytes)
9727 }
9728
9729 pub fn usage_heartbeat(&self) {
9733 self.model_management.beat();
9734 }
9735
9736 pub fn effective_update_policy(
9743 &self,
9744 prefs: &crate::update_prefs::UpdatePreferences,
9745 start_clock: bool,
9746 ) -> (crate::update_prefs::UpdatePolicy, Option<u64>) {
9747 use crate::update_prefs::UpdatePolicy;
9748 if prefs.policy != UpdatePolicy::Auto || prefs.policy_chosen {
9749 return (prefs.policy, None);
9750 }
9751 let marker = self.config.state_root.join(AUTO_GRACE_FILE);
9752 let now = std::time::SystemTime::now()
9753 .duration_since(std::time::UNIX_EPOCH)
9754 .map(|d| d.as_secs())
9755 .unwrap_or(0);
9756 let since = std::fs::read_to_string(&marker)
9757 .ok()
9758 .and_then(|text| text.trim().parse::<u64>().ok())
9759 .unwrap_or_else(|| {
9760 if start_clock {
9761 let _ = std::fs::create_dir_all(&self.config.state_root);
9762 let _ = std::fs::write(&marker, now.to_string());
9763 }
9764 now
9765 });
9766 match maintenance::auto_grace_until(since, now, AUTO_GRACE_SECS) {
9767 Some(until) => (UpdatePolicy::Notify, Some(until)),
9768 None => (UpdatePolicy::Auto, None),
9769 }
9770 }
9771
9772 pub async fn maintenance_pass(&self, dry_run: bool) -> maintenance::MaintenanceReport {
9781 self.maintenance_pass_in(&hf_cache::hub_dir(), &HardwareInfo::detect(), dry_run)
9782 .await
9783 }
9784
9785 pub async fn maintenance_pass_in(
9787 &self,
9788 hub: &Path,
9789 hw: &HardwareInfo,
9790 dry_run: bool,
9791 ) -> maintenance::MaintenanceReport {
9792 let prefs = self.update_prefs();
9793 let (policy, grace_until) = self.effective_update_policy(&prefs, !dry_run);
9794 let nudge = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
9795 let disk = crate::download::disk_space(hub).map(|(free_bytes, total_bytes)| {
9796 maintenance::DiskSpace {
9797 free_bytes,
9798 total_bytes,
9799 }
9800 });
9801 let mut report = self
9802 .maintenance_pass_with(hub, hw, dry_run, policy, &nudge, disk)
9803 .await;
9804 report.auto_grace_until = grace_until;
9805 report
9806 }
9807
9808 pub async fn maintenance_pass_with(
9811 &self,
9812 hub: &Path,
9813 hw: &HardwareInfo,
9814 dry_run: bool,
9815 policy: crate::update_prefs::UpdatePolicy,
9816 nudge: &crate::nudge::NudgeState,
9817 disk: Option<maintenance::DiskSpace>,
9818 ) -> maintenance::MaintenanceReport {
9819 if !dry_run {
9820 self.enforce_catalog_revocations().await;
9821 }
9822 let mut report = maintenance::MaintenanceReport {
9823 policy,
9824 dry_run,
9825 ..Default::default()
9826 };
9827 if policy == crate::update_prefs::UpdatePolicy::Off {
9828 return report;
9829 }
9830 if !dry_run {
9831 report.errors.extend(self.resume_retirements().await);
9832 }
9833 let portfolio = self.portfolio_in(hub, hw).await;
9834 let now = portfolio.policy.now;
9835 report.disk = disk;
9836 let was_pressured = self
9837 .disk_pressured
9838 .load(std::sync::atomic::Ordering::Relaxed);
9839 report.pressured = maintenance::disk_pressure(report.disk, was_pressured);
9840 if !dry_run {
9841 self.disk_pressured
9842 .store(report.pressured, std::sync::atomic::Ordering::Relaxed);
9843 }
9844 let keep = maintenance::kept_models(nudge, now);
9845 let churned = crate::action_ledger::read_actions(&self.action_ledger_path(), 0)
9848 .into_iter()
9849 .filter(|a| {
9850 matches!(
9851 a.kind,
9852 action_ledger::ConciergeActionKind::Retire
9853 | action_ledger::ConciergeActionKind::RetireStarted
9854 )
9855 })
9856 .map(|a| a.model_id)
9857 .collect();
9858 let ctx = maintenance::MaintenanceContext {
9859 policy,
9860 pressured: report.pressured,
9861 keep,
9862 churned,
9863 superseded_unpressured_secs:
9864 maintenance::MaintenanceContext::DEFAULT_SUPERSEDED_UNPRESSURED_SECS,
9865 };
9866 report.plan = maintenance::decide(&portfolio, &ctx);
9867 report.portfolio = Some(portfolio.clone());
9868 report.resource_policy = self
9869 .size_memory_policy(
9870 hw,
9871 portfolio.policy.tracking_since,
9872 !dry_run && policy == crate::update_prefs::UpdatePolicy::Auto,
9873 now,
9874 )
9875 .await;
9876 if dry_run || policy != crate::update_prefs::UpdatePolicy::Auto {
9877 return report;
9878 }
9879 if let Some(act) = &report.plan.act {
9880 match self
9881 .retire_model_in(&act.model_id, hub, false, Some(&act.digest))
9882 .await
9883 {
9884 Ok(outcome) => {
9885 report.retired = outcome.executed.then(|| maintenance::Retired {
9886 model_id: act.model_id.clone(),
9887 also_retired: outcome.plan.also_retires.clone(),
9888 freed_bytes: outcome.plan.freed_bytes,
9889 restore: format!("car models pull {}", act.model_id),
9890 });
9891 if !outcome.executed {
9892 report.errors.push(format!(
9895 "{}: not retired this pass ({:?})",
9896 act.model_id, outcome.plan.refusals
9897 ));
9898 }
9899 report.errors.extend(outcome.errors);
9900 }
9901 Err(error) => report.errors.push(format!("{}: {error}", act.model_id)),
9902 }
9903 }
9904 if report.retired.is_none() && report.pressured {
9907 let mut orphans: Vec<_> = portfolio
9910 .orphans
9911 .iter()
9912 .filter(|o| o.owner == retire::OrphanOwner::Car)
9913 .collect();
9914 orphans.sort_by(|a, b| b.bytes.cmp(&a.bytes));
9915 for orphan in orphans {
9916 match self.retire_orphan_repo(&orphan.repo, hub).await {
9917 Ok(freed_bytes) => {
9918 report.orphan_retired = Some(maintenance::RetiredOrphan {
9919 repo: orphan.repo.clone(),
9920 freed_bytes,
9921 });
9922 break;
9923 }
9924 Err(error) => report.errors.push(format!("{}: {error}", orphan.repo)),
9925 }
9926 }
9927 }
9928 if report.retired.is_none() && report.orphan_retired.is_none() && !report.pressured {
9930 match self.apply_one_curated_upgrade(hub, nudge, now).await {
9931 Ok(upgraded) => report.upgraded = upgraded,
9932 Err(error) => report.errors.push(error),
9933 }
9934 }
9935 let owned = retire::ownership(
9936 retire::catalog_ownership(),
9937 retire::recorded_files(&self.config.state_root, hub),
9938 );
9939 let (discarded, errors) = retire::discard_abandoned_partials(hub, &owned);
9940 report.errors.extend(errors);
9941 if !discarded.is_empty() {
9942 let bytes: u64 = discarded.iter().map(|(_, b)| b).sum();
9943 self.record_concierge_action(action_ledger::ConciergeActionEntry {
9944 seq: 0,
9945 kind: action_ledger::ConciergeActionKind::DiscardPartials,
9946 model_id: String::new(),
9947 use_case: None,
9948 project: None,
9949 prior_model_id: None,
9950 detail: format!(
9951 "{} abandoned partial downloads, {bytes} bytes",
9952 discarded.len()
9953 ),
9954 timestamp: now,
9955 });
9956 report.partials_discarded = discarded.into_iter().map(|(path, _)| path).collect();
9957 }
9958 report
9959 }
9960
9961 pub async fn portfolio(&self) -> portfolio::Portfolio {
9964 self.portfolio_in(&hf_cache::hub_dir(), &HardwareInfo::detect())
9965 .await
9966 }
9967
9968 pub async fn portfolio_in(&self, hub: &Path, hw: &HardwareInfo) -> portfolio::Portfolio {
9970 let now = std::time::SystemTime::now()
9971 .duration_since(std::time::UNIX_EPOCH)
9972 .map(|d| d.as_secs())
9973 .unwrap_or(0);
9974 let observable_since = self
9977 .model_management
9978 .usage_tracking_since()
9979 .zip(self.model_management.usage_covered_since())
9980 .map(|(tracking, covered)| tracking.max(covered));
9981 let policy = portfolio::PortfolioPolicy::new(now, observable_since);
9982 let (refreshed, owned) = self.retire_snapshot(hub);
9983 let protected = self.protected_models(&refreshed, true);
9984 let mut ledger_last: std::collections::HashMap<String, u64> =
9985 std::collections::HashMap::new();
9986 let mut ledger_failures: std::collections::HashMap<String, Vec<u64>> =
9993 std::collections::HashMap::new();
9994 for entry in crate::outcome::read_ledger(
9995 &self.config.state_models_dir().join("outcome_ledger.jsonl"),
9996 0,
9997 ) {
9998 if entry.counts_against_model() {
9999 ledger_failures
10000 .entry(entry.model_id.clone())
10001 .or_default()
10002 .push(entry.timestamp);
10003 }
10004 let last = ledger_last.entry(entry.model_id).or_default();
10005 *last = (*last).max(entry.timestamp);
10006 }
10007 let installed_at = |m: &ModelSchema| -> Option<u64> {
10010 let repo = retire::hf_references(m).into_iter().next()?.repo;
10011 let snapshots = hf_cache::checked_repo_dir_in(hub, &repo)?.join("snapshots");
10012 std::fs::read_dir(snapshots)
10013 .ok()?
10014 .filter_map(Result::ok)
10015 .filter_map(|e| e.metadata().ok()?.modified().ok())
10016 .filter_map(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
10017 .map(|d| d.as_secs())
10018 .max()
10019 };
10020 let installed = |m: &ModelSchema| {
10021 m.is_local()
10022 && m.downloads_weights()
10023 && m.weights_ready
10024 && self.model_management.car_enabled(&m.id).unwrap_or(true)
10025 };
10026 let runnable = |m: &ModelSchema| {
10027 let fit = recommend::model_fit(m, hw, None);
10028 fit.platform_compatible && fit.fit != recommend::ModelFitStatus::TooBig
10029 };
10030 let outcomes: std::collections::HashMap<String, (u64, u64)> = {
10032 let tracker = self.outcome_tracker.read().await;
10033 refreshed
10034 .all()
10035 .filter(|m| installed(m))
10036 .filter_map(|m| {
10037 tracker
10038 .profile(&m.id)
10039 .map(|p| (m.id.clone(), (p.success_count, p.fail_count)))
10040 })
10041 .collect()
10042 };
10043 let has_run = |id: &str| outcomes.get(id).is_some_and(|(success, _)| *success > 0);
10051 let dismissed =
10053 crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path())
10054 .dismissed;
10055 let superseded: std::collections::HashMap<String, String> = refreshed
10056 .available_upgrades()
10057 .into_iter()
10058 .filter(|u| !dismissed.contains(&format!("{}=>{}", u.from_id, u.to_id)))
10059 .filter(|u| {
10060 refreshed
10061 .get(&u.to_id)
10062 .is_some_and(|to| installed(to) && runnable(to) && has_run(&to.id))
10063 })
10064 .map(|u| (u.from_id, u.to_id))
10065 .collect();
10066 let superseded: std::collections::HashMap<String, String> = superseded
10068 .iter()
10069 .filter(|(from, to)| superseded.get(*to) != Some(*from))
10070 .map(|(from, to)| (from.clone(), to.clone()))
10071 .collect();
10072 let budget = recommend::memory_budget_mb(hw);
10073 let keep = maintenance::kept_models(
10074 &crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path()),
10075 now,
10076 );
10077 let mut facts = Vec::new();
10078 for m in refreshed.all().filter(|m| installed(m)) {
10079 let fit = recommend::model_fit(m, hw, None);
10080 let success_count = outcomes.get(&m.id).map_or(0, |(success, _)| *success);
10081 let since_install = installed_at(m)
10085 .unwrap_or(0)
10086 .max(policy.tracking_since.unwrap_or(u64::MAX));
10087 let fail_count = ledger_failures.get(&m.id).map_or(0, |times| {
10088 times.iter().filter(|t| **t >= since_install).count() as u64
10089 });
10090 let retire = self
10091 .retire_plan_on(&refreshed, &owned, &m.id, hub, None)
10092 .ok()
10093 .map(|(plan, _)| portfolio::RetireSummary {
10094 freed_bytes: plan.freed_bytes,
10095 refusals: plan
10096 .refusals
10097 .iter()
10098 .map(|r| {
10099 serde_json::to_value(r)
10100 .ok()
10101 .and_then(|v| {
10102 v.get("reason").and_then(|r| r.as_str()).map(str::to_owned)
10103 })
10104 .unwrap_or_else(|| format!("{r:?}"))
10105 })
10106 .collect(),
10107 also_retires: plan.also_retires.clone(),
10108 digest: plan.digest,
10109 });
10110 facts.push(portfolio::ModelFacts {
10111 model_id: m.id.clone(),
10112 name: m.name.clone(),
10113 generates_text: m.has_capability(ModelCapability::Generate),
10114 platform_compatible: fit.platform_compatible,
10115 never_fits: fit.fit == recommend::ModelFitStatus::TooBig,
10116 estimated_peak_mb: fit.estimated_peak_mb,
10117 hardware_budget_mb: budget,
10118 protected: protected.get(&m.id).cloned(),
10119 kept: keep.contains(&m.id),
10120 last_used: self.model_management.last_used(&m.id),
10121 ledger_last_used: ledger_last.get(&m.id).copied(),
10122 success_count,
10123 fail_count,
10124 superseded_by: superseded.get(&m.id).cloned(),
10125 retire,
10126 });
10127 }
10128 let referenced: std::collections::HashSet<String> = refreshed
10129 .all()
10130 .flat_map(retire::hf_references)
10131 .map(|r| r.repo)
10132 .collect();
10133 let mut assessed = portfolio::assess_all(&facts, &policy);
10134 assessed.orphans = retire::unreferenced_repos(hub, &referenced, &owned);
10135 assessed.recent_actions = self.recent_concierge_actions(now);
10136 assessed.revoked_pins = self.revoked_pins(&refreshed);
10137 assessed.revoked_copies =
10138 retire::revoked_copies(&self.config.state_root, &self.config.models_dir);
10139 assessed
10140 }
10141
10142 fn revoked_pins(&self, registry: &UnifiedRegistry) -> Vec<portfolio::RevokedPin> {
10150 let pins = self
10151 .external_pins
10152 .read()
10153 .unwrap_or_else(std::sync::PoisonError::into_inner)
10154 .clone();
10155 if pins.is_empty() {
10156 return Vec::new();
10157 }
10158 let names =
10159 catalog::load_revoked_names(&catalog::revoked_models_path(&self.config.state_root));
10160 pins.into_iter()
10161 .filter_map(|(pin, mut agents)| {
10162 let resolved = registry
10163 .get(&pin)
10164 .or_else(|| registry.find_by_name(&pin))
10165 .map(|schema| schema.id.clone());
10166 let model_id = if registry.is_revoked(&pin) {
10167 pin.clone()
10168 } else if let Some(id) = resolved {
10169 id
10170 } else {
10171 names.get(&pin)?.clone()
10172 };
10173 if !registry.is_revoked(&model_id) {
10174 return None;
10175 }
10176 agents.sort();
10177 agents.dedup();
10178 Some(portfolio::RevokedPin {
10179 pin,
10180 model_id,
10181 agents,
10182 })
10183 })
10184 .collect()
10185 }
10186
10187 fn protected_models(
10194 &self,
10195 registry: &UnifiedRegistry,
10196 include_lanes: bool,
10197 ) -> std::collections::BTreeMap<String, String> {
10198 let mut out = std::collections::BTreeMap::new();
10199 let mut protect = |key: &str, why: String| {
10200 let Some(schema) = registry.get(key).or_else(|| registry.find_by_name(key)) else {
10201 return;
10202 };
10203 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
10206 if let Some(twin) = registry.resolve_mlx_equivalent(schema) {
10207 out.entry(twin.id.clone()).or_insert_with(|| why.clone());
10208 }
10209 out.entry(schema.id.clone()).or_insert(why);
10210 };
10211 let lanes = if include_lanes {
10212 self.lane_defaults().defaults
10213 } else {
10214 Vec::new()
10215 };
10216 for lane in lanes {
10217 let use_case = serde_json::to_value(lane.use_case)
10218 .ok()
10219 .and_then(|v| v.as_str().map(str::to_owned))
10220 .unwrap_or_else(|| format!("{:?}", lane.use_case));
10221 protect(&lane.model_id, format!("the {use_case} lane default"));
10222 }
10223 let config = &self.config;
10224 for (key, what) in [
10225 (Some(&config.generation_model), "the generation model"),
10226 (
10227 config.preferred_generation_model.as_ref(),
10228 "the preferred generation model",
10229 ),
10230 (Some(&config.embedding_model), "the embedding model"),
10231 (
10232 config.preferred_embedding_model.as_ref(),
10233 "the preferred embedding model",
10234 ),
10235 (
10236 Some(&config.classification_model),
10237 "the classification model",
10238 ),
10239 (
10240 config.preferred_classification_model.as_ref(),
10241 "the preferred classification model",
10242 ),
10243 ] {
10244 if let Some(key) = key {
10245 protect(key, format!("{what} in CAR's configuration"));
10246 }
10247 }
10248 for (capability, what) in [
10249 (
10250 ModelCapability::SpeechToText,
10251 "the local speech-to-text default",
10252 ),
10253 (
10254 ModelCapability::TextToSpeech,
10255 "the local text-to-speech default",
10256 ),
10257 ] {
10258 if let Some(name) = self.speech_health_default_name(capability, true, false) {
10259 protect(&name, what.to_string());
10260 }
10261 }
10262 protect(
10263 LOCAL_TOOL_MODEL_PULL_ID,
10264 "the local tool model CAR's setup guidance installs".to_string(),
10265 );
10266 let pins = self
10267 .external_pins
10268 .read()
10269 .unwrap_or_else(std::sync::PoisonError::into_inner)
10270 .clone();
10271 for (key, agents) in pins {
10272 protect(&key, pinned_by(&agents));
10273 }
10274 out
10275 }
10276
10277 pub async fn resume_retirements(&self) -> Vec<String> {
10282 let _concierge = self.concierge_action_lock.lock().await;
10283 let mut errors: Vec<String> = self
10284 .model_management
10285 .resume_detached_trees()
10286 .into_iter()
10287 .map(|e| e.to_string())
10288 .collect();
10289 errors.extend(
10290 self.model_management
10291 .resume_retiring()
10292 .into_iter()
10293 .map(|e| e.to_string()),
10294 );
10295 errors
10296 }
10297
10298 pub fn model_management_store(&self) -> &model_management::ModelManagementStore {
10299 &self.model_management
10300 }
10301
10302 fn ensure_model_enabled(&self, model_id: &str) -> Result<(), InferenceError> {
10303 if self.model_management.car_enabled(model_id)? {
10304 return Ok(());
10305 }
10306 Err(InferenceError::InferenceFailed(format!(
10307 "local model {model_id} was removed from CAR; reinstall it before use"
10308 )))
10309 }
10310
10311 pub fn available_model_upgrades(&self) -> Vec<ModelUpgrade> {
10313 self.unified_registry().available_upgrades()
10314 }
10315
10316 pub async fn check_upgrade_nudge(
10322 &self,
10323 inference_active: bool,
10324 ) -> (crate::nudge::NudgeDecision, crate::nudge::NudgeState) {
10325 let findings = self.detect_upgrades().await;
10326 let mut prefs = self.update_prefs();
10327 prefs.policy = self.effective_update_policy(&prefs, false).0;
10328 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10329 let now = std::time::SystemTime::now()
10330 .duration_since(std::time::UNIX_EPOCH)
10331 .map(|d| d.as_secs())
10332 .unwrap_or(0);
10333 let decision = crate::nudge::decide_nudge(
10334 &findings,
10335 &prefs,
10336 &state,
10337 now,
10338 crate::nudge::DEFAULT_THROTTLE_SECS,
10339 inference_active,
10340 );
10341 (decision, state)
10342 }
10343
10344 pub fn dismiss_upgrade_nudge(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10347 let path = crate::nudge::NudgeState::default_path();
10348 let mut state = crate::nudge::NudgeState::load_from(&path);
10349 state.dismiss(dismiss_key);
10350 state
10351 .save_to(&path)
10352 .map_err(InferenceError::InferenceFailed)
10353 }
10354
10355 pub async fn check_concierge(
10362 &self,
10363 inference_active: bool,
10364 ) -> (
10365 Vec<crate::concierge::ConciergeSuggestion>,
10366 crate::nudge::NudgeState,
10367 ) {
10368 let prefs = self.update_prefs();
10369 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
10370 let hw = crate::hardware::HardwareInfo::detect();
10371 let schemas = self.list_schemas();
10372 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10373 let now = std::time::SystemTime::now()
10374 .duration_since(std::time::UNIX_EPOCH)
10375 .map(|d| d.as_secs())
10376 .unwrap_or(0);
10377 let suggestions = crate::concierge::decide_concierge(
10378 &refs,
10379 &hw,
10380 crate::concierge::DEFAULT_WATCHED_USE_CASES,
10381 crate::intent::QualityTier::Balanced,
10382 &prefs,
10383 &state,
10384 now,
10385 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
10386 inference_active,
10387 );
10388 (suggestions, state)
10389 }
10390
10391 pub fn dismiss_concierge_suggestion(&self, dismiss_key: &str) -> Result<(), InferenceError> {
10396 self.dismiss_upgrade_nudge(dismiss_key)
10397 }
10398
10399 pub fn dismiss_concierge_labeled(
10403 &self,
10404 dismiss_key: &str,
10405 reason: crate::concierge::DismissReason,
10406 ) -> Result<(), String> {
10407 let path = crate::nudge::NudgeState::default_path();
10408 let mut state = crate::nudge::NudgeState::load_from(&path);
10409 let now = std::time::SystemTime::now()
10410 .duration_since(std::time::UNIX_EPOCH)
10411 .map(|d| d.as_secs())
10412 .unwrap_or(0);
10413 state.dismiss_labeled(dismiss_key, reason, now);
10414 state.save_to(&path).map_err(|e| e.to_string())
10415 }
10416
10417 pub async fn check_canaries(&self) -> Vec<crate::intent::UseCase> {
10425 use crate::action_ledger::ConciergeActionKind;
10426 use crate::concierge::{
10427 canary_verdict, CanaryVerdict, CANARY_MIN_SAMPLES, CANARY_REGRESSION_MARGIN,
10428 };
10429 use std::collections::BTreeMap;
10430
10431 let actions = self.concierge_actions(0);
10432 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
10433 let entries = crate::outcome::read_ledger(&ledger_path, 0);
10434
10435 let mut latest: BTreeMap<
10437 crate::intent::UseCase,
10438 &crate::action_ledger::ConciergeActionEntry,
10439 > = BTreeMap::new();
10440 for a in &actions {
10441 if a.project.is_some() {
10442 continue; }
10444 if let Some(uc) = a.use_case {
10445 latest.insert(uc, a);
10446 }
10447 }
10448
10449 let mut reverts: Vec<(crate::intent::UseCase, u64)> = Vec::new();
10454 let tracker = self.outcome_tracker.read().await;
10455 for (uc, a) in latest {
10456 if a.kind != ConciergeActionKind::SetDefault {
10459 continue;
10460 }
10461 let Some(prior) = a.prior_model_id.as_deref() else {
10462 continue; };
10464
10465 let (mut succ, mut total) = (0u64, 0u64);
10467 for e in &entries {
10468 if e.timestamp < a.timestamp
10469 || e.model_id != a.model_id
10470 || crate::usage_profile::use_case_for_task(e.task) != uc
10471 {
10472 continue;
10473 }
10474 match e.success {
10475 Some(true) => {
10476 succ += 1;
10477 total += 1;
10478 }
10479 Some(false) => total += 1,
10480 None => {}
10481 }
10482 }
10483 let new_rate = if total == 0 {
10484 None
10485 } else {
10486 Some(succ as f64 / total as f64)
10487 };
10488
10489 let (mut base_succ, mut base_total) = (0u64, 0u64);
10494 if let Some(profile) = tracker.profile(prior) {
10495 for t in [
10496 crate::outcome::InferenceTask::Generate,
10497 crate::outcome::InferenceTask::Embed,
10498 crate::outcome::InferenceTask::Classify,
10499 crate::outcome::InferenceTask::Code,
10500 crate::outcome::InferenceTask::Reasoning,
10501 ] {
10502 if crate::usage_profile::use_case_for_task(t) != uc {
10503 continue;
10504 }
10505 if let Some(ts) = profile.task_stats(t) {
10506 base_succ += ts.successes;
10507 base_total += ts.successes + ts.failures;
10508 }
10509 }
10510 }
10511 if base_total == 0 {
10514 continue;
10515 }
10516 let baseline = base_succ as f64 / base_total as f64;
10517
10518 if canary_verdict(
10519 new_rate,
10520 total,
10521 baseline,
10522 CANARY_MIN_SAMPLES,
10523 CANARY_REGRESSION_MARGIN,
10524 ) == CanaryVerdict::Revert
10525 {
10526 reverts.push((uc, a.seq));
10530 }
10531 }
10532 drop(tracker);
10533
10534 let mut reverted = Vec::new();
10537 for (uc, seq) in reverts {
10538 if self.rollback_lane_inner(uc, None, Some(seq)).await.is_ok() {
10539 tracing::info!(lane = ?uc, "concierge canary: auto-reverted a worse model switch");
10540 reverted.push(uc);
10541 }
10542 }
10543 reverted
10544 }
10545
10546 pub async fn concierge_ask(
10563 &self,
10564 question: Option<&str>,
10565 ) -> Result<concierge_answer::ConciergeAnswer, String> {
10566 let maintenance = self.maintenance_pass(true).await;
10567 let portfolio = match maintenance.portfolio.clone() {
10568 Some(portfolio) => portfolio,
10569 None => self.portfolio().await,
10570 };
10571 let memory_policy =
10572 serde_json::to_value(&self.active_local_resource_policy().policy.profile)
10573 .ok()
10574 .and_then(|v| v.as_str().map(str::to_owned))
10575 .unwrap_or_else(|| "unknown".into());
10576 let summary = concierge_answer::summarize(&portfolio, &maintenance, &memory_policy);
10577 let Some(question) = question else {
10578 return Ok(concierge_answer::ConciergeAnswer {
10579 answer: summary.clone(),
10580 summary,
10581 model_used: None,
10582 ungrounded: Vec::new(),
10583 });
10584 };
10585 use std::fmt::Write as _;
10586 let status = self.concierge_status(false).await;
10587 let hw = crate::hardware::HardwareInfo::detect();
10588 let schemas = self.list_schemas();
10589 let refs: Vec<&ModelSchema> = schemas.iter().collect();
10590
10591 let mut evidence = String::new();
10592 evidence.push_str("OBSERVED USAGE (last 30 days):\n");
10593 if status.lanes.is_empty() {
10594 evidence.push_str(" (no usage recorded yet)\n");
10595 }
10596 for lane in &status.lanes {
10597 let rate = lane
10598 .success_rate()
10599 .map(|r| format!("{:.0}% success", r * 100.0))
10600 .unwrap_or_else(|| "no resolved signal".into());
10601 let _ = writeln!(
10602 evidence,
10603 " {:?}: {} calls, {}{}",
10604 lane.use_case,
10605 lane.calls,
10606 rate,
10607 if lane.failing_models.is_empty() {
10608 String::new()
10609 } else {
10610 format!(
10611 ", failing on {}",
10612 lane.failing_models
10613 .iter()
10614 .cloned()
10615 .collect::<Vec<_>>()
10616 .join(", ")
10617 )
10618 }
10619 );
10620 }
10621 evidence.push_str("\nMODEL HEALTH:\n");
10622 for m in &status.models {
10623 let success = match m.success_rate {
10624 Some(r) => format!("{:.0}% success", r * 100.0),
10625 None => "no resolved signal".to_string(),
10626 };
10627 let _ = writeln!(
10628 evidence,
10629 " {}: {} calls, {}, {:.0}ms avg{}",
10630 m.model_id,
10631 m.calls,
10632 success,
10633 m.avg_latency_ms,
10634 if m.excluded { " (excluded)" } else { "" }
10635 );
10636 }
10637 let mut menu_lanes: Vec<crate::intent::UseCase> =
10643 crate::concierge::DEFAULT_WATCHED_USE_CASES.to_vec();
10644 for lane in &status.lanes {
10645 if !menu_lanes.contains(&lane.use_case) {
10646 menu_lanes.push(lane.use_case);
10647 }
10648 }
10649 evidence.push_str("\nGROUNDED CANDIDATES (fit verified for this machine):\n");
10650 for uc in menu_lanes {
10651 let set = crate::recommend::recommend(
10652 &refs,
10653 &hw,
10654 uc,
10655 crate::intent::QualityTier::Balanced,
10656 crate::intent::Privacy::OnDevice,
10657 );
10658 for p in set.picks.iter().take(3) {
10659 let _ = writeln!(
10660 evidence,
10661 " [{:?}] {} — {}{}",
10662 uc,
10663 p.display_name,
10664 if p.already_installed {
10665 "installed"
10666 } else {
10667 "available"
10668 },
10669 if p.fit == crate::recommend::FitStatus::Fits {
10670 ", fits"
10671 } else {
10672 ", does NOT fit"
10673 }
10674 );
10675 }
10676 }
10677 if let Some(s) = &status.decision.suggestion {
10678 let _ = writeln!(evidence, "\nCURRENT SUGGESTION: {}", s.message);
10679 }
10680
10681 evidence.push_str("\nLOCAL MODELS ON THIS MACHINE:\n");
10682 evidence.push_str(&summary);
10683
10684 let prompt = format!(
10685 "You are CAR's model concierge. Answer the user's question ONLY from the \
10686 EVIDENCE provided as context: this machine's local models and what each is \
10687 doing, what maintenance would do, observed usage and health, and the \
10688 grounded candidate menu (the only models you may recommend acquiring). \
10689 Name models by the ids the evidence uses. NEVER invent a model and NEVER \
10690 claim a model fits or is better than the evidence states. If the evidence \
10691 doesn't answer the question, say so. Answer in a few short sentences or \
10692 bullets.\n\nQUESTION: {question}"
10693 );
10694 let req = crate::tasks::generate::GenerateRequest {
10695 prompt,
10696 context: Some(evidence.clone()),
10697 intent: Some(crate::intent::IntentHint {
10698 task: Some(crate::intent::TaskHint::Chat),
10699 prefer_local: true,
10700 ..Default::default()
10701 }),
10702 params: crate::tasks::generate::GenerateParams {
10703 temperature: 0.2,
10704 max_tokens: 600,
10705 workload: crate::tasks::generate::RoutingWorkload::Background,
10706 ..Default::default()
10707 },
10708 ..Default::default()
10709 };
10710 match self.generate_tracked(req).await {
10711 Ok(result) => {
10712 let reply = concierge_answer::trim_repetition(&result.text);
10713 let ungrounded = concierge_answer::ungrounded_ids(&reply, &evidence);
10714 let answer = if ungrounded.is_empty() {
10715 reply
10716 } else {
10717 format!(
10718 "{reply}\n\n(Not verified for this machine: {}. Only the models listed \
10719 above are checked.)",
10720 ungrounded.join(", ")
10721 )
10722 };
10723 Ok(concierge_answer::ConciergeAnswer {
10724 answer,
10725 summary,
10726 model_used: Some(result.model_used),
10727 ungrounded,
10728 })
10729 }
10730 Err(error) => {
10731 tracing::warn!(%error, "concierge.ask: no model could answer; returning the facts");
10732 Ok(concierge_answer::ConciergeAnswer {
10733 answer: format!(
10734 "{summary}\n(No model could answer the question itself: {error}.)"
10735 ),
10736 summary,
10737 model_used: None,
10738 ungrounded: Vec::new(),
10739 })
10740 }
10741 }
10742 }
10743
10744 pub async fn refresh_catalog(&self) -> Result<usize, String> {
10754 let url = catalog::catalog_url();
10755 let pubkey = catalog::catalog_public_key();
10756 let verified = self.fetch_checked_catalog(&url, &pubkey).await?;
10761 let path = crate::catalog::cache_path(&self.config.state_root);
10762 self.adopt_verified_catalog(&path, &verified, &pubkey).await
10763 }
10764
10765 async fn fetch_checked_catalog(
10770 &self,
10771 url: &str,
10772 pubkey: &str,
10773 ) -> Result<catalog::VerifiedCatalog, String> {
10774 use catalog::CheckFailure::{Older, Rejected, Unreachable};
10775 let http = crate::tls_client::catalog_refresh_client();
10776 let fetched = match crate::catalog::fetch_signed(&http, url).await {
10779 Err(error) => Err((Unreachable, error)),
10780 Ok((bytes, sig)) => {
10781 crate::catalog::verify_fetched(bytes, sig, pubkey).map_err(|e| (Rejected, e))
10782 }
10783 };
10784 let cached = crate::catalog::load_doc(
10788 &crate::catalog::cache_path(&self.config.state_root),
10789 Some(pubkey),
10790 )
10791 .map(|doc| doc.version);
10792 let outcome = match &fetched {
10793 Ok(verified) => match cached {
10794 Some(cached) if verified.version() < cached => Err((
10795 Older,
10796 format!(
10797 "the source served catalog v{}, older than the cached v{cached}",
10798 verified.version()
10799 ),
10800 )),
10801 _ => Ok(verified.version()),
10802 },
10803 Err((failure, error)) => Err((*failure, error.clone())),
10804 };
10805 catalog::record_check(
10806 &catalog::status_path(&self.config.state_root),
10807 url,
10808 outcome
10809 .as_ref()
10810 .map(|v| *v)
10811 .map_err(|(f, e)| (*f, e.as_str())),
10812 now_unix(),
10813 );
10814 fetched.map_err(|(_, error)| error)
10815 }
10816
10817 pub fn mark_catalog_loop_alive(&self) {
10819 catalog::record_loop_alive(&catalog::status_path(&self.config.state_root), now_unix());
10820 }
10821
10822 pub fn mark_daemon_alive_for_catalog(&self) {
10825 catalog::record_daemon_alive(&catalog::status_path(&self.config.state_root), now_unix());
10826 }
10827
10828 pub fn catalog_freshness(&self) -> catalog::CatalogFreshness {
10831 catalog::catalog_freshness(&catalog::load_status(&catalog::status_path(
10832 &self.config.state_root,
10833 )))
10834 }
10835
10836 pub async fn refresh_catalog_on_schedule(&self) -> Result<Option<usize>, String> {
10841 if matches!(self.update_prefs().policy, update_prefs::UpdatePolicy::Off) {
10842 catalog::record_off(&catalog::status_path(&self.config.state_root), now_unix());
10843 return Ok(None);
10844 }
10845 self.refresh_catalog_from(&catalog::catalog_url(), &catalog::catalog_public_key())
10846 .await
10847 }
10848
10849 async fn refresh_catalog_from(&self, url: &str, pubkey: &str) -> Result<Option<usize>, String> {
10851 let verified = self.fetch_checked_catalog(url, pubkey).await?;
10852 let path = crate::catalog::cache_path(&self.config.state_root);
10853 let cached = crate::catalog::load_doc(&path, Some(pubkey)).map(|doc| doc.version);
10854 if cached.is_some_and(|version| verified.version() <= version) {
10855 return Ok(None);
10856 }
10857 self.adopt_verified_catalog(&path, &verified, pubkey)
10858 .await
10859 .map(Some)
10860 }
10861
10862 async fn adopt_verified_catalog(
10869 &self,
10870 path: &Path,
10871 verified: &catalog::VerifiedCatalog,
10872 pubkey: &str,
10873 ) -> Result<usize, String> {
10874 let count = crate::catalog::install_if_newer(path, verified, pubkey).await?;
10875 let newest = crate::catalog::load_verified(path, Some(pubkey))
10876 .ok_or_else(|| format!("re-reading the verified catalog at {}", path.display()))?;
10877 let revoked = newest.revoked().to_vec();
10878 self.unlink_revoked_signed_models(&revoked).await;
10879 self.swap_signed_catalog(newest);
10880 self.clear_revoked_lane_defaults(&revoked).await;
10881 Ok(count)
10882 }
10883
10884 pub async fn enforce_catalog_revocations(&self) {
10889 let revoked = self.current_registry().revoked_ids();
10890 self.clear_revoked_lane_defaults(&revoked).await;
10891 }
10892
10893 async fn clear_revoked_lane_defaults(&self, revoked: &[String]) {
10901 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
10902 if revoked.is_empty() {
10903 return;
10904 }
10905 let _guard = self.concierge_action_lock.lock().await;
10906 let now = std::time::SystemTime::now()
10907 .duration_since(std::time::UNIX_EPOCH)
10908 .map(|d| d.as_secs())
10909 .unwrap_or(0);
10910 for lane in self.lane_defaults().defaults {
10911 if !revoked.contains(&lane.model_id) {
10912 continue;
10913 }
10914 match self.clear_lane_default(lane.project.as_deref(), lane.use_case) {
10915 Ok(true) => self.record_concierge_action(ConciergeActionEntry {
10916 seq: 0,
10917 kind: ConciergeActionKind::RevokedLaneCleared,
10918 model_id: lane.model_id.clone(),
10919 use_case: Some(lane.use_case),
10920 project: lane.project.clone(),
10921 prior_model_id: Some(lane.model_id.clone()),
10922 detail: "revoked by the signed model catalog".into(),
10923 timestamp: now,
10924 }),
10925 Ok(false) => {}
10926 Err(error) => {
10927 tracing::warn!(model = %lane.model_id, %error, "could not clear a revoked lane default")
10928 }
10929 }
10930 }
10931 for pin in self.revoked_pins(&self.current_registry()) {
10932 tracing::warn!(model = %pin.model_id, pin = %pin.pin, agents = ?pin.agents, "a pinned model was revoked by the signed catalog; re-point it");
10933 }
10934 }
10935
10936 async fn unlink_revoked_signed_models(&self, revoked: &[String]) {
10945 let registry = self.current_registry();
10946 let names_path = catalog::revoked_names_path(&self.config.state_root);
10947 let mut names = catalog::load_revoked_names(&names_path);
10948 let models_path = catalog::revoked_models_path(&self.config.state_root);
10949 let mut models = catalog::load_revoked_names(&models_path);
10950 names.retain(|_, id| revoked.contains(id));
10952 models.retain(|_, id| revoked.contains(id));
10953 let mut installed = Vec::new();
10954 for id in revoked {
10955 let Some(row) = registry.get(id) else {
10956 continue;
10957 };
10958 if registry.signed_origin_of(id).is_none() || registry.is_builtin(id) {
10959 continue;
10960 }
10961 if !retire::single_component(&row.name) {
10966 continue;
10967 }
10968 models.insert(row.name.clone(), id.clone());
10969 if row.is_local() && row.downloads_weights() {
10970 names.insert(row.name.clone(), id.clone());
10971 installed.push((id.clone(), row.name.clone()));
10972 }
10973 }
10974 for (path, record) in [(&names_path, &names), (&models_path, &models)] {
10975 if let Err(error) = catalog::save_revoked_names(path, record) {
10976 tracing::warn!(path = %path.display(), %error, "could not record revoked model names");
10977 }
10978 }
10979 let models_dir = self.config.models_dir.clone();
10980 for (id, name) in installed {
10981 let projection = models_dir.join(&name);
10982 if std::fs::symlink_metadata(&projection).is_err() || !retire::links_only(&projection) {
10983 continue;
10984 }
10985 match self.model_management.load_receipt(&id) {
10986 Ok(Some(receipt))
10987 if receipt.artifact_kind == model_management::ManagedArtifactKind::Symlink =>
10988 {
10989 if let Err(error) = self.remove_model_from_car(&id).await {
10990 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
10991 }
10992 }
10993 Ok(Some(_)) => {}
10994 Ok(None) => {
10995 if let Err(error) = retire::remove_links_only_tree(&projection, &models_dir) {
10996 tracing::warn!(%id, %error, "could not unlink a revoked model; discovery still skips it");
10997 }
10998 }
10999 Err(error) => {
11000 tracing::warn!(%id, %error, "could not read a revoked model's receipt")
11001 }
11002 }
11003 }
11004 }
11005
11006 pub async fn discover_models(&self) -> Result<usize, String> {
11015 use crate::schema::{ModelSource, TrustTier};
11016 let template = self
11023 .unified_registry()
11024 .all()
11025 .filter(|m| {
11026 m.provider.eq_ignore_ascii_case("openai")
11027 && m.trust_tier == TrustTier::Curated
11028 && matches!(m.source, ModelSource::RemoteApi { .. })
11029 })
11030 .max_by(|a, b| {
11031 a.capabilities
11032 .len()
11033 .cmp(&b.capabilities.len())
11034 .then_with(|| a.id.cmp(&b.id))
11035 })
11036 .cloned();
11037 let Some(template) = template else {
11038 return Ok(0); };
11040 let (endpoint, api_key_env) = match &template.source {
11041 ModelSource::RemoteApi {
11042 endpoint,
11043 api_key_env,
11044 ..
11045 } => (endpoint.clone(), api_key_env.clone()),
11046 _ => return Ok(0),
11047 };
11048 let Some(models_url) = crate::discovery::models_url_from_endpoint(&endpoint) else {
11049 return Ok(0);
11050 };
11051 let key = match car_secrets::resolve_env_or_keychain(&api_key_env) {
11052 Some(k) if !k.is_empty() => k,
11053 _ => return Ok(0), };
11055
11056 let http = reqwest::Client::builder()
11060 .timeout(std::time::Duration::from_secs(20))
11061 .build()
11062 .map_err(|e| format!("discovery client: {e}"))?;
11063 let ids = crate::discovery::fetch_model_ids(&http, &models_url, &key).await?;
11064
11065 let cache = crate::discovery::cache_path(&self.config.state_models_dir());
11067 let mut cached = crate::discovery::load_cache(&cache);
11068 let mut have: std::collections::HashSet<String> =
11069 cached.iter().map(|m| m.id.clone()).collect();
11070 for id in ids {
11071 if !crate::discovery::is_chat_model(&id) {
11072 continue;
11073 }
11074 let schema = crate::discovery::discovered_schema("openai", &id, &template);
11075 if self.unified_registry().get(&schema.id).is_some() {
11081 continue;
11082 }
11083 if have.insert(schema.id.clone()) {
11084 cached.push(schema);
11085 }
11086 }
11087 let count = cached.len();
11088 crate::discovery::save_cache(&cache, &cached)?;
11089 Ok(count)
11090 }
11091
11092 pub fn lane_defaults(&self) -> crate::lane_defaults::LaneDefaults {
11094 self.lane_defaults_cache.read().unwrap().clone()
11095 }
11096
11097 pub fn lane_default(
11101 &self,
11102 project: Option<&str>,
11103 use_case: crate::intent::UseCase,
11104 ) -> Option<String> {
11105 self.lane_defaults_cache
11106 .read()
11107 .unwrap()
11108 .resolve(project, use_case)
11109 .map(str::to_string)
11110 }
11111
11112 fn model_is_excluded(
11113 exclude_set: &std::collections::HashSet<String>,
11114 registry: &UnifiedRegistry,
11115 model: &str,
11116 ) -> bool {
11117 registry
11118 .get(model)
11119 .or_else(|| registry.find_by_name(model))
11120 .map(|schema| exclude_set.contains(&schema.id))
11121 .unwrap_or_else(|| exclude_set.contains(model))
11122 }
11123
11124 fn lane_pin_for(
11130 &self,
11131 req: &GenerateRequest,
11132 routing_registry: &UnifiedRegistry,
11133 ) -> Option<String> {
11134 let task = req.intent.as_ref().and_then(|h| h.task)?;
11135 let use_case = crate::usage_profile::use_case_for_task_hint(task);
11136 let id = self.lane_default(None, use_case)?;
11137 let known = routing_registry
11142 .get(&id)
11143 .or_else(|| routing_registry.find_by_name(&id));
11144 match known {
11145 Some(s) if s.available_now() => Some(id),
11146 _ => None,
11147 }
11148 }
11149
11150 pub fn set_lane_default(
11153 &self,
11154 project: Option<String>,
11155 use_case: crate::intent::UseCase,
11156 model_id: &str,
11157 ) -> Result<(), String> {
11158 if self.current_registry().is_revoked(model_id) {
11161 return Err(format!(
11162 "{model_id} was revoked by the signed model catalog and cannot be a lane default"
11163 ));
11164 }
11165 let now = std::time::SystemTime::now()
11166 .duration_since(std::time::UNIX_EPOCH)
11167 .map(|d| d.as_secs())
11168 .unwrap_or(0);
11169 let mut defaults = self.lane_defaults_cache.write().unwrap();
11172 defaults.set(project, use_case, model_id.to_string(), now);
11173 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11174 .map_err(|e| e.to_string())
11175 }
11176
11177 pub fn clear_lane_default(
11180 &self,
11181 project: Option<&str>,
11182 use_case: crate::intent::UseCase,
11183 ) -> Result<bool, String> {
11184 let mut defaults = self.lane_defaults_cache.write().unwrap();
11185 let removed = defaults.clear(project, use_case);
11186 crate::lane_defaults::save_to(&self.lane_defaults_path(), &defaults)
11187 .map_err(|e| e.to_string())?;
11188 Ok(removed)
11189 }
11190
11191 pub async fn user_set_lane_default(
11199 &self,
11200 use_case: crate::intent::UseCase,
11201 model_id: &str,
11202 project: Option<String>,
11203 ) -> Result<(), String> {
11204 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11205 let _guard = self.concierge_action_lock.lock().await;
11206 let prior = self.lane_default(project.as_deref(), use_case);
11207 self.set_lane_default(project.clone(), use_case, model_id)?;
11208 self.record_concierge_action(ConciergeActionEntry {
11209 seq: 0,
11210 kind: ConciergeActionKind::SetDefault,
11211 model_id: model_id.to_string(),
11212 use_case: Some(use_case),
11213 project,
11214 prior_model_id: prior,
11215 detail: "user set lane default".into(),
11216 timestamp: now_unix(),
11217 });
11218 Ok(())
11219 }
11220
11221 pub async fn user_clear_lane_default(
11226 &self,
11227 use_case: crate::intent::UseCase,
11228 project: Option<String>,
11229 ) -> Result<bool, String> {
11230 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11231 let _guard = self.concierge_action_lock.lock().await;
11232 let prior = self.lane_default(project.as_deref(), use_case);
11233 let removed = self.clear_lane_default(project.as_deref(), use_case)?;
11234 if removed {
11235 self.record_concierge_action(ConciergeActionEntry {
11236 seq: 0,
11237 kind: ConciergeActionKind::ClearDefault,
11238 model_id: prior.clone().unwrap_or_default(),
11239 use_case: Some(use_case),
11240 project,
11241 prior_model_id: prior,
11242 detail: "user cleared lane default".into(),
11243 timestamp: now_unix(),
11244 });
11245 }
11246 Ok(removed)
11247 }
11248
11249 pub fn concierge_actions(
11251 &self,
11252 limit: usize,
11253 ) -> Vec<crate::action_ledger::ConciergeActionEntry> {
11254 crate::action_ledger::read_actions(&self.action_ledger_path(), limit)
11255 }
11256
11257 fn lane_defaults_path(&self) -> PathBuf {
11259 self.config
11260 .state_root
11261 .join(crate::lane_defaults::LANE_DEFAULTS_FILE)
11262 }
11263
11264 fn action_ledger_path(&self) -> PathBuf {
11267 self.config
11268 .state_root
11269 .join(crate::action_ledger::ACTION_LEDGER_FILE)
11270 }
11271
11272 fn record_concierge_action(&self, mut entry: crate::action_ledger::ConciergeActionEntry) {
11273 entry.seq = self
11274 .concierge_action_seq
11275 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
11276 if let Err(e) = crate::action_ledger::append_action(&self.action_ledger_path(), &entry) {
11277 tracing::debug!("record concierge action failed: {e}");
11278 }
11279 }
11280
11281 pub async fn apply_concierge(
11293 &self,
11294 use_case: crate::intent::UseCase,
11295 model_id: &str,
11296 project: Option<String>,
11297 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11298 self.with_pinned_registry(self.apply_concierge_unpinned(use_case, model_id, project))
11299 .await
11300 }
11301
11302 async fn apply_concierge_unpinned(
11303 &self,
11304 use_case: crate::intent::UseCase,
11305 model_id: &str,
11306 project: Option<String>,
11307 ) -> Result<crate::action_ledger::ConciergeApplyResult, String> {
11308 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11309 let now = || {
11310 std::time::SystemTime::now()
11311 .duration_since(std::time::UNIX_EPOCH)
11312 .map(|d| d.as_secs())
11313 .unwrap_or(0)
11314 };
11315 if let Some(schema) = self
11320 .list_schemas()
11321 .into_iter()
11322 .find(|s| s.id == model_id || s.name == model_id)
11323 {
11324 let serves = use_case
11325 .required_capabilities()
11326 .iter()
11327 .all(|c| schema.capabilities.contains(c));
11328 if !serves {
11329 return Err(format!(
11330 "model '{model_id}' does not serve the {use_case:?} lane"
11331 ));
11332 }
11333 }
11334
11335 if self
11343 .unified_registry()
11344 .existing_local_artifact(model_id)
11345 .is_none()
11346 {
11347 self.pull_model(model_id).await.map_err(|e| e.to_string())?;
11348 }
11349
11350 let _guard = self.concierge_action_lock.lock().await;
11353 let prior = self.lane_default(project.as_deref(), use_case);
11356 self.record_concierge_action(ConciergeActionEntry {
11357 seq: 0, kind: ConciergeActionKind::Install,
11359 model_id: model_id.to_string(),
11360 use_case: Some(use_case),
11361 project: project.clone(),
11362 prior_model_id: None,
11363 detail: "concierge apply: ensured model present".into(),
11364 timestamp: now(),
11365 });
11366
11367 self.set_lane_default(project.clone(), use_case, model_id)?;
11369 self.record_concierge_action(ConciergeActionEntry {
11370 seq: 0, kind: ConciergeActionKind::SetDefault,
11372 model_id: model_id.to_string(),
11373 use_case: Some(use_case),
11374 project: project.clone(),
11375 prior_model_id: prior.clone(),
11376 detail: "concierge apply: set lane default".into(),
11377 timestamp: now(),
11378 });
11379
11380 Ok(crate::action_ledger::ConciergeApplyResult {
11381 model_id: model_id.to_string(),
11382 use_case,
11383 installed: true,
11384 set_default: true,
11385 prior_model_id: prior,
11386 })
11387 }
11388
11389 pub async fn rollback_lane(
11394 &self,
11395 use_case: crate::intent::UseCase,
11396 project: Option<String>,
11397 ) -> Result<Option<String>, String> {
11398 self.rollback_lane_inner(use_case, project, None).await
11399 }
11400
11401 async fn rollback_lane_inner(
11407 &self,
11408 use_case: crate::intent::UseCase,
11409 project: Option<String>,
11410 expected_anchor_seq: Option<u64>,
11411 ) -> Result<Option<String>, String> {
11412 use crate::action_ledger::{ConciergeActionEntry, ConciergeActionKind};
11413 let _guard = self.concierge_action_lock.lock().await;
11414 let actions = self.concierge_actions(0);
11419 let anchor = actions.iter().rev().find(|a| {
11420 matches!(
11421 a.kind,
11422 ConciergeActionKind::SetDefault | ConciergeActionKind::Rollback
11423 ) && a.use_case == Some(use_case)
11424 && a.project == project
11425 });
11426 let set = match anchor {
11427 None => return Err("no prior set-default to roll back".into()),
11428 Some(a) if a.kind == ConciergeActionKind::Rollback => {
11429 return Err("already rolled back to the prior default; nothing to undo".into())
11430 }
11431 Some(a) => a,
11432 };
11433 if let Some(seq) = expected_anchor_seq {
11436 if set.seq != seq {
11437 return Err("lane default changed since the canary decision; not reverting".into());
11438 }
11439 }
11440 let prior = set.prior_model_id.clone();
11441 let now = std::time::SystemTime::now()
11442 .duration_since(std::time::UNIX_EPOCH)
11443 .map(|d| d.as_secs())
11444 .unwrap_or(0);
11445
11446 match &prior {
11447 Some(m) => self.set_lane_default(project.clone(), use_case, m)?,
11448 None => {
11449 self.clear_lane_default(project.as_deref(), use_case)?;
11450 }
11451 }
11452 let detail = match &prior {
11453 Some(_) => "concierge rollback: restored prior lane default",
11454 None => "concierge rollback: cleared lane default (no prior)",
11455 };
11456 self.record_concierge_action(ConciergeActionEntry {
11457 seq: 0, kind: ConciergeActionKind::Rollback,
11459 model_id: prior.clone().unwrap_or_default(),
11460 use_case: Some(use_case),
11461 project,
11462 prior_model_id: Some(set.model_id.clone()),
11463 detail: detail.into(),
11464 timestamp: now,
11465 });
11466 Ok(prior)
11467 }
11468
11469 pub async fn concierge_status(
11474 &self,
11475 inference_active: bool,
11476 ) -> crate::concierge::ConciergeStatus {
11477 const USAGE_WINDOW_SECS: u64 = 30 * 24 * 60 * 60;
11479
11480 let prefs = self.update_prefs();
11481 let state = crate::nudge::NudgeState::load_from(&crate::nudge::NudgeState::default_path());
11482 let hw = crate::hardware::HardwareInfo::detect();
11483 let schemas = self.list_schemas();
11484 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11485 let now = std::time::SystemTime::now()
11486 .duration_since(std::time::UNIX_EPOCH)
11487 .map(|d| d.as_secs())
11488 .unwrap_or(0);
11489
11490 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
11491 let entries = crate::outcome::read_ledger(&ledger_path, 0);
11492 let usage =
11493 crate::usage_profile::UsageProfile::from_ledger(&entries, now, USAGE_WINDOW_SECS);
11494
11495 let decision = crate::concierge::evaluate_concierge(
11496 &refs,
11497 &hw,
11498 &usage,
11499 crate::intent::QualityTier::Balanced,
11500 &prefs,
11501 &state,
11502 now,
11503 crate::concierge::DEFAULT_CONCIERGE_THROTTLE_SECS,
11504 inference_active,
11505 );
11506
11507 let tracker = self.outcome_tracker.read().await;
11508 let models = tracker
11509 .export_profiles()
11510 .iter()
11511 .map(|p| crate::concierge::ModelHealth {
11512 model_id: p.model_id.clone(),
11513 calls: p.total_calls,
11514 success_rate: p.success_rate_resolved(),
11518 avg_latency_ms: p.avg_latency_ms(),
11519 quality: p.ema_quality,
11520 excluded: tracker.is_excluded(&p.model_id),
11521 })
11522 .collect();
11523 drop(tracker);
11524
11525 const STALE_VERIFY_SECS: u64 = 14 * 24 * 60 * 60;
11530 let actions = self.concierge_actions(0);
11531 let mut latest: std::collections::BTreeMap<
11532 crate::intent::UseCase,
11533 &crate::action_ledger::ConciergeActionEntry,
11534 > = std::collections::BTreeMap::new();
11535 for a in &actions {
11536 if a.project.is_none() {
11537 if let Some(uc) = a.use_case {
11538 latest.insert(uc, a);
11539 }
11540 }
11541 }
11542 let mut pending_verification = Vec::new();
11543 for (uc, a) in latest {
11544 if a.kind != crate::action_ledger::ConciergeActionKind::SetDefault {
11545 continue;
11546 }
11547 if now.saturating_sub(a.timestamp) < STALE_VERIFY_SECS {
11548 continue; }
11550 let resolved = entries
11551 .iter()
11552 .filter(|e| {
11553 e.timestamp >= a.timestamp
11554 && e.model_id == a.model_id
11555 && crate::usage_profile::use_case_for_task(e.task) == uc
11556 && e.success.is_some()
11557 })
11558 .count() as u64;
11559 if resolved < crate::concierge::CANARY_MIN_SAMPLES {
11560 pending_verification.push(crate::concierge::PendingVerification {
11561 use_case: uc,
11562 model_id: a.model_id.clone(),
11563 set_at: a.timestamp,
11564 resolved_samples: resolved,
11565 needed: crate::concierge::CANARY_MIN_SAMPLES,
11566 });
11567 }
11568 }
11569
11570 crate::concierge::ConciergeStatus {
11571 lanes: usage.active_lanes().into_iter().cloned().collect(),
11572 decision,
11573 models,
11574 pending_verification,
11575 catalog: self.catalog_freshness(),
11576 revoked_pins: self.revoked_pins(&self.current_registry()),
11577 }
11578 }
11579
11580 pub async fn detect_upgrades(&self) -> Vec<crate::upgrade::UpgradeFinding> {
11584 let prefs = self.update_prefs();
11585 let curated = self.unified_registry().available_upgrades();
11586 let schemas = self.list_schemas();
11587 let refs: Vec<&ModelSchema> = schemas.iter().collect();
11588 let probe = crate::upgrade::HuggingFaceProbe::new();
11589 let now = std::time::SystemTime::now()
11590 .duration_since(std::time::UNIX_EPOCH)
11591 .map(|d| d.as_secs())
11592 .unwrap_or(0);
11593 let mut findings = crate::upgrade::detect_upgrades(
11594 curated,
11595 &refs,
11596 &prefs,
11597 &probe,
11598 &crate::upgrade::UpgradeCache::default_path(),
11599 now,
11600 crate::upgrade::DEFAULT_TTL_SECS,
11601 )
11602 .await;
11603 let lane_models: std::collections::HashSet<String> = self
11608 .lane_defaults()
11609 .defaults
11610 .into_iter()
11611 .map(|d| d.model_id)
11612 .collect();
11613 let in_use = |m: &ModelSchema| {
11614 lane_models.contains(&m.id)
11615 || self.model_management.last_used(&m.id).is_some_and(|at| {
11616 now.saturating_sub(at) < portfolio::PortfolioPolicy::DEFAULT_IDLE_AFTER_SECS
11617 })
11618 };
11619 let installed: Vec<&ModelSchema> = refs
11620 .iter()
11621 .copied()
11622 .filter(|m| {
11623 m.is_local()
11624 && m.downloads_weights()
11625 && m.weights_ready
11626 && self.model_management.car_enabled(&m.id).unwrap_or(true)
11627 && in_use(m)
11628 })
11629 .collect();
11630 let fit = |m: &ModelSchema| {
11631 let fit = self.model_fit(m);
11632 fit.platform_compatible.then_some(fit.fit)
11633 };
11634 let registry = self.unified_registry();
11635 let found = crate::upgrade::suggest_only_unless_builtin(
11636 crate::upgrade::benchmark_findings(&installed, &refs, &fit),
11637 &|id| registry.is_builtin(id),
11638 );
11639 for finding in found {
11640 let duplicate = findings
11641 .iter()
11642 .any(|f| f.from_id == finding.from_id && f.to_id == finding.to_id);
11643 if !duplicate {
11644 findings.push(finding);
11645 }
11646 }
11647 findings
11648 }
11649
11650 pub fn list_schemas(&self) -> Vec<ModelSchema> {
11653 self.catalog_registry_snapshot()
11654 .list()
11655 .into_iter()
11656 .cloned()
11657 .collect()
11658 }
11659
11660 pub fn catalog_snapshot(&self) -> Result<CatalogSnapshot, String> {
11664 CatalogSnapshot::new(self.unified_registry().list().into_iter().cloned())
11665 }
11666
11667 pub fn registered_schema(&self, id: &str) -> Option<ModelSchema> {
11672 self.unified_registry().registered_schema(id).cloned()
11673 }
11674
11675 pub fn list_models(&self) -> Vec<models::ModelInfo> {
11676 self.registry.list_models()
11677 }
11678
11679 pub fn knows_model(&self, name: &str) -> bool {
11692 self.model_schema(name).is_some()
11693 }
11694
11695 pub fn model_schema(&self, name: &str) -> Option<ModelSchema> {
11713 self.resolve_schema(name)
11714 }
11715
11716 fn first_installed_local_model(&self, needs_tools: bool) -> Option<String> {
11729 let registry = self.unified_registry();
11737 let mut candidates: Vec<_> = registry
11738 .all()
11739 .filter(|s| s.is_local() && !s.is_vllm_mlx())
11746 .filter(|s| !s.is_foundation_models() || s.available)
11762 .filter(|s| s.has_capability(ModelCapability::Generate))
11763 .filter(|s| !needs_tools || s.has_capability(ModelCapability::ToolUse))
11767 .filter(|s| self.model_management.car_enabled(&s.id).unwrap_or(false))
11768 .filter(|s| self.unified_registry().ready_without_download(&s.id) == Some(true))
11769 .collect();
11770 candidates.sort_by_key(|s| s.size_mb());
11772
11773 if let Some(avail) = crate::hardware::available_ram_mb() {
11785 const WORKING_RESERVE_MB: u64 = 2048;
11786 candidates.retain(|s| s.size_mb().saturating_add(WORKING_RESERVE_MB) <= avail);
11787 }
11788
11789 candidates.first().map(|s| s.name.clone())
11790 }
11791
11792 pub async fn pull_model(&self, name: &str) -> Result<std::path::PathBuf, InferenceError> {
11794 self.pull_model_with_progress(name, &crate::download::ProgressSink::none())
11795 .await
11796 }
11797
11798 pub async fn pull_model_with_progress(
11803 &self,
11804 name: &str,
11805 sink: &crate::download::ProgressSink,
11806 ) -> Result<std::path::PathBuf, InferenceError> {
11807 self.with_pinned_registry(async {
11808 let pulled = self.pull_model_with_progress_unpinned(name, sink).await;
11809 if pulled.is_ok() {
11810 self.keep_pulled_row_if_withdrawn(name);
11811 }
11812 pulled
11813 })
11814 .await
11815 }
11816
11817 async fn pull_model_with_progress_unpinned(
11818 &self,
11819 name: &str,
11820 sink: &crate::download::ProgressSink,
11821 ) -> Result<std::path::PathBuf, InferenceError> {
11822 let schema = self
11823 .resolve_schema_name_first(name)
11824 .ok_or_else(|| InferenceError::ModelNotFound(name.to_string()))?;
11825 let _mutation = self.model_management.begin_mutation(&schema.id)?;
11826 if let Some(receipt) = self.model_management.load_receipt(&schema.id)? {
11827 let _ = self.model_management.can_remove(&schema.id)?;
11831 self.model_management.clear_tombstone(&schema.id)?;
11832 return Ok(receipt.managed_path);
11833 }
11834 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
11835 return Ok(receipt.managed_path);
11836 }
11837 let expected_managed = self.model_management.models_dir().join(&schema.name);
11838 if std::fs::symlink_metadata(&expected_managed).is_ok() {
11839 return Err(InferenceError::InferenceFailed(format!(
11840 "model {} has a pre-existing unreceipted artifact at {}; use models.adopt or move it before pulling",
11841 schema.id,
11842 expected_managed.display()
11843 )));
11844 }
11845 let staging = self.model_management.create_install_staging(&schema.id)?;
11846 let installed = match self
11847 .unified_registry()
11848 .ensure_local_with_progress_staged(&schema.id, sink, &staging)
11849 .await
11850 {
11851 Ok(installed) => installed,
11852 Err(error) => {
11853 self.model_management.discard_install_staging(&staging);
11854 return Err(error);
11855 }
11856 };
11857 let installed_is_staging = match (installed.canonicalize(), staging.canonicalize()) {
11858 (Ok(installed), Ok(staging)) => installed == staging,
11859 _ => false,
11860 };
11861 let generation = self
11862 .resource_policy_generation
11863 .load(std::sync::atomic::Ordering::Acquire);
11864 let managed_path = self.model_management.models_dir().join(&schema.name);
11865 let managed = if installed_is_staging {
11866 let receipt = self.model_management.install_receipt_for_publication(
11867 &schema.id,
11868 model_source_identity(&schema),
11869 generation,
11870 false,
11871 managed_path,
11872 model_management::ManagedArtifactKind::Directory,
11873 &staging,
11874 )?;
11875 self.model_management
11876 .begin_install_intent(receipt, Some(&staging))?;
11877 self.model_management
11878 .publish_install_staging(&schema.id, &staging, &schema.name)?
11879 } else {
11880 self.model_management.discard_install_staging(&staging);
11881 let receipt = self.model_management.install_receipt_for_publication(
11882 &schema.id,
11883 model_source_identity(&schema),
11884 generation,
11885 false,
11886 managed_path,
11887 model_management::ManagedArtifactKind::Symlink,
11888 &installed,
11889 )?;
11890 self.model_management.begin_install_intent(receipt, None)?;
11891 self.model_management
11892 .materialize_managed_projection(&schema.name, &installed)?
11893 };
11894 let receipt = self
11895 .model_management
11896 .resume_install_intent(&schema.id)?
11897 .ok_or_else(|| {
11898 InferenceError::InferenceFailed(format!(
11899 "model {} publication completed without a durable install intent",
11900 schema.id
11901 ))
11902 })?;
11903 debug_assert_eq!(receipt.managed_path, managed);
11904 Ok(receipt.managed_path)
11905 }
11906
11907 pub async fn adopt_model_into_car(
11911 &self,
11912 model_id: &str,
11913 ) -> Result<model_management::InstallReceipt, InferenceError> {
11914 let schema = self
11915 .resolve_schema(model_id)
11916 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
11917 if !schema.is_local() || !schema.downloads_weights() {
11918 return Err(InferenceError::InferenceFailed(format!(
11919 "model {} is not a CAR-manageable local artifact",
11920 schema.id
11921 )));
11922 }
11923 let existing = self
11924 .unified_registry()
11925 .existing_local_artifact(&schema.id)
11926 .ok_or_else(|| {
11927 InferenceError::InferenceFailed(format!(
11928 "model {} has no usable local artifact to adopt",
11929 schema.id
11930 ))
11931 })?;
11932 let _mutation = self.model_management.begin_mutation(&schema.id)?;
11933 let existing_is_managed_symlink = std::fs::symlink_metadata(&existing)
11934 .is_ok_and(|metadata| metadata.file_type().is_symlink())
11935 && existing.parent().is_some_and(|parent| {
11936 parent.canonicalize().ok() == self.model_management.models_dir().canonicalize().ok()
11937 });
11938 if let Some(receipt) = self.model_management.resume_install_intent(&schema.id)? {
11939 return Ok(receipt);
11940 }
11941 let generation = self
11942 .resource_policy_generation
11943 .load(std::sync::atomic::Ordering::Acquire);
11944 let managed = if existing_is_managed_symlink {
11945 existing
11946 } else {
11947 let managed = self.model_management.adopted_projection_path(&schema.id)?;
11948 let receipt = self.model_management.install_receipt_for_publication(
11949 &schema.id,
11950 model_source_identity(&schema),
11951 generation,
11952 true,
11953 managed,
11954 model_management::ManagedArtifactKind::Symlink,
11955 &existing,
11956 )?;
11957 self.model_management.begin_install_intent(receipt, None)?;
11958 self.model_management
11959 .materialize_adopted_projection(&schema.id, &existing)?
11960 };
11961 if existing_is_managed_symlink {
11962 self.model_management
11963 .record_managed_artifact(
11964 &schema.id,
11965 model_source_identity(&schema),
11966 None,
11967 generation,
11968 true,
11969 managed,
11970 )
11971 .map_err(InferenceError::from)
11972 } else {
11973 self.model_management
11974 .resume_install_intent(&schema.id)?
11975 .ok_or_else(|| {
11976 InferenceError::InferenceFailed(format!(
11977 "model {} adoption completed without a durable install intent",
11978 schema.id
11979 ))
11980 })
11981 }
11982 }
11983
11984 pub async fn remove_model_from_car(
11987 &self,
11988 model_id: &str,
11989 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
11990 self.remove_model_from_car_inner(model_id, false).await
11991 }
11992
11993 async fn remove_model_from_car_inner(
11997 &self,
11998 model_id: &str,
11999 maintenance_held: bool,
12000 ) -> Result<model_management::RemoveFromCarResult, InferenceError> {
12001 let schema = self
12002 .resolve_schema(model_id)
12003 .ok_or_else(|| InferenceError::ModelNotFound(model_id.to_string()))?;
12004 if !schema.is_local() || !schema.downloads_weights() {
12005 return Err(InferenceError::InferenceFailed(format!(
12006 "model {} is not a CAR-owned local artifact",
12007 schema.id
12008 )));
12009 }
12010 if self.model_management.load_receipt(&schema.id)?.is_none() {
12011 if self.model_management.car_enabled(&schema.id)? {
12012 return Err(model_management::ModelManagementError::MissingReceipt {
12013 model_id: schema.id.clone(),
12014 }
12015 .into());
12016 }
12017 let generation = self
12021 .resource_policy_generation
12022 .load(std::sync::atomic::Ordering::Acquire)
12023 .saturating_add(1);
12024 return self
12025 .model_management
12026 .begin_mutation(&schema.id)?
12027 .remove(generation)
12028 .map_err(InferenceError::from);
12029 }
12030 let mutation = self.model_management.begin_mutation(&schema.id)?;
12031 let _maintenance = if maintenance_held {
12032 None
12033 } else {
12034 Some(
12035 self.prepare_local_model_removal(&schema.id)
12036 .await
12037 .map_err(|error| InferenceError::InferenceFailed(error.to_string()))?,
12038 )
12039 };
12040 let generation = self
12041 .resource_policy_generation
12042 .load(std::sync::atomic::Ordering::Acquire)
12043 .saturating_add(1);
12044 mutation.remove(generation).map_err(InferenceError::from)
12045 }
12046
12047 pub fn update_prefs(&self) -> crate::update_prefs::UpdatePreferences {
12052 let cwd = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("."));
12053 crate::update_prefs::UpdatePreferences::load_effective(&cwd).unwrap_or_else(|error| {
12054 tracing::warn!(%error, "update preferences unreadable; acting as notify");
12057 crate::update_prefs::UpdatePreferences {
12058 policy: crate::update_prefs::UpdatePolicy::Notify,
12059 policy_chosen: true,
12060 ..Default::default()
12061 }
12062 })
12063 }
12064
12065 pub fn set_update_prefs(
12067 &self,
12068 prefs: &crate::update_prefs::UpdatePreferences,
12069 ) -> Result<(), InferenceError> {
12070 prefs.save().map_err(InferenceError::InferenceFailed)
12071 }
12072
12073 #[deprecated(note = "use async remove_model_from_car for receipt-backed safe removal")]
12077 pub fn remove_model(&self, name: &str) -> Result<(), InferenceError> {
12078 Err(InferenceError::InferenceFailed(format!(
12079 "legacy removal for {name} is disabled; use async receipt-backed remove_model_from_car"
12080 )))
12081 }
12082
12083 pub fn register_model(&mut self, schema: ModelSchema) {
12089 self.unified_registry_mut().register(schema);
12090 }
12091
12092 pub fn register_user_model(&mut self, schema: ModelSchema) {
12094 self.unified_registry_mut().register_user_model(schema);
12095 }
12096
12097 pub async fn discover_vllm_mlx_models(&mut self) -> usize {
12100 let config = vllm_mlx::VllmMlxConfig::default();
12101 if !config.auto_discover {
12102 return 0;
12103 }
12104 vllm_mlx::discover_and_register(&config, self.unified_registry_mut()).await
12105 }
12106
12107 pub fn outcome_tracker(&self) -> Arc<RwLock<OutcomeTracker>> {
12109 self.outcome_tracker.clone()
12110 }
12111
12112 async fn auto_save_outcomes(&self) {
12120 const OUTCOME_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
12121
12122 if crate::offload::is_offload_worker() {
12130 return;
12131 }
12132
12133 let due = {
12135 let last = self.last_outcome_flush.lock().unwrap();
12136 last.is_none_or(|t| t.elapsed() >= OUTCOME_FLUSH_INTERVAL)
12137 };
12138 if due {
12139 match self.persist_outcomes().await {
12140 Ok(did) => {
12141 if did {
12142 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12143 }
12144 }
12145 Err(e) => tracing::debug!("auto-save outcomes failed: {}", e),
12146 }
12147 }
12148
12149 if let Err(e) = self.save_key_pool_stats().await {
12150 tracing::debug!("auto-save key pool stats failed: {}", e);
12151 }
12152 }
12153
12154 async fn persist_outcomes(&self) -> Result<bool, std::io::Error> {
12160 const PENDING_TTL_SECS: u64 = 300;
12161 let entries = {
12165 let mut tracker = self.outcome_tracker.write().await;
12166 tracker.sweep_pending(PENDING_TTL_SECS);
12167 tracker.drain_ledger()
12168 };
12169 let mut did = false;
12170 let ledger_disabled = std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some();
12175 if !entries.is_empty() && !ledger_disabled {
12176 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12177 let _guard = self.ledger_io_lock.lock().await;
12178 crate::outcome::append_ledger_entries(&ledger_path, &entries)?;
12179 did = true;
12180 }
12181 let profiles_path = self.config.state_models_dir().join("outcome_profiles.json");
12183 let wrote = {
12184 let mut tracker = self.outcome_tracker.write().await;
12185 tracker.save_if_dirty(&profiles_path)?
12186 };
12187 Ok(did || wrote)
12188 }
12189
12190 pub async fn save_outcomes(&self) -> Result<(), std::io::Error> {
12195 let tracker = self.outcome_tracker.read().await;
12196 let path = self.config.state_models_dir().join("outcome_profiles.json");
12197 tracker.save_to_file(&path)
12198 }
12199
12200 pub async fn flush_outcomes(&self) -> Result<bool, std::io::Error> {
12206 self.flush_outcomes_inner().await
12207 }
12208
12209 pub async fn reconcile_outcome_ledger(
12225 &self,
12226 ) -> Result<outcome::LedgerReconciliation, std::io::Error> {
12227 if crate::offload::is_offload_worker()
12228 || std::env::var_os("CAR_NO_OUTCOME_LEDGER").is_some()
12229 || !self.outcome_profiles_loaded
12230 {
12231 return Ok(outcome::LedgerReconciliation::default());
12232 }
12233 let models_dir = self.config.state_models_dir();
12234 let ledger = models_dir.join("outcome_ledger.jsonl");
12235 let _ = std::fs::remove_file(ledger.with_extension("jsonl.migrated"));
12238 self.flush_outcomes_inner().await?;
12239 let seeds = quality_seeds(&models_dir);
12240 let _io = self.ledger_io_lock.lock().await;
12241 let registry = &self.unified_registry();
12242 let is_registered = |id: &str| registry.get(id).is_some();
12243 let mut tracker = self.outcome_tracker.write().await;
12244 let plan = outcome::plan_reconcile(&ledger, &tracker, &is_registered, &seeds);
12245 if plan.ledger.is_none() && plan.profiles.is_none() {
12246 return Ok(plan.report);
12247 }
12248 if let Some(entries) = &plan.ledger {
12249 outcome::write_reconciled_ledger(&ledger, entries)?;
12250 }
12251 if let Some(profiles) = plan.profiles {
12252 tracker.commit_reconciled(profiles);
12253 tracker.save_to_file(&models_dir.join("outcome_profiles.json"))?;
12254 }
12255 tracing::info!(
12256 reclassified = plan.report.reclassified,
12257 unregistered = plan.report.unregistered,
12258 profiles_rebuilt = plan.report.profiles_rebuilt,
12259 profiles_removed = plan.report.profiles_removed,
12260 "reconciled the outcome ledger with model profiles"
12261 );
12262 Ok(plan.report)
12263 }
12264
12265 async fn flush_outcomes_inner(&self) -> Result<bool, std::io::Error> {
12266 let did = self.persist_outcomes().await?;
12267 if did {
12268 *self.last_outcome_flush.lock().unwrap() = Some(Instant::now());
12269 }
12270 Ok(did)
12271 }
12272
12273 pub async fn prune_outcome_ledger(&self, max_entries: usize) -> std::io::Result<()> {
12276 let path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12277 let _guard = self.ledger_io_lock.lock().await;
12278 crate::outcome::prune_ledger(&path, max_entries)
12279 }
12280
12281 pub async fn save_key_pool_stats(&self) -> Result<(), std::io::Error> {
12283 let path = self.config.state_models_dir().join("key_pool_stats.json");
12284 self.remote_backend.key_pool.save_stats(&path).await
12285 }
12286
12287 pub async fn key_pool_stats(
12289 &self,
12290 ) -> std::collections::HashMap<String, Vec<key_pool::KeyStats>> {
12291 self.remote_backend.key_pool.all_stats().await
12292 }
12293
12294 pub async fn export_profiles(&self) -> Vec<ModelProfile> {
12296 let tracker = self.outcome_tracker.read().await;
12297 tracker.export_profiles()
12298 }
12299
12300 pub fn outcome_scoreboard(&self) -> crate::scoreboard::Scoreboard {
12308 let ledger_path = self.config.state_models_dir().join("outcome_ledger.jsonl");
12309 let entries = crate::outcome::read_ledger(&ledger_path, 0);
12310 crate::calibration::ShadowCalibration::from_ledger(&entries).emit();
12314 crate::scoreboard::Scoreboard::from_ledger(&entries, |id| {
12315 let s = self.resolve_schema(id)?;
12316 scoreboard_price_model(&s)
12317 })
12318 }
12319
12320 pub async fn import_profiles(&self, profiles: Vec<ModelProfile>) {
12322 let mut tracker = self.outcome_tracker.write().await;
12323 tracker.import_profiles(profiles);
12324 }
12325
12326 pub async fn prepare_speech_runtime(&self) -> Result<PathBuf, InferenceError> {
12349 self.ensure_speech_runtime().await.map(|r| r.root)
12355 }
12356
12357 pub fn set_speech_policy(&mut self, policy: SpeechPolicy) {
12359 self.speech_policy = policy;
12360 }
12361
12362 pub fn set_routing_config(&mut self, config: RoutingConfig) {
12363 self.adaptive_router.set_config(config);
12364 }
12365
12366 pub async fn install_curated_speech(
12368 &mut self,
12369 ) -> Result<Vec<SpeechInstallReport>, InferenceError> {
12370 if let Err(error) = self.prepare_speech_runtime().await {
12383 tracing::warn!(
12384 %error,
12385 "managed speech runtime could not be provisioned; installing the \
12386 models that do not depend on it"
12387 );
12388 }
12389 let schemas = self.list_schemas();
12390 let mut repos = Vec::new();
12391 for schema in &schemas {
12392 if !schema.is_mlx() || !schema.tags.iter().any(|tag| tag == "speech") {
12393 continue;
12394 }
12395 if !cfg!(all(
12410 target_os = "macos",
12411 target_arch = "aarch64",
12412 not(car_skip_mlx)
12413 )) {
12414 continue;
12415 }
12416 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
12417 if !repos.iter().any(|existing: &String| existing == hf_repo) {
12418 repos.push(hf_repo.clone());
12419 }
12420 }
12421 }
12422
12423 let mut installed = Vec::new();
12424 for repo in repos {
12425 let model_for_record = self
12426 .unified_registry()
12427 .all()
12428 .find(|m| matches!(&m.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo))
12429 .map(|m| m.id.clone())
12430 .unwrap_or_else(|| repo.clone());
12431 let _guard = crate::download::acquire_model_lock(&model_for_record).await;
12434 let (snapshot, fetched) =
12435 crate::retire::collecting_fetches(download_hf_repo_snapshot(&repo)).await;
12436 crate::retire::record_download(
12437 &self.config.state_root,
12438 &crate::hf_cache::hub_dir(),
12439 &model_for_record,
12440 &fetched,
12441 );
12442 let (snapshot_path, files_downloaded) = snapshot?;
12443 let name = schemas
12444 .iter()
12445 .find(|schema| {
12446 matches!(&schema.source, ModelSource::Mlx { hf_repo, .. } if hf_repo == &repo)
12447 })
12448 .map(|schema| schema.name.clone())
12449 .unwrap_or_else(|| repo.clone());
12450 installed.push(SpeechInstallReport {
12451 name,
12452 hf_repo: repo,
12453 snapshot_path,
12454 files_downloaded,
12455 });
12456 }
12457
12458 for schema in &schemas {
12462 if !schema.tags.iter().any(|tag| tag == "speech") {
12463 continue;
12464 }
12465 if let ModelSource::WhisperCpp { model } = &schema.source {
12466 let model = model.clone();
12467 let name = schema.name.clone();
12468 let path = tokio::task::spawn_blocking(move || car_whisper::ensure_model(&model))
12469 .await
12470 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))?
12471 .map_err(|e| {
12472 InferenceError::InferenceFailed(format!("whisper model fetch: {e}"))
12473 })?;
12474 installed.push(SpeechInstallReport {
12475 name,
12476 hf_repo: "ggerganov/whisper.cpp".to_string(),
12477 snapshot_path: path,
12478 files_downloaded: 1,
12479 });
12480 }
12481 }
12482
12483 self.unified_registry_mut().refresh_availability();
12484 Ok(installed)
12485 }
12486
12487 pub fn speech_health(&self) -> SpeechHealthReport {
12489 let local_stt_default =
12490 self.speech_health_default_name(ModelCapability::SpeechToText, true, false);
12491 let local_tts_default =
12492 self.speech_health_default_name(ModelCapability::TextToSpeech, true, false);
12493 let remote_stt_default =
12494 self.speech_health_default_name(ModelCapability::SpeechToText, false, true);
12495 let remote_tts_default =
12496 self.speech_health_default_name(ModelCapability::TextToSpeech, false, true);
12497
12498 let mut local_models = Vec::new();
12499 let mut remote_models = Vec::new();
12500 for schema in self.list_schemas() {
12501 let capability = if schema.has_capability(ModelCapability::SpeechToText) {
12502 Some(ModelCapability::SpeechToText)
12503 } else if schema.has_capability(ModelCapability::TextToSpeech) {
12504 Some(ModelCapability::TextToSpeech)
12505 } else {
12506 None
12507 };
12508 let Some(capability) = capability else {
12509 continue;
12510 };
12511
12512 let selected_by_default = local_stt_default
12513 .as_ref()
12514 .is_some_and(|name| name == &schema.name)
12515 || local_tts_default
12516 .as_ref()
12517 .is_some_and(|name| name == &schema.name)
12518 || remote_stt_default
12519 .as_ref()
12520 .is_some_and(|name| name == &schema.name)
12521 || remote_tts_default
12522 .as_ref()
12523 .is_some_and(|name| name == &schema.name);
12524
12525 let health = SpeechModelHealth {
12526 id: schema.id.clone(),
12527 name: schema.name.clone(),
12528 provider: schema.provider.clone(),
12529 capability,
12530 is_local: schema.is_local(),
12531 available: schema.available,
12532 cached: speech_model_cached(&schema),
12533 selected_by_default,
12534 source: speech_model_source_label(&schema),
12535 };
12536 if schema.is_local() {
12537 local_models.push(health);
12538 } else {
12539 remote_models.push(health);
12540 }
12541 }
12542
12543 let runtime = {
12550 let rt =
12551 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
12552 SpeechRuntimeHealth {
12553 root: rt.root.clone(),
12554 installed: rt.is_ready(),
12555 python: rt.python.clone(),
12556 stt_command: rt.stt_program.clone(),
12557 tts_command: rt.tts_program.clone(),
12558 configured_python: std::env::var("CAR_SPEECH_PYTHON")
12559 .ok()
12560 .filter(|value| !value.trim().is_empty()),
12561 detected_python: detect_speech_python(),
12562 }
12563 };
12564
12565 SpeechHealthReport {
12566 runtime,
12567 local_models,
12568 remote_models,
12569 elevenlabs_configured: crate::tasks::transcribe::provider_configured_for_passive_status(
12573 "ELEVENLABS_API_KEY",
12574 )
12575 || crate::tasks::synthesize::provider_configured_for_passive_status(
12576 "ELEVENLABS_API_KEY",
12577 ),
12578 prefer_local: self.speech_policy.prefer_local,
12579 allow_remote_fallback: self.speech_policy.allow_remote_fallback,
12580 preferred_local_stt: self.speech_policy.preferred_local_stt.clone(),
12581 preferred_local_tts: self.speech_policy.preferred_local_tts.clone(),
12582 preferred_remote_stt: self.speech_policy.preferred_remote_stt.clone(),
12583 preferred_remote_tts: self.speech_policy.preferred_remote_tts.clone(),
12584 local_stt_default,
12585 local_tts_default,
12586 remote_stt_default,
12587 remote_tts_default,
12588 }
12589 }
12590
12591 pub async fn model_health(&self) -> ModelHealthReport {
12594 self.model_health_with(false).await
12595 }
12596
12597 pub async fn model_health_for_explicit_status(&self) -> ModelHealthReport {
12607 self.model_health_with(true).await
12608 }
12609
12610 async fn model_health_with(&self, authoritative_credentials: bool) -> ModelHealthReport {
12611 let schemas: Vec<ModelSchema> = self
12612 .catalog_registry_snapshot_with(authoritative_credentials)
12613 .list()
12614 .into_iter()
12615 .cloned()
12616 .collect();
12617 let total_models = schemas.len();
12618 let available_models = schemas
12619 .iter()
12620 .filter(|schema| schema.available_now())
12621 .count();
12622 let local_models = schemas.iter().filter(|schema| schema.is_local()).count();
12623 let remote_models = total_models.saturating_sub(local_models);
12624
12625 let defaults = vec![
12626 self.model_default_health(
12627 ModelCapability::Generate,
12628 self.preferred_model_for_capability(ModelCapability::Generate)
12629 .unwrap_or(&self.config.generation_model),
12630 ),
12631 self.model_default_health(
12632 ModelCapability::Embed,
12633 self.preferred_model_for_capability(ModelCapability::Embed)
12634 .unwrap_or(&self.config.embedding_model),
12635 ),
12636 self.model_default_health(
12637 ModelCapability::Classify,
12638 self.preferred_model_for_capability(ModelCapability::Classify)
12639 .unwrap_or(&self.config.classification_model),
12640 ),
12641 ];
12642
12643 let credential_memo: std::sync::Mutex<std::collections::HashMap<String, bool>> =
12646 std::sync::Mutex::new(std::collections::HashMap::new());
12647 let credential = |env_var: &str| -> bool {
12648 if !authoritative_credentials {
12656 return std::env::var(env_var).is_ok_and(|value| !value.trim().is_empty());
12657 }
12658 let mut memo = credential_memo
12659 .lock()
12660 .unwrap_or_else(|poisoned| poisoned.into_inner());
12661 if let Some(known) = memo.get(env_var) {
12662 return *known;
12663 }
12664 let present = crate::registry::credential_present_without_reading_it(env_var);
12665 memo.insert(env_var.to_string(), present);
12666 present
12667 };
12668
12669 let mut providers = std::collections::BTreeMap::new();
12670 for schema in &schemas {
12671 let entry =
12672 providers
12673 .entry(schema.provider.clone())
12674 .or_insert_with(|| ProviderAccumulator {
12675 configured: false,
12676 local_models: 0,
12677 remote_models: 0,
12678 available_models: 0,
12679 capabilities: std::collections::HashSet::new(),
12680 });
12681
12682 entry.configured |= model_source_configured(schema, &credential);
12683 if schema.is_local() {
12684 entry.local_models += 1;
12685 } else {
12686 entry.remote_models += 1;
12687 }
12688 if schema.available_now() {
12689 entry.available_models += 1;
12690 }
12691 for capability in &schema.capabilities {
12692 entry.capabilities.insert(*capability);
12693 }
12694 }
12695
12696 let providers = providers
12697 .into_iter()
12698 .map(|(provider, acc)| ModelProviderHealth {
12699 provider,
12700 configured: acc.configured,
12701 local_models: acc.local_models,
12702 remote_models: acc.remote_models,
12703 available_models: acc.available_models,
12704 capabilities: sort_capabilities(acc.capabilities.into_iter().collect()),
12705 })
12706 .collect();
12707
12708 let capabilities = all_model_capabilities()
12709 .into_iter()
12710 .map(|capability| {
12711 let relevant: Vec<&ModelSchema> = schemas
12712 .iter()
12713 .filter(|schema| schema.has_capability(capability))
12714 .collect();
12715 let available: Vec<&ModelSchema> = relevant
12716 .iter()
12717 .copied()
12718 .filter(|schema| schema.available_now())
12719 .collect();
12720 ModelCapabilityHealth {
12721 capability,
12722 total_models: relevant.len(),
12723 available_models: available.len(),
12724 local_available_models: available
12725 .iter()
12726 .filter(|schema| schema.is_local())
12727 .count(),
12728 remote_available_models: available
12729 .iter()
12730 .filter(|schema| !schema.is_local())
12731 .count(),
12732 }
12733 })
12734 .collect();
12735
12736 let routing = self.routing_scenarios().await;
12737 let routing_config = self.adaptive_router.config().clone();
12738 let benchmark_priors =
12739 load_benchmark_prior_health(&self.config.state_models_dir(), &schemas);
12740
12741 ModelHealthReport {
12742 total_models,
12743 available_models,
12744 local_models,
12745 remote_models,
12746 defaults,
12747 providers,
12748 capabilities,
12749 routing_prefer_local: routing_config.prefer_local,
12750 routing_quality_first_cold_start: routing_config.quality_first_cold_start,
12751 routing_min_observations: routing_config.min_observations,
12752 routing_bootstrap_min_task_observations: routing_config.bootstrap_min_task_observations,
12753 routing_bootstrap_quality_floor: routing_config.bootstrap_quality_floor,
12754 routing_quality_weight: routing_config.quality_weight,
12755 routing_latency_weight: routing_config.latency_weight,
12756 routing_cost_weight: routing_config.cost_weight,
12757 routing_scenarios: routing,
12758 benchmark_priors,
12759 speech: self.speech_health(),
12760 }
12761 }
12762
12763 async fn routing_scenarios(&self) -> Vec<RoutingScenarioHealth> {
12764 let tracker = self.outcome_tracker.read().await;
12765 let config = self.adaptive_router.config().clone();
12766 let scenarios = [
12767 (
12768 "interactive_text",
12769 "Summarize the benefits of local-first AI routing in two sentences.",
12770 "text",
12771 RoutingWorkload::Interactive,
12772 false,
12773 false,
12774 ),
12775 (
12776 "background_code",
12777 "Write a Python function named fibonacci(n) that returns the nth Fibonacci number.",
12778 "code",
12779 RoutingWorkload::Background,
12780 false,
12781 false,
12782 ),
12783 (
12784 "interactive_tool_use",
12785 "Use the provided weather tool to get the weather for Boston.",
12786 "tool_use",
12787 RoutingWorkload::Interactive,
12788 true,
12789 false,
12790 ),
12791 (
12792 "interactive_vision",
12793 "What is in this image? Answer in one word.",
12794 "vision",
12795 RoutingWorkload::Interactive,
12796 false,
12797 true,
12798 ),
12799 ];
12800
12801 let routing_registry = self.catalog_registry_snapshot();
12808
12809 scenarios
12810 .into_iter()
12811 .map(
12812 |(name, prompt, task_family, workload, has_tools, has_vision)| {
12813 let decision = self.adaptive_router.route_context_aware(
12814 prompt,
12815 0,
12816 &routing_registry,
12817 &tracker,
12818 has_tools,
12819 has_vision,
12820 workload,
12821 );
12822 let quality_first_cold_start = if has_tools || has_vision {
12823 config.quality_first_cold_start
12824 } else if task_family == "code"
12825 && matches!(workload, RoutingWorkload::Background)
12826 {
12827 false
12828 } else {
12829 config.quality_first_cold_start
12830 };
12831 RoutingScenarioHealth {
12832 name: name.to_string(),
12833 task_family: task_family.to_string(),
12834 workload,
12835 has_tools,
12836 has_vision,
12837 prefer_local: if task_family == "speech" {
12838 self.speech_policy.prefer_local
12839 } else {
12840 config.prefer_local
12841 },
12842 quality_first_cold_start,
12843 bootstrap_min_task_observations: config.bootstrap_min_task_observations,
12844 bootstrap_quality_floor: config.bootstrap_quality_floor,
12845 model_id: decision.model_id,
12846 model_name: decision.model_name,
12847 reason: decision.reason,
12848 strategy: decision.strategy,
12849 }
12850 },
12851 )
12852 .collect()
12853 }
12854
12855 pub async fn smoke_test_speech(
12857 &self,
12858 local: bool,
12859 remote: bool,
12860 ) -> Result<SpeechSmokeReport, InferenceError> {
12861 let mut report = SpeechSmokeReport::default();
12862
12863 if local {
12864 let tts = self
12865 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
12866 .ok_or_else(|| {
12867 InferenceError::InferenceFailed(
12868 "no local text-to-speech model available".into(),
12869 )
12870 })?;
12871 let stt = self
12872 .preferred_speech_schema(ModelCapability::SpeechToText, true, false)
12873 .ok_or_else(|| {
12874 InferenceError::InferenceFailed(
12875 "no local speech-to-text model available".into(),
12876 )
12877 })?;
12878 report.local = Some(
12879 self.run_speech_smoke_path("local", &tts, &stt, "Testing CAR local speech path.")
12880 .await?,
12881 );
12882 } else {
12883 report.skipped.push("local".to_string());
12884 }
12885
12886 if remote {
12887 let tts = self
12888 .preferred_speech_schema(ModelCapability::TextToSpeech, false, true)
12889 .ok_or_else(|| {
12890 InferenceError::InferenceFailed(
12891 "no remote text-to-speech model available".into(),
12892 )
12893 })?;
12894 let stt = self
12895 .preferred_speech_schema(ModelCapability::SpeechToText, false, true)
12896 .ok_or_else(|| {
12897 InferenceError::InferenceFailed(
12898 "no remote speech-to-text model available".into(),
12899 )
12900 })?;
12901 report.remote = Some(
12902 self.run_speech_smoke_path("remote", &tts, &stt, "Testing CAR remote speech path.")
12903 .await?,
12904 );
12905 } else {
12906 report.skipped.push("remote".to_string());
12907 }
12908
12909 Ok(report)
12910 }
12911
12912 fn speech_candidates(
12913 &self,
12914 capability: ModelCapability,
12915 explicit: Option<&str>,
12916 ) -> Result<Vec<ModelSchema>, InferenceError> {
12917 if let Some(model) = explicit {
12918 let schema = self
12919 .resolve_schema(model)
12920 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
12921 if !schema.has_capability(capability) {
12922 return Err(InferenceError::InferenceFailed(format!(
12923 "model {} does not support {:?}",
12924 schema.name, capability
12925 )));
12926 }
12927 return Ok(vec![schema]);
12928 }
12929
12930 let mut candidates: Vec<ModelSchema> = self
12931 .unified_registry()
12932 .query(&ModelFilter {
12933 capabilities: vec![capability],
12934 ..Default::default()
12935 })
12936 .into_iter()
12937 .cloned()
12938 .collect();
12939
12940 if candidates.is_empty() {
12941 return Err(InferenceError::InferenceFailed(format!(
12942 "no models registered for capability {:?}",
12943 capability
12944 )));
12945 }
12946
12947 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
12948 if !self.speech_policy.allow_remote_fallback
12949 && candidates.iter().any(|model| model.is_local())
12950 {
12951 candidates.retain(|model| model.is_local());
12952 }
12953
12954 Ok(candidates)
12955 }
12956
12957 #[allow(dead_code)] fn resolve_external_hf_repo(
12963 &self,
12964 explicit: Option<&str>,
12965 capability: ModelCapability,
12966 ) -> Option<String> {
12967 let id = explicit?;
12968 let schema = self.resolve_schema(id)?;
12969 if !schema.has_capability(capability) {
12970 return Some(id.to_string());
12971 }
12972 if let ModelSource::Mlx { hf_repo, .. } = &schema.source {
12973 return Some(hf_repo.clone());
12974 }
12975 Some(id.to_string())
12976 }
12977
12978 fn media_generation_candidates(
12979 &self,
12980 capability: ModelCapability,
12981 explicit: Option<&str>,
12982 ) -> Result<Vec<ModelSchema>, InferenceError> {
12983 if let Some(model) = explicit {
12984 let schema = self
12985 .resolve_schema(model)
12986 .ok_or_else(|| InferenceError::ModelNotFound(model.to_string()))?;
12987 if !schema.has_capability(capability) {
12988 return Err(InferenceError::InferenceFailed(format!(
12989 "model {} does not support {:?}",
12990 schema.name, capability
12991 )));
12992 }
12993 return Ok(vec![schema]);
12994 }
12995
12996 let mut candidates: Vec<ModelSchema> = self
12997 .unified_registry()
12998 .query(&ModelFilter {
12999 capabilities: vec![capability],
13000 local_only: true,
13001 ..Default::default()
13002 })
13003 .into_iter()
13004 .cloned()
13005 .collect();
13006 candidates.sort_by_key(|schema| (!schema.available, schema.size_mb()));
13007 if candidates.is_empty() {
13008 return Err(InferenceError::InferenceFailed(format!(
13009 "no models registered for capability {:?}",
13010 capability
13011 )));
13012 }
13013 Ok(candidates)
13014 }
13015
13016 fn preferred_speech_schema(
13017 &self,
13018 capability: ModelCapability,
13019 local_only: bool,
13020 remote_only: bool,
13021 ) -> Option<ModelSchema> {
13022 let available_only = remote_only;
13023 let mut candidates: Vec<ModelSchema> = self
13024 .unified_registry()
13025 .query(&ModelFilter {
13026 capabilities: vec![capability],
13027 available_only,
13028 ..Default::default()
13029 })
13030 .into_iter()
13031 .filter(|schema| {
13032 (!local_only || schema.is_local()) && (!remote_only || schema.is_remote())
13033 })
13034 .cloned()
13035 .collect();
13036 candidates.sort_by_key(|model| self.speech_sort_key(capability, model));
13037 candidates.into_iter().next()
13038 }
13039
13040 fn speech_health_default_name(
13041 &self,
13042 capability: ModelCapability,
13043 local_only: bool,
13044 remote_only: bool,
13045 ) -> Option<String> {
13046 let preferred = match capability {
13047 ModelCapability::SpeechToText if local_only => {
13048 self.speech_policy.preferred_local_stt.as_ref()
13049 }
13050 ModelCapability::SpeechToText if remote_only => {
13051 self.speech_policy.preferred_remote_stt.as_ref()
13052 }
13053 ModelCapability::TextToSpeech if local_only => {
13054 self.speech_policy.preferred_local_tts.as_ref()
13055 }
13056 ModelCapability::TextToSpeech if remote_only => {
13057 self.speech_policy.preferred_remote_tts.as_ref()
13058 }
13059 _ => None,
13060 };
13061
13062 preferred
13063 .filter(|name| {
13064 self.unified_registry().list().iter().any(|schema| {
13065 schema.name == **name
13066 && schema.has_capability(capability)
13067 && (!local_only || schema.is_local())
13068 && (!remote_only || schema.is_remote())
13069 })
13070 })
13071 .cloned()
13072 .or_else(|| {
13073 self.preferred_speech_schema(capability, local_only, remote_only)
13074 .map(|schema| schema.name)
13075 })
13076 }
13077
13078 fn model_default_health(
13079 &self,
13080 capability: ModelCapability,
13081 configured_model: &str,
13082 ) -> ModelDefaultHealth {
13083 let schema = self.resolve_schema_name_first(configured_model);
13084
13085 ModelDefaultHealth {
13086 capability,
13087 configured_model: configured_model.to_string(),
13088 available: schema.as_ref().is_some_and(ModelSchema::available_now),
13089 is_local: schema.as_ref().is_some_and(ModelSchema::is_local),
13090 provider: schema.map(|model| model.provider),
13091 }
13092 }
13093
13094 fn speech_sort_key(
13095 &self,
13096 capability: ModelCapability,
13097 model: &ModelSchema,
13098 ) -> (u8, u8, u8, u8, u64, u64) {
13099 let policy_preference = match capability {
13100 ModelCapability::SpeechToText if model.is_local() => {
13101 self.speech_policy.preferred_local_stt.as_ref()
13102 }
13103 ModelCapability::SpeechToText => self.speech_policy.preferred_remote_stt.as_ref(),
13104 ModelCapability::TextToSpeech if model.is_local() => {
13105 self.speech_policy.preferred_local_tts.as_ref()
13106 }
13107 ModelCapability::TextToSpeech => self.speech_policy.preferred_remote_tts.as_ref(),
13108 _ => None,
13109 };
13110 let local_rank = if self.speech_policy.prefer_local {
13111 if model.is_local() {
13112 0
13113 } else {
13114 1
13115 }
13116 } else if model.is_remote() {
13117 0
13118 } else {
13119 1
13120 };
13121 let availability_rank = if model.available {
13122 0
13123 } else if model.is_local() {
13124 1
13125 } else {
13126 2
13127 };
13128 let policy_rank: u8 = if policy_preference.is_some_and(|preferred| preferred == &model.name)
13129 {
13130 0
13131 } else {
13132 1
13133 };
13134 let speech_rank = match capability {
13135 ModelCapability::TextToSpeech => {
13151 if model.name == "Kokoro-82M-bf16" {
13152 0
13153 } else if model.name == "Kokoro-82M-6bit" {
13154 1
13155 } else if model.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
13156 3
13159 } else {
13160 2
13161 }
13162 }
13163 ModelCapability::SpeechToText => {
13164 if model.name == "Parakeet-TDT-0.6B-v3-MLX" {
13165 0
13166 } else {
13167 1
13168 }
13169 }
13170 _ => 0,
13171 };
13172 let latency_rank = model.performance.latency_p50_ms.unwrap_or(u64::MAX);
13173 let size_rank = model.cost.size_mb.unwrap_or(u64::MAX);
13174 (
13175 local_rank,
13176 availability_rank,
13177 policy_rank,
13178 speech_rank,
13179 latency_rank,
13180 size_rank,
13181 )
13182 }
13183
13184 async fn run_speech_smoke_path(
13185 &self,
13186 path: &str,
13187 tts: &ModelSchema,
13188 stt: &ModelSchema,
13189 text: &str,
13190 ) -> Result<SpeechSmokePathReport, InferenceError> {
13191 let work_dir = temp_work_dir(&format!("speech-smoke-{path}"))?;
13192 let audio_path = work_dir.join(format!("{path}.wav"));
13193 let synth = self
13194 .synthesize(SynthesizeRequest {
13195 text: text.to_string(),
13196 model: Some(tts.name.clone()),
13197 voice: default_speech_voice(tts),
13198 language: Some("en".to_string()),
13199 output_path: Some(audio_path.display().to_string()),
13200 ..SynthesizeRequest::default()
13201 })
13202 .await?;
13203 let transcript = self
13204 .transcribe(TranscribeRequest {
13205 audio_path: synth.audio_path.clone(),
13206 model: Some(stt.name.clone()),
13207 language: Some("en".to_string()),
13208 prompt: None,
13209 timestamps: false,
13210 })
13211 .await?;
13212
13213 Ok(SpeechSmokePathReport {
13214 path: path.to_string(),
13215 tts_model: synth.model_used.unwrap_or_else(|| tts.name.clone()),
13216 stt_model: transcript.model_used.unwrap_or_else(|| stt.name.clone()),
13217 audio_path: PathBuf::from(synth.audio_path),
13218 transcript: transcript.text,
13219 })
13220 }
13221
13222 async fn ensure_speech_runtime(&self) -> Result<SpeechRuntime, InferenceError> {
13223 let mut guard = self.speech_runtime.lock().await;
13224 if let Some(runtime) = guard.as_ref() {
13225 if runtime.is_ready() {
13226 return Ok(runtime.clone());
13227 }
13228 }
13229
13230 let runtime =
13231 SpeechRuntime::new(speech_runtime_root_from_models_dir(&self.config.models_dir));
13232 if !runtime.is_ready() {
13233 bootstrap_speech_runtime(&runtime).await?;
13234 }
13235 if !runtime.is_ready() {
13236 return Err(InferenceError::InferenceFailed(format!(
13237 "managed speech runtime is not ready at {}",
13238 runtime.root.display()
13239 )));
13240 }
13241
13242 *guard = Some(runtime.clone());
13243 Ok(runtime)
13244 }
13245
13246 async fn transcribe_whisper(
13255 &self,
13256 schema: &ModelSchema,
13257 model: &str,
13258 req: &TranscribeRequest,
13259 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13260 ) -> Result<TranscribeResult, InferenceError> {
13261 let model = model.to_string();
13262 let model_path = car_whisper::ensure_model(&model)
13263 .map_err(|e| InferenceError::InferenceFailed(format!("whisper download: {e}")))?;
13264 let measured_bytes = backend_cache::estimate_model_size(&model_path);
13265 let reservation = reservation.ok_or_else(|| {
13266 InferenceError::InferenceFailed("local Whisper path missing admission".into())
13267 })?;
13268 reservation
13269 .reconcile_measured_weights(measured_bytes)
13270 .map_err(InferenceError::from)?;
13271 let detached_lease = reservation.detached_lease();
13272 let language = req.language.clone().unwrap_or_else(|| "auto".to_string());
13274 let audio_path = std::path::PathBuf::from(&req.audio_path);
13275 let name = schema.name.clone();
13276 let req_language = req.language.clone();
13277 let text =
13278 run_admitted_blocking(detached_lease, move || -> Result<String, InferenceError> {
13279 let stt = car_whisper::WhisperStt::load_from_path(&model_path, &language)
13281 .map_err(|e| InferenceError::InferenceFailed(format!("whisper load: {e}")))?;
13282 stt.transcribe_file(&audio_path).map_err(|e| {
13283 InferenceError::InferenceFailed(format!("whisper transcribe: {e}"))
13284 })
13285 })
13286 .await
13287 .map_err(|e| InferenceError::InferenceFailed(format!("whisper join: {e}")))??;
13288 Ok(TranscribeResult::text_only(text, Some(name), req_language))
13289 }
13290
13291 async fn transcribe_local_mlx(
13292 &self,
13293 schema: &ModelSchema,
13294 req: &TranscribeRequest,
13295 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13296 ) -> Result<TranscribeResult, InferenceError> {
13297 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13298 let reservation = reservation.ok_or_else(|| {
13299 InferenceError::InferenceFailed("local STT path missing admission".into())
13300 })?;
13301 reservation
13302 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13303 .map_err(InferenceError::from)?;
13304 self.transcribe_via_speech_runtime(schema, req, reservation.detached_lease())
13313 .await
13314 }
13315
13316 async fn transcribe_via_speech_runtime(
13323 &self,
13324 schema: &ModelSchema,
13325 req: &TranscribeRequest,
13326 detached_lease: resource_policy::DetachedLocalLease,
13327 ) -> Result<TranscribeResult, InferenceError> {
13328 {
13329 let runtime = self.ensure_speech_runtime().await?;
13330 let hf_repo = match &schema.source {
13331 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13332 _ => {
13333 return Err(InferenceError::InferenceFailed(format!(
13334 "speech runtime needs an MLX model repo; {} is not one",
13335 schema.id
13336 )))
13337 }
13338 };
13339 let output_dir = temp_work_dir("stt")?;
13340 let output_prefix = output_dir.join("transcript");
13341 let mut args = vec![
13342 "--model".to_string(),
13343 hf_repo,
13344 "--audio".to_string(),
13345 req.audio_path.clone(),
13346 "--output-path".to_string(),
13347 output_prefix.display().to_string(),
13348 "--format".to_string(),
13349 "json".to_string(),
13350 ];
13351 if let Some(language) = &req.language {
13352 args.push("--language".to_string());
13353 args.push(normalize_lang_code(language));
13354 }
13355 if let Some(prompt) = &req.prompt {
13356 args.push("--context".to_string());
13357 args.push(prompt.clone());
13358 }
13359 if req.timestamps {
13360 args.push("--verbose".to_string());
13361 }
13362
13363 let output =
13364 run_mlx_audio_command(&runtime, "stt.generate", &args, detached_lease).await?;
13365 let text = read_transcription_result(&output_prefix)?
13366 .or_else(|| extract_text_from_payload(&output.stdout))
13367 .ok_or_else(|| {
13368 InferenceError::InferenceFailed(format!(
13369 "mlx-audio transcription returned no text: {}",
13370 output.stderr
13371 ))
13372 })?;
13373
13374 Ok(TranscribeResult {
13375 text,
13376 model_used: Some(schema.name.clone()),
13377 language: req.language.clone(),
13378 words: Vec::new(),
13379 routing_explanation: None,
13380 })
13381 }
13382 }
13383
13384 async fn synthesize_local_mlx(
13385 &self,
13386 schema: &ModelSchema,
13387 req: &SynthesizeRequest,
13388 reservation: Option<&mut resource_policy::LocalLoadReservation>,
13389 ) -> Result<SynthesizeResult, InferenceError> {
13390 let requested = req.requested_advanced_controls();
13395 let repo_supports_advanced = match &schema.source {
13396 ModelSource::Mlx { hf_repo, .. } => hf_repo.to_ascii_lowercase().contains("qwen3-tts"),
13397 _ => false,
13398 };
13399 if !requested.is_empty() && !repo_supports_advanced {
13400 if req.strict_capabilities {
13401 return Err(InferenceError::InferenceFailed(format!(
13402 "model {name} does not support Qwen3-TTS advanced controls {requested:?}; \
13403 route to a Qwen3-TTS model or set strict_capabilities = false to degrade",
13404 name = schema.name,
13405 )));
13406 }
13407 tracing::warn!(
13408 model = %schema.name,
13409 fields = ?requested,
13410 "Qwen3-TTS advanced controls set on non-Qwen3-TTS backend — ignored \
13411 (set strict_capabilities=true to error instead)"
13412 );
13413 }
13414
13415 let model_dir = self.unified_registry().ensure_local(&schema.id).await?;
13416 let reservation = reservation.ok_or_else(|| {
13417 InferenceError::InferenceFailed("local TTS path missing admission".into())
13418 })?;
13419 reservation
13420 .reconcile_measured_weights(backend_cache::estimate_model_size(&model_dir))
13421 .map_err(InferenceError::from)?;
13422
13423 self.synthesize_via_speech_runtime(schema, req, reservation.detached_lease())
13433 .await
13434 }
13435
13436 async fn synthesize_via_speech_runtime(
13449 &self,
13450 schema: &ModelSchema,
13451 req: &SynthesizeRequest,
13452 detached_lease: resource_policy::DetachedLocalLease,
13453 ) -> Result<SynthesizeResult, InferenceError> {
13454 let runtime = self.ensure_speech_runtime().await?;
13455 let primary_hf_repo = match &schema.source {
13456 ModelSource::Mlx { hf_repo, .. } => hf_repo.clone(),
13457 _ => {
13458 return Err(InferenceError::InferenceFailed(format!(
13459 "speech runtime needs an MLX model repo; {} is not one",
13460 schema.id
13461 )))
13462 }
13463 };
13464 let (produced, model_used) = match self
13465 .synthesize_local_mlx_repo(
13466 &runtime,
13467 &primary_hf_repo,
13468 schema.name.as_str(),
13469 req,
13470 detached_lease.clone(),
13471 )
13472 .await
13473 {
13474 Ok(result) => result,
13475 Err(primary_err)
13476 if primary_hf_repo == "mlx-community/Kokoro-82M-6bit"
13477 && kokoro_runtime_fallback_enabled() =>
13478 {
13479 let fallback_repo = "mlx-community/Kokoro-82M-bf16";
13480 let fallback_name = "Kokoro-82M-bf16";
13481 match self
13482 .synthesize_local_mlx_repo(
13483 &runtime,
13484 fallback_repo,
13485 fallback_name,
13486 req,
13487 detached_lease.clone(),
13488 )
13489 .await
13490 {
13491 Ok(result) => result,
13492 Err(fallback_err) => {
13493 return Err(InferenceError::InferenceFailed(format!(
13494 "{primary_err}; fallback {fallback_name} also failed: {fallback_err}"
13495 )));
13496 }
13497 }
13498 }
13499 Err(err) => return Err(err),
13500 };
13501 let final_path =
13502 materialize_audio_output(&produced, req.output_path.as_deref(), &req.format)?;
13503
13504 Ok(SynthesizeResult {
13505 audio_path: final_path.display().to_string(),
13506 media_type: media_type_for_format(&req.format),
13507 model_used: Some(model_used),
13508 voice_used: req.voice.clone(),
13509 routing_explanation: None,
13510 })
13511 }
13512
13513 async fn synthesize_local_mlx_repo(
13514 &self,
13515 runtime: &SpeechRuntime,
13516 hf_repo: &str,
13517 model_name: &str,
13518 req: &SynthesizeRequest,
13519 detached_lease: resource_policy::DetachedLocalLease,
13520 ) -> Result<(PathBuf, String), InferenceError> {
13521 let output_dir = temp_work_dir("tts")?;
13522 let mut args = vec![
13523 "--model".to_string(),
13524 hf_repo.to_string(),
13525 "--text".to_string(),
13526 req.text.clone(),
13527 "--output_path".to_string(),
13528 output_dir.display().to_string(),
13529 ];
13530 if let Some(voice) = &req.voice {
13531 args.push("--voice".to_string());
13532 args.push(voice.clone());
13533 }
13534 if let Some(speed) = req.speed {
13535 args.push("--speed".to_string());
13536 args.push(speed.to_string());
13537 }
13538 let repo_lower = hf_repo.to_ascii_lowercase();
13539 if repo_lower.contains("kokoro") {
13540 args.push("--lang_code".to_string());
13541 args.push(kokoro_lang_code(req.language.as_deref()).to_string());
13542 } else if let Some(language) = &req.language {
13543 args.push("--lang_code".to_string());
13544 args.push(normalize_lang_code(language));
13545 }
13546
13547 if repo_lower.contains("qwen3-tts") {
13553 if let Some(ref_audio) = &req.reference_audio_path {
13554 args.push("--ref_audio".to_string());
13555 args.push(ref_audio.clone());
13556 }
13557 if let Some(ref_text) = &req.reference_text {
13558 args.push("--ref_text".to_string());
13559 args.push(ref_text.clone());
13560 }
13561 if let Some(instruct) = &req.voice_instruction {
13562 args.push("--instruct".to_string());
13563 args.push(instruct.clone());
13564 }
13565 }
13566
13567 let output = if repo_lower.contains("kokoro") {
13568 let device = std::env::var("CAR_SPEECH_KOKORO_DEVICE")
13569 .or_else(|_| std::env::var("CAR_SPEECH_MLX_DEVICE"))
13570 .unwrap_or_else(|_| "cpu".to_string());
13571 let extra_env = vec![
13572 ("MLX_DEVICE".to_string(), device),
13574 ("PYTORCH_ENABLE_MPS_FALLBACK".to_string(), "1".to_string()),
13576 ];
13577 run_mlx_audio_command_with_env(
13578 runtime,
13579 "tts.generate",
13580 &args,
13581 &extra_env,
13582 detached_lease,
13583 )
13584 .await?
13585 } else {
13586 run_mlx_audio_command(runtime, "tts.generate", &args, detached_lease).await?
13587 };
13588 let produced = find_audio_file(&output_dir)?.ok_or_else(|| {
13589 let hint = if repo_lower.contains("kokoro") {
13590 ". Kokoro models may crash on GPU — try CAR_SPEECH_KOKORO_DEVICE=cpu or use the default Qwen3-TTS model"
13591 } else {
13592 ""
13593 };
13594 InferenceError::InferenceFailed(format!(
13595 "mlx-audio synthesis produced no audio file: {}{}",
13596 output.stderr, hint
13597 ))
13598 })?;
13599 Ok((produced, model_name.to_string()))
13600 }
13601
13602 async fn transcribe_elevenlabs(
13603 &self,
13604 schema: &ModelSchema,
13605 req: &TranscribeRequest,
13606 ) -> Result<TranscribeResult, InferenceError> {
13607 let (endpoint, api_key) = elevenlabs_auth(
13608 schema,
13609 crate::tasks::transcribe::resolve_provider_credential_for_request,
13610 )?;
13611 let file_name = Path::new(&req.audio_path)
13612 .file_name()
13613 .and_then(|f| f.to_str())
13614 .unwrap_or("audio.wav")
13615 .to_string();
13616 let audio_bytes = tokio::fs::read(&req.audio_path).await?;
13617 let file_part = Part::bytes(audio_bytes).file_name(file_name);
13618 let mut form = Form::new()
13619 .text("model_id", schema.name.clone())
13620 .part("file", file_part);
13621 if let Some(language) = &req.language {
13622 form = form.text("language_code", language.clone());
13623 }
13624
13625 let resp = self
13626 .remote_backend
13627 .client
13628 .post(format!(
13629 "{}/v1/speech-to-text",
13630 endpoint.trim_end_matches('/')
13631 ))
13632 .header("xi-api-key", api_key)
13633 .multipart(form)
13634 .send()
13635 .await
13636 .map_err(|e| {
13637 self.remote_backend
13638 .request_error("ElevenLabs STT request failed", &e)
13639 })?;
13640 let status = resp.status();
13641 let body = resp.text().await.map_err(|e| {
13642 InferenceError::InferenceFailed(format!("read ElevenLabs STT body: {e}"))
13643 })?;
13644 if !status.is_success() {
13645 return Err(InferenceError::InferenceFailed(format!(
13646 "ElevenLabs STT returned {status}: {body}"
13647 )));
13648 }
13649 let payload: serde_json::Value = serde_json::from_str(&body).map_err(|e| {
13650 InferenceError::InferenceFailed(format!("parse ElevenLabs STT response: {e}"))
13651 })?;
13652 let text = payload
13653 .get("text")
13654 .and_then(|v| v.as_str())
13655 .map(str::to_string)
13656 .ok_or_else(|| {
13657 InferenceError::InferenceFailed("ElevenLabs STT response missing text".into())
13658 })?;
13659
13660 Ok(TranscribeResult {
13661 text,
13662 model_used: Some(schema.name.clone()),
13663 language: payload
13664 .get("language_code")
13665 .and_then(|v| v.as_str())
13666 .map(str::to_string),
13667 words: Vec::new(),
13668 routing_explanation: None,
13669 })
13670 }
13671
13672 async fn synthesize_elevenlabs(
13673 &self,
13674 schema: &ModelSchema,
13675 req: &SynthesizeRequest,
13676 ) -> Result<SynthesizeResult, InferenceError> {
13677 let requested = req.requested_advanced_controls();
13681 if !requested.is_empty() {
13682 if req.strict_capabilities {
13683 return Err(InferenceError::InferenceFailed(format!(
13684 "ElevenLabs backend does not support Qwen3-TTS advanced controls \
13685 {requested:?}; route to a Qwen3-TTS model or set strict_capabilities = false"
13686 )));
13687 }
13688 tracing::warn!(
13689 model = %schema.name,
13690 fields = ?requested,
13691 "Qwen3-TTS advanced controls ignored by ElevenLabs backend"
13692 );
13693 }
13694 let (endpoint, api_key) = elevenlabs_auth(
13695 schema,
13696 crate::tasks::synthesize::resolve_provider_credential_for_request,
13697 )?;
13698 let voice_id = req
13699 .voice
13700 .clone()
13701 .unwrap_or_else(|| "JBFqnCBsd6RMkjVDRZzb".to_string());
13702 let output_format = elevenlabs_output_format(&req.format);
13703 let url = format!(
13704 "{}/v1/text-to-speech/{}?output_format={}",
13705 endpoint.trim_end_matches('/'),
13706 voice_id,
13707 output_format
13708 );
13709
13710 let mut body = serde_json::json!({
13711 "text": req.text,
13712 "model_id": schema.name,
13713 });
13714 if let Some(language) = &req.language {
13715 body["language_code"] = serde_json::Value::String(language.clone());
13716 }
13717
13718 let resp = self
13719 .remote_backend
13720 .client
13721 .post(url)
13722 .header("xi-api-key", api_key)
13723 .header("Content-Type", "application/json")
13724 .json(&body)
13725 .send()
13726 .await
13727 .map_err(|e| {
13728 self.remote_backend
13729 .request_error("ElevenLabs TTS request failed", &e)
13730 })?;
13731 let status = resp.status();
13732 let audio = resp.bytes().await.map_err(|e| {
13733 InferenceError::InferenceFailed(format!("read ElevenLabs TTS body: {e}"))
13734 })?;
13735 if !status.is_success() {
13736 let err_body = String::from_utf8_lossy(&audio);
13737 return Err(InferenceError::InferenceFailed(format!(
13738 "ElevenLabs TTS returned {status}: {err_body}"
13739 )));
13740 }
13741
13742 let final_path = requested_or_temp_output(req.output_path.as_deref(), &req.format)?;
13743 ensure_parent_dir(&final_path)?;
13744 tokio::fs::write(&final_path, &audio).await?;
13745
13746 Ok(SynthesizeResult {
13747 audio_path: final_path.display().to_string(),
13748 media_type: media_type_for_format(&req.format),
13749 model_used: Some(schema.name.clone()),
13750 voice_used: Some(voice_id),
13751 routing_explanation: None,
13752 })
13753 }
13754}
13755
13756#[derive(Default)]
13757struct ProviderAccumulator {
13758 configured: bool,
13759 local_models: usize,
13760 remote_models: usize,
13761 available_models: usize,
13762 capabilities: std::collections::HashSet<ModelCapability>,
13763}
13764
13765struct CommandOutput {
13770 stdout: String,
13771 stderr: String,
13772}
13773
13774#[derive(Debug, Clone)]
13775struct SpeechRuntime {
13776 root: PathBuf,
13777 python: PathBuf,
13778 stt_program: PathBuf,
13779 tts_program: PathBuf,
13780}
13781
13782impl SpeechRuntime {
13783 fn new(root: PathBuf) -> Self {
13784 let python = managed_venv::interpreter(&root);
13785 let stt_program = managed_venv::venv_program(&root, "mlx_audio.stt.generate");
13786 let tts_program = managed_venv::venv_program(&root, "mlx_audio.tts.generate");
13787 Self {
13788 root,
13789 python,
13790 stt_program,
13791 tts_program,
13792 }
13793 }
13794
13795 fn is_ready(&self) -> bool {
13796 managed_venv::interpreter_healthy(&self.root)
13801 && self.stt_program.exists()
13802 && self.tts_program.exists()
13803 }
13804
13805 fn command_for(&self, subcommand: &str) -> Result<&Path, InferenceError> {
13806 match subcommand {
13807 "stt.generate" => Ok(&self.stt_program),
13808 "tts.generate" => Ok(&self.tts_program),
13809 _ => Err(InferenceError::InferenceFailed(format!(
13810 "unknown speech subcommand: {subcommand}"
13811 ))),
13812 }
13813 }
13814}
13815
13816async fn run_mlx_audio_command(
13817 runtime: &SpeechRuntime,
13818 subcommand: &str,
13819 args: &[String],
13820 detached_lease: resource_policy::DetachedLocalLease,
13821) -> Result<CommandOutput, InferenceError> {
13822 run_mlx_audio_command_with_env(runtime, subcommand, args, &[], detached_lease).await
13823}
13824
13825async fn run_admitted_blocking<F, T>(
13829 detached_lease: resource_policy::DetachedLocalLease,
13830 operation: F,
13831) -> Result<T, tokio::task::JoinError>
13832where
13833 F: FnOnce() -> T + Send + 'static,
13834 T: Send + 'static,
13835{
13836 tokio::task::spawn_blocking(move || {
13837 let _detached_lease = detached_lease;
13838 operation()
13839 })
13840 .await
13841}
13842
13843struct DetachedSpeechProcess {
13844 child: Option<tokio::process::Child>,
13845 lease: Option<resource_policy::DetachedLocalLease>,
13846}
13847
13848impl Drop for DetachedSpeechProcess {
13849 fn drop(&mut self) {
13850 let Some(mut child) = self.child.take() else {
13851 return;
13852 };
13853 let lease = self.lease.take();
13854 let _ = child.start_kill();
13855 if tokio::runtime::Handle::try_current().is_ok() {
13856 tokio::spawn(async move {
13857 let _ = child.wait().await;
13858 drop(lease);
13859 });
13860 } else {
13861 std::mem::forget((child, lease));
13864 }
13865 }
13866}
13867
13868async fn run_mlx_audio_command_with_env(
13869 runtime: &SpeechRuntime,
13870 subcommand: &str,
13871 args: &[String],
13872 envs: &[(String, String)],
13873 detached_lease: resource_policy::DetachedLocalLease,
13874) -> Result<CommandOutput, InferenceError> {
13875 let program = runtime.command_for(subcommand)?;
13876 let mut command = Command::new(program);
13877 command.args(args);
13878 crate::hf_cache::apply_to(&mut command);
13879 for (key, value) in envs {
13880 command.env(key, value);
13881 }
13882 command
13883 .stdout(std::process::Stdio::piped())
13884 .stderr(std::process::Stdio::piped())
13885 .kill_on_drop(true);
13886 let child = command
13887 .spawn()
13888 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
13889 let mut owned = DetachedSpeechProcess {
13890 child: Some(child),
13891 lease: Some(detached_lease),
13892 };
13893 let mut stdout = owned
13894 .child
13895 .as_mut()
13896 .expect("speech child owned")
13897 .stdout
13898 .take()
13899 .ok_or_else(|| {
13900 InferenceError::InferenceFailed(format!("{} stdout unavailable", program.display()))
13901 })?;
13902 let mut stderr = owned
13903 .child
13904 .as_mut()
13905 .expect("speech child owned")
13906 .stderr
13907 .take()
13908 .ok_or_else(|| {
13909 InferenceError::InferenceFailed(format!("{} stderr unavailable", program.display()))
13910 })?;
13911 let stdout_reader = tokio::spawn(async move {
13912 let mut bytes = Vec::new();
13913 stdout.read_to_end(&mut bytes).await.map(|_| bytes)
13914 });
13915 let stderr_reader = tokio::spawn(async move {
13916 let mut bytes = Vec::new();
13917 stderr.read_to_end(&mut bytes).await.map(|_| bytes)
13918 });
13919 let status = owned
13920 .child
13921 .as_mut()
13922 .expect("speech child owned")
13923 .wait()
13924 .await
13925 .map_err(|err| InferenceError::InferenceFailed(format!("{}: {err}", program.display())))?;
13926 owned.child.take();
13927 owned.lease.take();
13928 let stdout = stdout_reader
13929 .await
13930 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout join: {err}")))?
13931 .map_err(|err| InferenceError::InferenceFailed(format!("speech stdout: {err}")))?;
13932 let stderr = stderr_reader
13933 .await
13934 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr join: {err}")))?
13935 .map_err(|err| InferenceError::InferenceFailed(format!("speech stderr: {err}")))?;
13936
13937 if status.success() {
13938 Ok(CommandOutput {
13939 stdout: String::from_utf8_lossy(&stdout).to_string(),
13940 stderr: String::from_utf8_lossy(&stderr).to_string(),
13941 })
13942 } else {
13943 Err(InferenceError::InferenceFailed(format!(
13944 "{} exited with {}: {}",
13945 program.display(),
13946 status,
13947 String::from_utf8_lossy(&stderr)
13948 )))
13949 }
13950}
13951
13952async fn bootstrap_speech_runtime(runtime: &SpeechRuntime) -> Result<(), InferenceError> {
13953 let python = select_speech_python()?;
13954
13955 let outcome = managed_venv::ensure_venv(&runtime.root, &python)
13961 .await
13962 .map_err(|err| InferenceError::InferenceFailed(err.to_string()))?;
13963 if outcome == managed_venv::VenvOutcome::Recreated {
13964 tracing::warn!(
13965 root = %runtime.root.display(),
13966 "managed speech runtime had an unusable interpreter; rebuilt it \
13967 (installed packages were discarded and are being reinstalled)"
13968 );
13969 }
13970
13971 run_command(
13972 "uv",
13973 &[
13974 "pip".to_string(),
13975 "install".to_string(),
13976 "--python".to_string(),
13977 runtime.python.display().to_string(),
13978 speech_runtime_mlx_audio_spec(),
13979 "misaki[en]".to_string(),
13980 speech_runtime_spacy_model_spec(),
13981 ],
13982 )
13983 .await?;
13984
13985 Ok(())
13986}
13987
13988async fn run_command(program: &str, args: &[String]) -> Result<(), InferenceError> {
13989 let output = Command::new(program)
13990 .args(args)
13991 .output()
13992 .await
13993 .map_err(|err| InferenceError::InferenceFailed(format!("{program}: {err}")))?;
13994
13995 if output.status.success() {
13996 Ok(())
13997 } else {
13998 Err(InferenceError::InferenceFailed(format!(
13999 "{} exited with {}: {}",
14000 program,
14001 output.status,
14002 String::from_utf8_lossy(&output.stderr)
14003 )))
14004 }
14005}
14006
14007fn select_speech_python() -> Result<String, InferenceError> {
14008 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14009 if !path.trim().is_empty() {
14010 return Ok(path);
14011 }
14012 }
14013
14014 for candidate in ["python3.13", "python3.12", "python3.11"] {
14015 if command_in_path(candidate) {
14016 return Ok(candidate.to_string());
14017 }
14018 }
14019
14020 Ok(SPEECH_RUNTIME_FALLBACK_PYTHON.to_string())
14030}
14031
14032const SPEECH_RUNTIME_FALLBACK_PYTHON: &str = "3.12";
14038
14039fn detect_speech_python() -> Option<String> {
14040 if let Ok(path) = std::env::var("CAR_SPEECH_PYTHON") {
14041 if !path.trim().is_empty() {
14042 return Some(path);
14043 }
14044 }
14045
14046 ["python3.13", "python3.12", "python3.11"]
14047 .into_iter()
14048 .find(|candidate| command_in_path(candidate))
14049 .map(str::to_string)
14050}
14051
14052fn speech_runtime_root_from_models_dir(_models_dir: &Path) -> PathBuf {
14053 if let Ok(path) = std::env::var("CAR_SPEECH_RUNTIME_DIR") {
14054 if !path.trim().is_empty() {
14055 return PathBuf::from(path);
14056 }
14057 }
14058
14059 std::env::var_os("HOME")
14060 .or_else(|| std::env::var_os("USERPROFILE"))
14061 .map(PathBuf::from)
14062 .unwrap_or_else(|| PathBuf::from("."))
14063 .join(".car")
14064 .join("speech-runtime")
14065}
14066
14067fn command_in_path(name: &str) -> bool {
14068 std::env::var_os("PATH")
14069 .map(|paths| {
14070 std::env::split_paths(&paths).any(|dir| {
14071 let path = dir.join(name);
14072 path.exists() && path.is_file()
14073 })
14074 })
14075 .unwrap_or(false)
14076}
14077
14078fn speech_model_cached(schema: &ModelSchema) -> bool {
14079 match &schema.source {
14080 ModelSource::Mlx { hf_repo, .. } => huggingface_repo_has_snapshot(hf_repo),
14081 ModelSource::WhisperCpp { model } => car_whisper::model_cached(model),
14082 ModelSource::WindowsSpeech {} => cfg!(target_os = "windows"),
14084 ModelSource::Proprietary { .. } => false,
14087 _ => false,
14088 }
14089}
14090
14091fn model_source_configured(schema: &ModelSchema, credential: &dyn Fn(&str) -> bool) -> bool {
14111 match &schema.source {
14112 ModelSource::RemoteApi {
14113 protocol: ApiProtocol::OpenRouter,
14114 ..
14115 } => crate::openrouter::credential_source().is_some(),
14116 ModelSource::RemoteApi {
14117 api_key_env,
14118 api_key_envs,
14119 ..
14120 } => credential(api_key_env) || api_key_envs.iter().any(|env_var| credential(env_var)),
14121 ModelSource::Proprietary { auth, .. } => match auth {
14122 ProprietaryAuth::ApiKeyEnv { env_var } => credential(env_var),
14123 ProprietaryAuth::BearerTokenEnv { env_var } => credential(env_var),
14124 ProprietaryAuth::OAuth2Pkce { .. } => matches!(
14125 car_auth::credential_authority_hint().state,
14126 car_auth::CredentialAuthorityState::Configured
14127 ),
14128 ProprietaryAuth::ChatGptSubscription {} => {
14129 credential(car_auth::CODEX_AUTH_RECORD_KEY)
14130 || crate::registry::passive_codex_oauth_available()
14131 }
14132 },
14133 ModelSource::VllmMlx { .. } => {
14134 std::env::var("VLLM_MLX_ENDPOINT").is_ok() || schema.available
14135 }
14136 ModelSource::Ollama { .. } => schema.available,
14137 ModelSource::CodexCli { .. } => crate::backend::codex_cli::is_available(),
14138 ModelSource::Mlx { .. }
14139 | ModelSource::ManagedVllmMlx { .. }
14140 | ModelSource::Local { .. }
14141 | ModelSource::WhisperCpp { .. } => true,
14142 ModelSource::WindowsSpeech {} => schema.available,
14144 ModelSource::AppleFoundationModels { .. } => schema.available,
14145 ModelSource::Delegated { .. } => true,
14150 }
14151}
14152
14153fn scoreboard_price_model(schema: &ModelSchema) -> Option<crate::scoreboard::PriceModel> {
14154 if schema.is_subscription_billed() {
14155 return None;
14156 }
14157 match (schema.cost.input_per_mtok, schema.cost.output_per_mtok) {
14158 (Some(input_per_mtok), Some(output_per_mtok)) => Some(crate::scoreboard::PriceModel {
14162 input_per_mtok,
14163 output_per_mtok,
14164 cache: schema.cache_rates(),
14165 is_estimate: !schema.cost.pricing_tiers.is_empty(),
14166 }),
14167 _ => None,
14168 }
14169}
14170
14171fn all_model_capabilities() -> [ModelCapability; 13] {
14172 [
14173 ModelCapability::Generate,
14174 ModelCapability::Embed,
14175 ModelCapability::Classify,
14176 ModelCapability::Code,
14177 ModelCapability::Reasoning,
14178 ModelCapability::Summarize,
14179 ModelCapability::ToolUse,
14180 ModelCapability::MultiToolCall,
14181 ModelCapability::Vision,
14182 ModelCapability::SpeechToText,
14183 ModelCapability::TextToSpeech,
14184 ModelCapability::ImageGeneration,
14185 ModelCapability::VideoGeneration,
14186 ]
14187}
14188
14189fn sort_capabilities(mut capabilities: Vec<ModelCapability>) -> Vec<ModelCapability> {
14190 capabilities.sort_by_key(|capability| {
14191 all_model_capabilities()
14192 .iter()
14193 .position(|candidate| candidate == capability)
14194 .unwrap_or(usize::MAX)
14195 });
14196 capabilities
14197}
14198
14199fn speech_model_source_label(schema: &ModelSchema) -> String {
14200 match &schema.source {
14201 ModelSource::Mlx { hf_repo, .. } => format!("mlx:{hf_repo}"),
14202 ModelSource::ManagedVllmMlx { hf_repo, .. } => {
14203 format!("managed-vllm-mlx:{hf_repo}")
14204 }
14205 ModelSource::WhisperCpp { model } => format!("whisper:{model}"),
14206 ModelSource::WindowsSpeech {} => "windows-speech".to_string(),
14207 ModelSource::Proprietary {
14208 provider, endpoint, ..
14209 } => format!("proprietary:{provider}:{endpoint}"),
14210 ModelSource::RemoteApi { endpoint, .. } => format!("remote:{endpoint}"),
14211 ModelSource::CodexCli { model } => format!("codex-cli:{model}"),
14212 ModelSource::Local { hf_repo, .. } => format!("local:{hf_repo}"),
14213 ModelSource::VllmMlx {
14214 endpoint,
14215 model_name,
14216 } => format!("vllm-mlx:{endpoint}:{model_name}"),
14217 ModelSource::Ollama { model_tag, host } => format!("ollama:{host}:{model_tag}"),
14218 ModelSource::AppleFoundationModels { use_case } => {
14219 format!(
14220 "apple-foundation:{}",
14221 use_case.as_deref().unwrap_or("default")
14222 )
14223 }
14224 ModelSource::Delegated { hint } => {
14225 format!("delegated:{}", hint.as_deref().unwrap_or("(none)"))
14226 }
14227 }
14228}
14229
14230fn rerank_prompt(instruction: &str, query: &str, document: &str) -> String {
14238 const SYSTEM: &str = "Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".";
14239 format!(
14240 "<|im_start|>system\n{SYSTEM}<|im_end|>\n\
14241 <|im_start|>user\n<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {document}<|im_end|>\n\
14242 <|im_start|>assistant\n<think>\n\n</think>\n\n"
14243 )
14244}
14245
14246fn parse_api_returned_status(err: &str) -> Option<u16> {
14270 let lower = err.to_ascii_lowercase();
14271 let idx = lower.find("api returned ")?;
14272 let digits: String = lower[idx + "api returned ".len()..]
14273 .chars()
14274 .take_while(|c| c.is_ascii_digit())
14275 .take(3)
14276 .collect();
14277 digits.parse().ok()
14278}
14279
14280pub const NO_BACKEND_RECOVERY_MARKER: &str = "no inference backend is available.";
14293
14294const AUTO_GRACE_FILE: &str = "concierge-auto-since";
14297const AUTO_GRACE_SECS: u64 = 7 * 24 * 60 * 60;
14299
14300#[doc(hidden)]
14304pub const LOCAL_TOOL_MODEL_PULL_ID: &str =
14305 if cfg!(all(
14308 target_os = "macos",
14309 target_arch = "aarch64",
14310 not(car_skip_mlx)
14311 )) {
14312 "mlx/qwen3-4b:4bit"
14313 } else {
14314 "qwen/qwen3-4b:q4_k_m"
14315 };
14316
14317#[doc(hidden)]
14324pub fn no_backend_recovery_hint(underlying: &str) -> Option<String> {
14325 let is_no_backend = underlying.contains("no credential")
14326 || underlying.contains("model not found")
14327 || underlying.contains("no models available")
14328 || underlying.contains("no inference runner");
14329 if !is_no_backend {
14330 return None;
14331 }
14332 Some(format!(
14338 "{NO_BACKEND_RECOVERY_MARKER} To use Parslee's hosted models, run:\n \
14339 car auth login\n\
14340 Or, for a model you name yourself with --model, install a local \
14341 tool-capable one:\n \
14342 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
14343 (underlying error: {underlying})"
14344 ))
14345}
14346
14347pub fn is_auth_rejection_message(underlying: &str) -> bool {
14365 let l = underlying.to_ascii_lowercase();
14366 l.contains("org lookup failed")
14367 || l.contains("authentication required")
14368 || (l.contains("401") && l.contains("unauthorized"))
14369 || (l.contains("403") && l.contains("forbidden"))
14370 || l.contains("invalid_grant")
14371 || l.contains("token expired")
14372}
14373
14374pub fn is_auth_failure_message(message: &str) -> bool {
14381 let lower = message.to_ascii_lowercase();
14382 AUTH_FAILURE_MESSAGE_MARKERS
14383 .iter()
14384 .any(|marker| lower.contains(marker))
14385 || is_auth_rejection_message(message)
14386}
14387
14388fn record_auth_dead_lane(slot: &mut Option<String>, candidate: &str, error: &str) {
14397 if slot.is_none() && is_auth_rejection_message(error) {
14398 *slot = Some(candidate.to_string());
14399 }
14400}
14401
14402fn is_local_last_resort(appended_id: Option<&str>, candidate_id: &str) -> bool {
14407 appended_id == Some(candidate_id)
14408}
14409
14410fn record_generation_span(
14426 model: &str,
14427 latency_ms: u64,
14428 ttft_ms: Option<u64>,
14429 usage: Option<&TokenUsage>,
14430) {
14431 let span = tracing::Span::current();
14432 span.record("model", model);
14433 span.record("gen_ai.response.model", model);
14434 span.record("latency_ms", latency_ms);
14435 if let Some(ttft) = ttft_ms {
14436 span.record("ttft_ms", ttft);
14437 }
14438 if let Some(u) = usage {
14439 span.record("prompt_tokens", u.prompt_tokens);
14440 span.record("completion_tokens", u.completion_tokens);
14441 span.record("gen_ai.usage.input_tokens", u.prompt_tokens);
14442 span.record("gen_ai.usage.output_tokens", u.completion_tokens);
14443 }
14444}
14445
14446fn report_local_last_resort_served(
14447 appended_id: Option<&str>,
14448 candidate_id: &str,
14449 resolved_id: &str,
14450) -> bool {
14451 let served = is_local_last_resort(appended_id, candidate_id);
14452 if served {
14453 tracing::warn!(
14454 local_model = %resolved_id,
14455 "on-device last-resort model served inference turn"
14456 );
14457 }
14458 served
14459}
14460
14461fn auth_expired_recovery_hint(underlying: &str) -> Option<String> {
14471 if !is_auth_rejection_message(underlying) {
14472 return None;
14473 }
14474 Some(format!(
14475 "your Parslee session has expired or was rejected — re-authenticate:\n \
14476 car auth login\n\
14477 Or, for a model you name yourself with --model, install a local \
14478 tool-capable one:\n \
14479 car models pull {LOCAL_TOOL_MODEL_PULL_ID}\n\
14480 (underlying error: {underlying})"
14481 ))
14482}
14483
14484fn score_from_rerank_output(text: &str, model_name: &str) -> f32 {
14485 let normalized: String = text
14490 .to_ascii_lowercase()
14491 .chars()
14492 .map(|c| if c.is_ascii_alphanumeric() { c } else { ' ' })
14493 .collect();
14494 for tok in normalized.split_ascii_whitespace().take(5) {
14495 match tok {
14496 "yes" => return 1.0,
14497 "no" => return 0.0,
14498 _ => continue,
14499 }
14500 }
14501 tracing::warn!(
14502 model = %model_name,
14503 output = %text,
14504 "rerank: first tokens contain neither `yes` nor `no`; returning neutral 0.5"
14505 );
14506 0.5
14507}
14508
14509fn default_speech_voice(schema: &ModelSchema) -> Option<String> {
14510 if schema.provider == "elevenlabs" {
14511 Some("JBFqnCBsd6RMkjVDRZzb".to_string())
14512 } else if schema.name == "Kokoro-82M-6bit" || schema.name == "Kokoro-82M-bf16" {
14513 Some("af_heart".to_string())
14514 } else if schema.name == "Qwen3-TTS-12Hz-1.7B-Base-5bit" {
14515 Some("Chelsie".to_string())
14516 } else {
14517 None
14518 }
14519}
14520
14521#[allow(dead_code)] fn huggingface_repo_has_snapshot(repo_id: &str) -> bool {
14523 crate::registry::latest_huggingface_repo_snapshot(repo_id).is_some()
14524}
14525
14526#[allow(dead_code)] fn count_files_recursive(root: &Path) -> usize {
14528 let Ok(entries) = std::fs::read_dir(root) else {
14529 return 0;
14530 };
14531 entries
14532 .filter_map(Result::ok)
14533 .map(|entry| entry.path())
14534 .map(|path| {
14535 if path.is_dir() {
14536 count_files_recursive(&path)
14537 } else if path.is_file() {
14538 1
14539 } else {
14540 0
14541 }
14542 })
14543 .sum()
14544}
14545
14546async fn download_hf_repo_snapshot(repo_id: &str) -> Result<(PathBuf, usize), InferenceError> {
14547 let api = crate::hf_cache::api()
14548 .with_progress(false)
14549 .build()
14550 .map_err(|e| InferenceError::DownloadFailed(format!("init hf api: {e}")))?;
14551 let repo = api.model(repo_id.to_string());
14552 let info = repo
14553 .info()
14554 .await
14555 .map_err(|e| InferenceError::DownloadFailed(format!("{repo_id}: {e}")))?;
14556
14557 let snapshot_path = crate::hf_cache::repo_dir(repo_id)
14558 .join("snapshots")
14559 .join(&info.sha);
14560 let mut downloaded = 0usize;
14561 for sibling in &info.siblings {
14562 let local_path = snapshot_path.join(&sibling.rfilename);
14563 if crate::download::cache_file_usable(&local_path) {
14571 downloaded += 1;
14572 continue;
14573 }
14574 let _ = std::fs::remove_file(&local_path);
14579 repo.download(&sibling.rfilename).await.map_err(|e| {
14580 InferenceError::DownloadFailed(format!("{repo_id}/{}: {e}", sibling.rfilename))
14581 })?;
14582 crate::retire::note_fetched(repo_id, &sibling.rfilename, Some(info.sha.clone()));
14583 downloaded += 1;
14584 }
14585
14586 Ok((snapshot_path, downloaded))
14587}
14588
14589fn temp_work_dir(prefix: &str) -> Result<PathBuf, InferenceError> {
14590 let unique = SystemTime::now()
14591 .duration_since(UNIX_EPOCH)
14592 .map_err(|e| InferenceError::InferenceFailed(format!("clock error: {e}")))?
14593 .as_nanos();
14594 let dir = std::env::temp_dir().join(format!("car-inference-{prefix}-{unique}"));
14595 std::fs::create_dir_all(&dir)?;
14596 Ok(dir)
14597}
14598
14599fn ensure_parent_dir(path: &Path) -> Result<(), InferenceError> {
14600 if let Some(parent) = path.parent() {
14601 std::fs::create_dir_all(parent)?;
14602 }
14603 Ok(())
14604}
14605
14606fn requested_or_temp_output(
14607 output_path: Option<&str>,
14608 format: &str,
14609) -> Result<PathBuf, InferenceError> {
14610 if let Some(path) = output_path {
14611 return Ok(PathBuf::from(path));
14612 }
14613 let dir = temp_work_dir("audio-out")?;
14614 Ok(dir.join(format!("speech.{format}")))
14615}
14616
14617#[allow(dead_code)] fn requested_or_temp_media_output(
14619 output_path: Option<&str>,
14620 format: &str,
14621 stem: &str,
14622) -> Result<PathBuf, InferenceError> {
14623 if let Some(path) = output_path {
14624 return Ok(PathBuf::from(path));
14625 }
14626 let dir = temp_work_dir(&format!("{stem}-out"))?;
14627 Ok(dir.join(format!("{stem}.{format}")))
14628}
14629
14630fn materialize_audio_output(
14631 produced: &Path,
14632 requested: Option<&str>,
14633 format: &str,
14634) -> Result<PathBuf, InferenceError> {
14635 if let Some(path) = requested {
14636 let dest = PathBuf::from(path);
14637 ensure_parent_dir(&dest)?;
14638 std::fs::copy(produced, &dest)?;
14639 Ok(dest)
14640 } else {
14641 let dest = requested_or_temp_output(None, format)?;
14642 ensure_parent_dir(&dest)?;
14643 std::fs::copy(produced, &dest)?;
14644 Ok(dest)
14645 }
14646}
14647
14648#[cfg(target_os = "windows")]
14654fn winrt_synthesize_wav(text: &str, voice: &str, rate: f64) -> Result<Vec<u8>, InferenceError> {
14655 use windows::core::HSTRING;
14656 use windows::Media::SpeechSynthesis::SpeechSynthesizer;
14657 use windows::Storage::Streams::DataReader;
14658
14659 let err = |m: String| InferenceError::InferenceFailed(m);
14660 let synth =
14661 SpeechSynthesizer::new().map_err(|e| err(format!("SpeechSynthesizer::new: {e}")))?;
14662 if let Ok(opts) = synth.Options() {
14663 let _ = opts.SetSpeakingRate(rate.clamp(0.5, 6.0));
14664 }
14665 if !voice.is_empty() {
14666 if let Ok(all) = SpeechSynthesizer::AllVoices() {
14667 let want = voice.to_lowercase();
14668 let count = all.Size().unwrap_or(0);
14669 for i in 0..count {
14670 if let Ok(info) = all.GetAt(i) {
14671 if let Ok(name) = info.DisplayName() {
14672 if name.to_string_lossy().to_lowercase().contains(&want) {
14673 let _ = synth.SetVoice(&info);
14674 break;
14675 }
14676 }
14677 }
14678 }
14679 }
14680 }
14681 let stream = synth
14682 .SynthesizeTextToStreamAsync(&HSTRING::from(text))
14683 .map_err(|e| err(format!("SynthesizeTextToStreamAsync: {e}")))?
14684 .get()
14685 .map_err(|e| err(format!("synthesize await: {e}")))?;
14686 let size = stream
14687 .Size()
14688 .map_err(|e| err(format!("stream size: {e}")))?;
14689 let input = stream
14690 .GetInputStreamAt(0)
14691 .map_err(|e| err(format!("input stream: {e}")))?;
14692 let reader =
14693 DataReader::CreateDataReader(&input).map_err(|e| err(format!("data reader: {e}")))?;
14694 reader
14695 .LoadAsync(size as u32)
14696 .map_err(|e| err(format!("load async: {e}")))?
14697 .get()
14698 .map_err(|e| err(format!("load await: {e}")))?;
14699 let mut buf = vec![0u8; size as usize];
14700 reader
14701 .ReadBytes(&mut buf)
14702 .map_err(|e| err(format!("read bytes: {e}")))?;
14703 Ok(buf)
14704}
14705
14706#[allow(dead_code)] fn read_transcription_result(output_prefix: &Path) -> Result<Option<String>, InferenceError> {
14708 let candidates = [
14709 output_prefix.with_extension("json"),
14710 output_prefix.to_path_buf(),
14711 ];
14712
14713 for path in candidates {
14714 if path.exists() {
14715 let contents = std::fs::read_to_string(path)?;
14716 if let Some(text) = extract_text_from_payload(&contents) {
14717 return Ok(Some(text));
14718 }
14719 }
14720 }
14721
14722 Ok(None)
14723}
14724
14725#[allow(dead_code)] fn extract_text_from_payload(payload: &str) -> Option<String> {
14727 let value: serde_json::Value = serde_json::from_str(payload).ok()?;
14728 if let Some(text) = value.get("text").and_then(|v| v.as_str()) {
14729 return Some(text.to_string());
14730 }
14731 if let Some(transcripts) = value.get("transcripts").and_then(|v| v.as_array()) {
14732 let joined = transcripts
14733 .iter()
14734 .filter_map(|item| item.get("text").and_then(|v| v.as_str()))
14735 .collect::<Vec<_>>()
14736 .join("\n");
14737 if !joined.is_empty() {
14738 return Some(joined);
14739 }
14740 }
14741 if let Some(items) = value.as_array() {
14742 let joined = items
14743 .iter()
14744 .filter_map(|item| {
14745 item.get("text")
14746 .or_else(|| item.get("Content"))
14747 .and_then(|v| v.as_str())
14748 })
14749 .collect::<Vec<_>>()
14750 .join(" ");
14751 if !joined.is_empty() {
14752 return Some(joined);
14753 }
14754 }
14755 None
14756}
14757
14758#[allow(dead_code)] fn find_audio_file(output_dir: &Path) -> Result<Option<PathBuf>, InferenceError> {
14760 let mut audio_files = Vec::new();
14761 collect_audio_files(output_dir, &mut audio_files)?;
14762 audio_files.sort();
14763 Ok(audio_files.into_iter().next())
14764}
14765
14766#[allow(dead_code)] fn collect_audio_files(dir: &Path, audio_files: &mut Vec<PathBuf>) -> Result<(), InferenceError> {
14768 for entry in std::fs::read_dir(dir)? {
14769 let path = entry?.path();
14770 if path.is_dir() {
14771 collect_audio_files(&path, audio_files)?;
14772 } else if matches!(
14773 path.extension().and_then(|ext| ext.to_str()),
14774 Some("wav" | "mp3" | "flac" | "pcm" | "m4a")
14775 ) {
14776 audio_files.push(path);
14777 }
14778 }
14779 Ok(())
14780}
14781
14782fn media_type_for_format(format: &str) -> String {
14783 match format.to_ascii_lowercase().as_str() {
14784 "mp3" => "audio/mpeg".to_string(),
14785 "flac" => "audio/flac".to_string(),
14786 "pcm" => "audio/L16".to_string(),
14787 "m4a" => "audio/mp4".to_string(),
14788 _ => "audio/wav".to_string(),
14789 }
14790}
14791
14792fn kokoro_lang_code(language: Option<&str>) -> &'static str {
14793 match language.unwrap_or("en").to_ascii_lowercase().as_str() {
14794 "en-gb" | "british" | "british english" => "b",
14795 "ja" | "japanese" => "j",
14796 "zh" | "zh-cn" | "mandarin" | "chinese" => "z",
14797 "es" | "spanish" => "e",
14798 "fr" | "french" => "f",
14799 _ => "a",
14800 }
14801}
14802
14803#[allow(dead_code)] fn normalize_lang_code(language: &str) -> String {
14805 match language.to_ascii_lowercase().as_str() {
14806 "english" | "en-us" | "en_us" => "en".to_string(),
14807 "spanish" => "es".to_string(),
14808 "french" => "fr".to_string(),
14809 "japanese" => "ja".to_string(),
14810 "chinese" | "mandarin" => "zh".to_string(),
14811 other => match other {
14812 "en" | "es" | "fr" | "ja" | "zh" => other.to_string(),
14813 _ => "en".to_string(),
14814 },
14815 }
14816}
14817
14818fn elevenlabs_auth(
14819 schema: &ModelSchema,
14820 resolve_credential: fn(&str) -> Option<String>,
14821) -> Result<(String, String), InferenceError> {
14822 match &schema.source {
14823 ModelSource::Proprietary {
14824 endpoint,
14825 auth: schema::ProprietaryAuth::ApiKeyEnv { env_var },
14826 ..
14827 } => {
14828 let key = resolve_credential(env_var).ok_or_else(|| {
14829 InferenceError::InferenceFailed(format!(
14830 "missing API key {env_var}; set the environment variable or \
14831 store it with `car secrets put {env_var}`"
14832 ))
14833 })?;
14834 Ok((endpoint.clone(), key))
14835 }
14836 _ => Err(InferenceError::InferenceFailed(format!(
14837 "model {} is not an ElevenLabs proprietary model",
14838 schema.id
14839 ))),
14840 }
14841}
14842
14843fn elevenlabs_output_format(format: &str) -> &'static str {
14844 match format.to_ascii_lowercase().as_str() {
14845 "mp3" => "mp3_44100_128",
14846 "pcm" => "pcm_16000",
14847 _ => "wav_44100",
14848 }
14849}
14850
14851fn quality_seeds(
14855 state_models_dir: &Path,
14856) -> std::collections::HashMap<String, outcome::QualitySeed> {
14857 let mut seeds = std::collections::HashMap::new();
14858 for path in benchmark_priors_paths(state_models_dir) {
14859 let Ok(priors) = routing_ext::load_benchmark_priors(&path) else {
14860 continue;
14861 };
14862 for (model_id, prior) in priors {
14863 seeds.insert(
14864 model_id,
14865 outcome::QualitySeed {
14866 overall: Some(prior.overall_score),
14867 tasks: prior.task_scores,
14868 },
14869 );
14870 }
14871 }
14872 seeds
14873}
14874
14875fn benchmark_priors_paths(state_models_dir: &Path) -> Vec<PathBuf> {
14880 let mut paths = Vec::new();
14881
14882 let direct = state_models_dir.join("benchmark_priors.json");
14883 if !paths.contains(&direct) {
14884 paths.push(direct);
14885 }
14886
14887 if let Some(parent) = state_models_dir.parent() {
14888 let parent_path = parent.join("benchmark_priors.json");
14889 if !paths.contains(&parent_path) {
14890 paths.push(parent_path);
14891 }
14892 }
14893
14894 if let Some(path) = std::env::var_os("CAR_BENCHMARK_PRIORS_PATH") {
14895 let path = PathBuf::from(path);
14896 if !paths.contains(&path) {
14897 paths.push(path);
14898 }
14899 }
14900
14901 paths
14902}
14903
14904fn load_benchmark_prior_health(
14905 state_models_dir: &Path,
14906 schemas: &[ModelSchema],
14907) -> Vec<ModelBenchmarkPriorHealth> {
14908 let mut priors = std::collections::BTreeMap::new();
14909 for path in benchmark_priors_paths(state_models_dir) {
14910 let Ok(loaded) = routing_ext::load_benchmark_priors(&path) else {
14911 continue;
14912 };
14913 for (model_id, prior) in loaded {
14914 let model_name = schemas
14915 .iter()
14916 .find(|schema| schema.id == model_id)
14917 .map(|schema| schema.name.clone());
14918 priors.insert(
14919 model_id.clone(),
14920 ModelBenchmarkPriorHealth {
14921 model_id,
14922 model_name,
14923 overall_score: prior.overall_score,
14924 overall_latency_ms: prior.overall_latency_ms,
14925 task_scores: prior.task_scores,
14926 task_latency_ms: prior.task_latency_ms,
14927 source_path: path.clone(),
14928 },
14929 );
14930 }
14931 }
14932
14933 priors.into_values().collect()
14934}
14935
14936fn kokoro_runtime_fallback_enabled() -> bool {
14937 std::env::var("CAR_SPEECH_KOKORO_FALLBACK")
14938 .ok()
14939 .map(|value| {
14940 !matches!(
14941 value.trim().to_ascii_lowercase().as_str(),
14942 "0" | "false" | "off"
14943 )
14944 })
14945 .unwrap_or(true)
14946}
14947
14948fn speech_runtime_mlx_audio_spec() -> String {
14949 std::env::var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC")
14950 .ok()
14951 .filter(|value| !value.trim().is_empty())
14952 .unwrap_or_else(|| "mlx-audio==0.4.2".to_string())
14953}
14954
14955fn speech_runtime_spacy_model_spec() -> String {
14956 std::env::var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC")
14957 .ok()
14958 .filter(|value| !value.trim().is_empty())
14959 .unwrap_or_else(|| {
14960 "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl".to_string()
14961 })
14962}
14963
14964#[cfg(test)]
14965pub(crate) fn run_in_isolated_test_process(test_name: &str, sentinel: &str) -> bool {
14966 static CHILD_PROCESS_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
14967
14968 if std::env::var_os(sentinel).is_some() {
14969 return true;
14970 }
14971
14972 let _child_process = CHILD_PROCESS_MUTEX
14973 .lock()
14974 .unwrap_or_else(std::sync::PoisonError::into_inner);
14975 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
14976 command
14977 .arg("--exact")
14978 .arg(test_name)
14979 .arg("--nocapture")
14980 .arg("--test-threads=1")
14981 .env(sentinel, "1");
14982 for name in [
14983 car_home::ENV_VAR,
14984 "CAR_SECRETS_FILE_DIR",
14985 "CAR_TEST_NATIVE_KEYCHAIN",
14986 "CAR_AUTH_LOCK_PATH",
14987 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
14988 car_auth::PARSLEE_API_BASE_KEY,
14989 crate::openrouter::API_KEY_ENV,
14990 "OPENAI_API_KEY",
14991 "ANTHROPIC_API_KEY",
14992 "GOOGLE_API_KEY",
14993 "ELEVENLABS_API_KEY",
14994 "SSL_CERT_FILE",
14995 "SSL_CERT_DIR",
14996 ] {
14997 command.env_remove(name);
14998 }
14999 const CHILD_BUDGET: std::time::Duration = std::time::Duration::from_secs(300);
15006 let mut stdout = tempfile::tempfile().expect("create isolated-test stdout capture");
15007 let mut stderr = tempfile::tempfile().expect("create isolated-test stderr capture");
15008 command
15009 .stdin(std::process::Stdio::null())
15010 .stdout(stdout.try_clone().expect("clone stdout capture"))
15011 .stderr(stderr.try_clone().expect("clone stderr capture"));
15012 #[cfg(unix)]
15013 {
15014 use std::os::unix::process::CommandExt;
15015 command.process_group(0);
15016 }
15017
15018 let mut child = command.spawn().expect("spawn isolated inference test");
15019 let deadline = std::time::Instant::now() + CHILD_BUDGET;
15020 let mut timed_out = false;
15021 let status = loop {
15022 match child.try_wait().expect("poll isolated inference test") {
15023 Some(status) => break status,
15024 None if std::time::Instant::now() < deadline => {
15025 std::thread::sleep(std::time::Duration::from_millis(50));
15026 }
15027 None => {
15028 timed_out = true;
15029 #[cfg(unix)]
15030 unsafe {
15031 libc::kill(-(child.id() as i32), libc::SIGKILL);
15032 }
15033 let _ = child.kill();
15034 break child
15035 .wait()
15036 .expect("reap timed-out isolated inference test");
15037 }
15038 }
15039 };
15040
15041 use std::io::{Read, Seek};
15042 stdout.rewind().expect("rewind isolated-test stdout");
15043 stderr.rewind().expect("rewind isolated-test stderr");
15044 let mut stdout_bytes = Vec::new();
15045 let mut stderr_bytes = Vec::new();
15046 stdout
15047 .read_to_end(&mut stdout_bytes)
15048 .expect("read isolated-test stdout");
15049 stderr
15050 .read_to_end(&mut stderr_bytes)
15051 .expect("read isolated-test stderr");
15052 assert!(
15053 !timed_out,
15054 "isolated {test_name} exceeded {}s and was killed\nstdout:\n{}\nstderr:\n{}",
15055 CHILD_BUDGET.as_secs(),
15056 String::from_utf8_lossy(&stdout_bytes),
15057 String::from_utf8_lossy(&stderr_bytes),
15058 );
15059 assert!(
15060 status.success(),
15061 "isolated {test_name} failed\nstdout:\n{}\nstderr:\n{}",
15062 String::from_utf8_lossy(&stdout_bytes),
15063 String::from_utf8_lossy(&stderr_bytes),
15064 );
15065 false
15066}
15067
15068fn pinned_by(agents: &[String]) -> String {
15070 let quoted: Vec<String> = agents.iter().map(|a| format!("\"{a}\"")).collect();
15071 match quoted.as_slice() {
15072 [one] => format!("pinned by the agent {one}"),
15073 _ => format!("pinned by the agents {}", quoted.join(", ")),
15074 }
15075}
15076
15077#[cfg(test)]
15078mod tests {
15079
15080 #[test]
15090 fn every_successful_generation_records_its_span() {
15091 let source = include_str!("lib.rs");
15092 let start = source
15093 .find("async fn generate_tracked_inner")
15094 .expect("the tracked-inference entry point");
15095 let open = source[start..].find('{').expect("function body") + start;
15096 let mut depth = 0usize;
15097 let mut end = open;
15098 for (offset, ch) in source[open..].char_indices() {
15099 match ch {
15100 '{' => depth += 1,
15101 '}' => {
15102 depth -= 1;
15103 if depth == 0 {
15104 end = open + offset;
15105 break;
15106 }
15107 }
15108 _ => {}
15109 }
15110 }
15111 let body = &source[start..end];
15112 let returns = body.matches("return Ok(InferenceResult {").count();
15113 let records = body.matches("record_generation_span(").count();
15114 assert!(
15115 returns >= 3,
15116 "control: the function still has several success returns, got {returns}"
15117 );
15118 assert_eq!(
15119 returns, records,
15120 "every successful return must record its generation span: \
15121 {returns} returns, {records} recorder calls"
15122 );
15123 }
15124 use super::*;
15125
15126 #[test]
15127 fn classify_defaults_to_jev_only_while_it_is_available() {
15128 let catalog: Vec<ModelSchema> =
15129 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
15130 let mut jev = catalog
15131 .iter()
15132 .find(|s| s.id == "parslee/jev")
15133 .unwrap()
15134 .clone();
15135 let local = catalog
15136 .iter()
15137 .find(|s| s.id == "mlx/qwen3-0.6b:6bit")
15138 .unwrap()
15139 .clone();
15140 jev.available = false;
15141 assert_eq!(
15142 default_system_one_classifier([&jev, &local].into_iter()),
15143 None,
15144 "signed out: keep the local default"
15145 );
15146 jev.available = true;
15147 assert_eq!(
15148 default_system_one_classifier([&local, &jev].into_iter()).as_deref(),
15149 Some("parslee/jev")
15150 );
15151 let chat = catalog
15153 .iter()
15154 .find(|s| s.id == "parslee/fast")
15155 .unwrap()
15156 .clone();
15157 assert_eq!(default_system_one_classifier([&chat].into_iter()), None);
15158 }
15159
15160 #[tokio::test]
15161 async fn generate_pinned_to_a_system_one_row_is_refused_streaming_included() {
15162 let dir = tempfile::tempdir().unwrap();
15163 let engine = InferenceEngine::new(test_config(dir.path().to_path_buf()));
15164 let req = || GenerateRequest {
15165 prompt: "hello".into(),
15166 model: Some("parslee/jev".into()),
15167 ..Default::default()
15168 };
15169 let refused = |e: &InferenceError| {
15170 matches!(
15171 e,
15172 InferenceError::UnsupportedMode {
15173 backend: "parslee-system-one",
15174 ..
15175 }
15176 )
15177 };
15178 match engine.generate_tracked(req()).await {
15179 Err(e) => assert!(refused(&e), "{e}"),
15180 Ok(_) => panic!("generate on a System One row must be refused"),
15181 }
15182 match engine.generate_tracked_stream(req()).await {
15183 Err(e) => assert!(refused(&e), "{e}"),
15184 Ok(_) => panic!("streaming generate on a System One row must be refused"),
15185 }
15186 }
15187 use std::ffi::OsString;
15188 use tempfile::TempDir;
15189
15190 #[test]
15197 fn local_models_keep_the_callers_default_output_budget() {
15198 use crate::tasks::generate::DEFAULT_MAX_TOKENS;
15199
15200 let catalog = crate::registry::builtin_catalog();
15201 let local = catalog
15202 .iter()
15203 .find(|s| s.id == "mlx/qwen3-8b:4bit")
15204 .expect("mlx/qwen3-8b:4bit is a builtin catalog entry");
15205
15206 assert!(local.is_local(), "mlx/* is decoded in-process");
15207 assert_eq!(
15208 local.effective_max_output(),
15209 32_768,
15210 "the budget that made the reported turn ~24 minutes long"
15211 );
15212 assert_eq!(
15213 resolved_max_tokens(DEFAULT_MAX_TOKENS, local),
15214 DEFAULT_MAX_TOKENS,
15215 "a local model must keep the budget the caller actually asked for"
15216 );
15217
15218 let mut remote = catalog
15221 .iter()
15222 .find(|s| !s.is_local())
15223 .expect("the builtin catalog ships remote models")
15224 .clone();
15225 remote.context_length = local.context_length;
15226 remote.max_output_tokens = local.max_output_tokens;
15227 assert!(!remote.is_local());
15228 assert_eq!(
15229 resolved_max_tokens(DEFAULT_MAX_TOKENS, &remote),
15230 32_768,
15231 "remote models still get their advertised output budget"
15232 );
15233
15234 assert_eq!(resolved_max_tokens(512, local), 512);
15236 assert_eq!(resolved_max_tokens(512, &remote), 512);
15237
15238 let codex = catalog
15239 .iter()
15240 .find(|s| s.id == "openai/gpt-5.6-sol:high")
15241 .expect("subscription-backed Codex row is builtin");
15242 assert_eq!(
15243 resolved_max_tokens(DEFAULT_MAX_TOKENS, codex),
15244 DEFAULT_MAX_TOKENS,
15245 "an approximate instruction must not widen the default to 128K tokens"
15246 );
15247
15248 if let Some(vllm) = catalog.iter().find(|s| s.is_vllm_mlx()) {
15254 assert!(vllm.is_local(), "vLLM-MLX runs on this machine");
15255 assert!(
15256 !vllm.decodes_in_process(),
15257 "but CAR does not decode it token by token"
15258 );
15259 assert_eq!(
15260 resolved_max_tokens(DEFAULT_MAX_TOKENS, vllm),
15261 vllm.effective_max_output(),
15262 );
15263 }
15264 }
15265
15266 #[test]
15269 fn a_ceiling_stop_fails_instead_of_retrying() {
15270 use EmptyPassAction::*;
15271 const CEILING: Option<&str> = Some(LOCAL_DECODE_TIMEOUT_STOP_REASON);
15272
15273 assert_eq!(
15276 classify_empty_pass(true, CEILING, "", true),
15277 FailDecodeCeiling
15278 );
15279 assert_eq!(
15280 classify_empty_pass(false, CEILING, "", true),
15281 FailDecodeCeiling
15282 );
15283 assert_eq!(
15285 classify_empty_pass(true, CEILING, " \n ", true),
15286 FailDecodeCeiling
15287 );
15288
15289 assert_eq!(
15292 classify_empty_pass(true, Some("length"), "", true),
15293 RetryWithoutThinking
15294 );
15295 assert_eq!(
15296 classify_empty_pass(true, None, "", true),
15297 RetryWithoutThinking
15298 );
15299
15300 assert_eq!(classify_empty_pass(false, Some("stop"), "", true), Accept);
15303
15304 assert_eq!(classify_empty_pass(true, CEILING, "partial", true), Accept);
15307 assert_eq!(classify_empty_pass(true, CEILING, "", false), Accept);
15308
15309 assert_ne!(LOCAL_DECODE_TIMEOUT_STOP_REASON, "timeout");
15313 assert_eq!(
15314 classify_empty_pass(true, Some("timeout"), "", true),
15315 RetryWithoutThinking
15316 );
15317 }
15318
15319 #[test]
15323 fn decode_deadline_and_heartbeat_predicates() {
15324 use std::time::Duration;
15325 let limit = Duration::from_secs(300);
15326
15327 assert!(!deadline_exceeded(Duration::from_secs(86_400), None));
15329
15330 assert!(!deadline_exceeded(Duration::from_secs(299), Some(limit)));
15331 assert!(deadline_exceeded(limit, Some(limit)));
15334 assert!(deadline_exceeded(Duration::from_secs(301), Some(limit)));
15335
15336 let every = Duration::from_secs(10);
15337 assert!(!heartbeat_due(
15339 Duration::from_secs(9),
15340 Duration::ZERO,
15341 every
15342 ));
15343 assert!(heartbeat_due(
15344 Duration::from_secs(10),
15345 Duration::ZERO,
15346 every
15347 ));
15348 assert!(!heartbeat_due(
15350 Duration::from_secs(11),
15351 Duration::from_secs(10),
15352 every
15353 ));
15354 assert!(heartbeat_due(
15355 Duration::from_secs(20),
15356 Duration::from_secs(10),
15357 every
15358 ));
15359 assert!(!heartbeat_due(
15361 Duration::from_secs(5),
15362 Duration::from_secs(10),
15363 every
15364 ));
15365 }
15366
15367 #[test]
15368 fn decode_timeout_parses_with_a_safe_fallback() {
15369 let default = std::time::Duration::from_secs(DEFAULT_LOCAL_DECODE_TIMEOUT_SECS);
15370 assert_eq!(parse_decode_timeout(None), Some(default));
15371 assert_eq!(
15372 parse_decode_timeout(Some(" 45 ")),
15373 Some(std::time::Duration::from_secs(45))
15374 );
15375 assert_eq!(
15376 parse_decode_timeout(Some("0")),
15377 None,
15378 "0 disables the ceiling"
15379 );
15380 assert_eq!(
15381 parse_decode_timeout(Some("banana")),
15382 Some(default),
15383 "garbage must fall back to the default, never silently disable it"
15384 );
15385 }
15386
15387 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15392 #[test]
15393 fn local_decode_stops_at_the_wall_clock_ceiling() {
15394 struct NeverStops;
15397
15398 impl crate::backend::local::TextDecoder for NeverStops {
15399 fn encode(&self, _text: &str) -> Result<Vec<u32>, InferenceError> {
15400 Ok(vec![1, 2, 3])
15401 }
15402 fn decode(&self, tokens: &[u32]) -> Result<String, InferenceError> {
15403 Ok("x".repeat(tokens.len()))
15404 }
15405 fn forward(
15406 &mut self,
15407 _tokens: &[u32],
15408 _pos: usize,
15409 ) -> Result<Vec<f32>, InferenceError> {
15410 std::thread::sleep(std::time::Duration::from_millis(5));
15411 let mut logits = vec![0.0f32; 16];
15412 logits[7] = 10.0;
15413 Ok(logits)
15414 }
15415 fn eos_ids(&self) -> Vec<u32> {
15416 vec![0]
15417 }
15418 fn context_length(&self) -> usize {
15419 4096
15420 }
15421 fn clear_kv_cache(&mut self) {}
15422 }
15423
15424 let params = GenerateParams {
15427 max_tokens: 100_000,
15428 temperature: 0.0,
15429 ..Default::default()
15430 };
15431 let ceiling = std::time::Duration::from_millis(300);
15432
15433 let mut backend = NeverStops;
15434 let started = std::time::Instant::now();
15435 let generated = match InferenceEngine::drive_generation_with_timeout(
15436 &mut backend,
15437 "anything",
15438 ¶ms,
15439 Some(ceiling),
15440 ) {
15441 Ok(generated) => generated,
15442 Err(e) => panic!(
15444 "a decode cut short by the ceiling returns its partial text, not an error: {}",
15445 e.into_inner()
15446 ),
15447 };
15448 let elapsed = started.elapsed();
15449
15450 assert_eq!(
15451 generated.stop_reason.as_deref(),
15452 Some(LOCAL_DECODE_TIMEOUT_STOP_REASON),
15453 "the caller must be able to tell a deadline stop from a clean finish"
15454 );
15455 assert!(
15456 InferenceResult {
15457 text: generated.text.clone(),
15458 bounding_boxes: Vec::new(),
15459 tool_calls: vec![],
15460 trace_id: String::new(),
15461 model_used: String::new(),
15462 model_identity: Default::default(),
15463 latency_ms: 0,
15464 time_to_first_token_ms: None,
15465 usage: None,
15466 provider_output_items: Vec::new(),
15467 thinking: Vec::new(),
15468 stop_reason: generated.stop_reason.clone(),
15469 auth_fallback_from: None,
15470 local_last_resort: false,
15471 fallback_from: Vec::new(),
15472 }
15473 .was_truncated(),
15474 "a ceiling stop is a cut-short answer, not a complete one"
15475 );
15476 assert!(
15477 generated.completion_tokens > 0,
15478 "the partial response is kept, not discarded"
15479 );
15480 assert!(
15481 generated.completion_tokens < params.max_tokens,
15482 "the loop stopped on the clock, not by exhausting the budget"
15483 );
15484 assert!(
15485 elapsed < std::time::Duration::from_secs(30),
15486 "bounded in wall clock; took {elapsed:?}"
15487 );
15488 }
15489
15490 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
15491
15492 impl RestoredEnvironment {
15493 fn capture(names: &[&'static str]) -> Self {
15494 Self(
15495 names
15496 .iter()
15497 .map(|name| (*name, std::env::var_os(name)))
15498 .collect(),
15499 )
15500 }
15501 }
15502
15503 impl Drop for RestoredEnvironment {
15504 fn drop(&mut self) {
15505 for (name, value) in &self.0 {
15506 unsafe {
15507 match value {
15508 Some(value) => std::env::set_var(name, value),
15509 None => std::env::remove_var(name),
15510 }
15511 }
15512 }
15513 }
15514 }
15515
15516 struct FixtureLocalOffload {
15517 emit_done: bool,
15518 }
15519
15520 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15525 struct ExactPinCaptureOffload {
15526 dispatched_models: std::sync::Mutex<Vec<String>>,
15527 }
15528
15529 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15530 impl ExactPinCaptureOffload {
15531 fn new() -> Self {
15532 Self {
15533 dispatched_models: std::sync::Mutex::new(Vec::new()),
15534 }
15535 }
15536
15537 fn dispatched_models(&self) -> Vec<String> {
15538 self.dispatched_models.lock().unwrap().clone()
15539 }
15540 }
15541
15542 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
15543 #[async_trait::async_trait]
15544 impl crate::offload::LocalGenerationOffload for ExactPinCaptureOffload {
15545 async fn generate(
15546 &self,
15547 request: GenerateRequest,
15548 ) -> Result<InferenceResult, InferenceError> {
15549 let model_id = request.model.expect("resolved worker model");
15550 self.dispatched_models
15551 .lock()
15552 .unwrap()
15553 .push(model_id.clone());
15554 Ok(InferenceResult {
15555 text: "exact pin".into(),
15556 tool_calls: vec![],
15557 bounding_boxes: vec![],
15558 trace_id: "worker-trace".into(),
15559 model_used: model_id,
15560 model_identity: Default::default(),
15561 latency_ms: 0,
15562 time_to_first_token_ms: None,
15563 usage: None,
15564 provider_output_items: vec![],
15565 thinking: vec![],
15566 stop_reason: Some("stop".into()),
15567 auth_fallback_from: None,
15568 local_last_resort: false,
15569 fallback_from: Vec::new(),
15570 })
15571 }
15572
15573 async fn stream(
15574 &self,
15575 request: GenerateRequest,
15576 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15577 let model_id = request.model.expect("resolved worker model");
15578 self.dispatched_models.lock().unwrap().push(model_id);
15579 let (tx, rx) = tokio::sync::mpsc::channel(2);
15580 tx.send(StreamEvent::Done {
15581 text: "exact pin".into(),
15582 tool_calls: vec![],
15583 })
15584 .await
15585 .unwrap();
15586 Ok(rx)
15587 }
15588 }
15589
15590 struct ReleaseRetryOffload {
15591 calls: std::sync::Mutex<Vec<String>>,
15592 model_id: String,
15593 resident: bool,
15594 release_acknowledged: bool,
15595 }
15596
15597 struct SequencedRetryProbe {
15598 calls: Arc<std::sync::atomic::AtomicUsize>,
15599 first_available_mb: u64,
15600 subsequent_available_mb: u64,
15601 }
15602
15603 impl crate::resource_policy::LiveMemoryProbe for SequencedRetryProbe {
15604 fn available_memory_mb(
15605 &self,
15606 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
15607 let call = self
15608 .calls
15609 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
15610 Ok(Some(if call == 0 {
15611 self.first_available_mb
15612 } else {
15613 self.subsequent_available_mb
15614 }))
15615 }
15616 }
15617
15618 #[async_trait::async_trait]
15619 impl crate::offload::LocalGenerationOffload for ReleaseRetryOffload {
15620 async fn generate(
15621 &self,
15622 _request: GenerateRequest,
15623 ) -> Result<InferenceResult, InferenceError> {
15624 unreachable!("release-and-retry fixture")
15625 }
15626
15627 async fn stream(
15628 &self,
15629 _request: GenerateRequest,
15630 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15631 unreachable!("release-and-retry fixture")
15632 }
15633
15634 async fn resident_models(&self) -> Vec<String> {
15635 self.resident
15636 .then(|| self.model_id.clone())
15637 .into_iter()
15638 .collect()
15639 }
15640
15641 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
15642 Some(format!("worker:retry-fixture:{model_id}"))
15643 }
15644
15645 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15646 self.calls
15647 .lock()
15648 .unwrap()
15649 .push(format!("release:{model_id}"));
15650 Ok(self.release_acknowledged)
15651 }
15652 }
15653
15654 struct RetiringReleaseOffload {
15655 calls: std::sync::Mutex<Vec<String>>,
15656 model_id: String,
15657 allocation_id: String,
15658 resident: std::sync::atomic::AtomicBool,
15659 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
15660 }
15661
15662 #[async_trait::async_trait]
15663 impl crate::offload::LocalGenerationOffload for RetiringReleaseOffload {
15664 async fn generate(
15665 &self,
15666 _request: GenerateRequest,
15667 ) -> Result<InferenceResult, InferenceError> {
15668 unreachable!("retiring-release fixture")
15669 }
15670
15671 async fn stream(
15672 &self,
15673 _request: GenerateRequest,
15674 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15675 unreachable!("retiring-release fixture")
15676 }
15677
15678 async fn resident_models(&self) -> Vec<String> {
15679 self.resident
15680 .load(std::sync::atomic::Ordering::Acquire)
15681 .then(|| self.model_id.clone())
15682 .into_iter()
15683 .collect()
15684 }
15685
15686 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15687 Some(self.allocation_id.clone())
15688 }
15689
15690 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15691 self.calls
15692 .lock()
15693 .unwrap()
15694 .push(format!("release:{model_id}"));
15695 self.coordinator
15696 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
15697 self.coordinator
15698 .finish_teardown_allocation(model_id, &self.allocation_id);
15699 self.resident
15700 .store(false, std::sync::atomic::Ordering::Release);
15701 Ok(true)
15702 }
15703 }
15704
15705 #[tokio::test]
15706 async fn release_and_retry_releases_resident_worker_after_live_memory_refusal() {
15707 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15708 let root = tempfile::tempdir().unwrap();
15709 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15710 let schema = crate::registry::builtin_catalog()
15711 .into_iter()
15712 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15713 .unwrap();
15714 let allocation_id = format!("worker:retry-fixture:{}", schema.id);
15715 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15716 let coordinator = Arc::new(
15717 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15718 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15719 metal_mac_for_fit(32),
15720 Arc::new(SequencedRetryProbe {
15721 calls: probe_calls.clone(),
15722 first_available_mb: 1_000,
15723 subsequent_available_mb: 1_048_576,
15724 }),
15725 ),
15726 );
15727 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
15728 engine.local_admission = coordinator.clone();
15729 let offload = Arc::new(RetiringReleaseOffload {
15730 calls: std::sync::Mutex::new(Vec::new()),
15731 model_id: schema.id.clone(),
15732 allocation_id,
15733 resident: std::sync::atomic::AtomicBool::new(true),
15734 coordinator,
15735 });
15736 crate::offload::set_local_offload(Some(offload.clone()));
15737
15738 let result = engine
15739 .reserve_local_request_with_worker_retry(&schema, 9_000)
15740 .await;
15741 crate::offload::set_local_offload(None);
15742
15743 assert!(
15744 result.is_ok(),
15745 "release followed by a second probe should admit"
15746 );
15747 assert_eq!(
15748 *offload.calls.lock().unwrap(),
15749 vec![format!("release:{}", schema.id)]
15750 );
15751 assert_eq!(
15752 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
15753 2,
15754 "admission must probe exactly once before and once after release"
15755 );
15756 }
15757
15758 #[tokio::test]
15759 async fn release_and_retry_ignores_sibling_allocation_after_exact_worker_retirement() {
15760 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15761 let root = tempfile::tempdir().unwrap();
15762 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15763 let schema = crate::registry::builtin_catalog()
15764 .into_iter()
15765 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15766 .unwrap();
15767 let worker_allocation_id = format!("worker:sibling-test:{}", schema.id);
15768 let sibling_allocation_id = format!("cache:sibling-test:{}", schema.id);
15769 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15770 let coordinator = Arc::new(
15771 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15772 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15773 metal_mac_for_fit(32),
15774 Arc::new(SequencedRetryProbe {
15775 calls: probe_calls.clone(),
15776 first_available_mb: 1_000,
15777 subsequent_available_mb: 1_048_576,
15778 }),
15779 ),
15780 );
15781 coordinator.mark_resident_allocation(&schema.id, &worker_allocation_id, 4_800);
15782 coordinator.mark_resident_allocation(&schema.id, &sibling_allocation_id, 500);
15783 engine.local_admission = coordinator.clone();
15784 let offload = Arc::new(RetiringReleaseOffload {
15785 calls: std::sync::Mutex::new(Vec::new()),
15786 model_id: schema.id.clone(),
15787 allocation_id: worker_allocation_id.clone(),
15788 resident: std::sync::atomic::AtomicBool::new(true),
15789 coordinator: coordinator.clone(),
15790 });
15791 crate::offload::set_local_offload(Some(offload.clone()));
15792
15793 let result = engine
15794 .reserve_local_request_with_worker_retry(&schema, 9_000)
15795 .await;
15796 crate::offload::set_local_offload(None);
15797
15798 assert!(
15799 result.is_ok(),
15800 "an unrelated resident allocation must not block the worker retry"
15801 );
15802 assert_eq!(
15803 coordinator.resident_allocation_ids(&schema.id),
15804 vec![sibling_allocation_id]
15805 );
15806 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 2);
15807 assert_eq!(
15808 *offload.calls.lock().unwrap(),
15809 vec![format!("release:{}", schema.id)]
15810 );
15811 }
15812
15813 struct UnretiredReleaseOffload {
15814 calls: std::sync::Mutex<Vec<String>>,
15815 model_id: String,
15816 allocation_id: String,
15817 resident: std::sync::atomic::AtomicBool,
15818 }
15819
15820 #[async_trait::async_trait]
15821 impl crate::offload::LocalGenerationOffload for UnretiredReleaseOffload {
15822 async fn generate(
15823 &self,
15824 _request: GenerateRequest,
15825 ) -> Result<InferenceResult, InferenceError> {
15826 unreachable!("unretired-release fixture")
15827 }
15828
15829 async fn stream(
15830 &self,
15831 _request: GenerateRequest,
15832 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15833 unreachable!("unretired-release fixture")
15834 }
15835
15836 async fn resident_models(&self) -> Vec<String> {
15837 self.resident
15838 .load(std::sync::atomic::Ordering::Acquire)
15839 .then(|| self.model_id.clone())
15840 .into_iter()
15841 .collect()
15842 }
15843
15844 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15845 Some(self.allocation_id.clone())
15846 }
15847
15848 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15849 self.calls
15850 .lock()
15851 .unwrap()
15852 .push(format!("release:{model_id}"));
15853 self.resident
15854 .store(false, std::sync::atomic::Ordering::Release);
15855 Ok(true)
15856 }
15857 }
15858
15859 #[tokio::test]
15860 async fn release_and_retry_rejects_ack_without_accounting_retirement() {
15861 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15862 let warnings_before =
15863 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
15864 let root = tempfile::tempdir().unwrap();
15865 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15866 let schema = crate::registry::builtin_catalog()
15867 .into_iter()
15868 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15869 .unwrap();
15870 let allocation_id = format!("worker:phantom:{}", schema.id);
15871 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15872 let coordinator = Arc::new(
15873 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15874 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15875 metal_mac_for_fit(32),
15876 Arc::new(SequencedRetryProbe {
15877 calls: probe_calls.clone(),
15878 first_available_mb: 1_000,
15879 subsequent_available_mb: 8_000,
15880 }),
15881 ),
15882 );
15883 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
15884 engine.local_admission = coordinator;
15885 let offload = Arc::new(UnretiredReleaseOffload {
15886 calls: std::sync::Mutex::new(Vec::new()),
15887 model_id: schema.id.clone(),
15888 allocation_id,
15889 resident: std::sync::atomic::AtomicBool::new(true),
15890 });
15891 crate::offload::set_local_offload(Some(offload.clone()));
15892
15893 let result = engine
15894 .reserve_local_request_with_worker_retry(&schema, 9_000)
15895 .await;
15896 crate::offload::set_local_offload(None);
15897
15898 assert!(matches!(
15899 result,
15900 Err(InferenceError::LocalResourceBlocked { preflight, .. })
15901 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
15902 ));
15903 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
15904 assert_eq!(
15905 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
15906 warnings_before + 1,
15907 "the refused retry must emit the stale-residency warning"
15908 );
15909 assert!(engine.local_admission.is_resident(&schema.id));
15910 assert!(offload.resident_models().await.is_empty());
15911 assert_eq!(
15912 *offload.calls.lock().unwrap(),
15913 vec![format!("release:{}", schema.id)]
15914 );
15915 }
15916
15917 struct PendingTeardownAckOffload {
15918 calls: std::sync::Mutex<Vec<String>>,
15919 model_id: String,
15920 allocation_id: String,
15921 resident: std::sync::atomic::AtomicBool,
15922 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
15923 }
15924
15925 #[async_trait::async_trait]
15926 impl crate::offload::LocalGenerationOffload for PendingTeardownAckOffload {
15927 async fn generate(
15928 &self,
15929 _request: GenerateRequest,
15930 ) -> Result<InferenceResult, InferenceError> {
15931 unreachable!("pending-teardown ACK fixture")
15932 }
15933
15934 async fn stream(
15935 &self,
15936 _request: GenerateRequest,
15937 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
15938 unreachable!("pending-teardown ACK fixture")
15939 }
15940
15941 async fn resident_models(&self) -> Vec<String> {
15942 self.resident
15943 .load(std::sync::atomic::Ordering::Acquire)
15944 .then(|| self.model_id.clone())
15945 .into_iter()
15946 .collect()
15947 }
15948
15949 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
15950 Some(self.allocation_id.clone())
15951 }
15952
15953 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
15954 self.calls
15955 .lock()
15956 .unwrap()
15957 .push(format!("release:{model_id}"));
15958 self.coordinator
15959 .mark_teardown_pending_allocation_with_charge(
15960 model_id,
15961 &self.allocation_id,
15962 4_800 * 1024 * 1024,
15963 );
15964 self.resident
15965 .store(false, std::sync::atomic::Ordering::Release);
15966 Ok(true)
15967 }
15968 }
15969
15970 #[tokio::test]
15971 async fn release_and_retry_rejects_ack_with_exact_worker_pending_teardown() {
15972 let _offload_guard = crate::offload::test_offload_lock().lock().await;
15973 let warnings_before =
15974 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed);
15975 let root = tempfile::tempdir().unwrap();
15976 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
15977 let schema = crate::registry::builtin_catalog()
15978 .into_iter()
15979 .find(|model| model.id == "mlx/qwen3-8b:4bit")
15980 .unwrap();
15981 let allocation_id = format!("worker:pending-test:{}", schema.id);
15982 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
15983 let coordinator = Arc::new(
15984 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
15985 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
15986 metal_mac_for_fit(32),
15987 Arc::new(SequencedRetryProbe {
15988 calls: probe_calls.clone(),
15989 first_available_mb: 1_000,
15990 subsequent_available_mb: 8_000,
15991 }),
15992 ),
15993 );
15994 engine.local_admission = coordinator.clone();
15995 let offload = Arc::new(PendingTeardownAckOffload {
15996 calls: std::sync::Mutex::new(Vec::new()),
15997 model_id: schema.id.clone(),
15998 allocation_id: allocation_id.clone(),
15999 resident: std::sync::atomic::AtomicBool::new(true),
16000 coordinator: coordinator.clone(),
16001 });
16002 crate::offload::set_local_offload(Some(offload.clone()));
16003
16004 let result = engine
16005 .reserve_local_request_with_worker_retry(&schema, 9_000)
16006 .await;
16007 crate::offload::set_local_offload(None);
16008
16009 assert!(matches!(
16010 result,
16011 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16012 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16013 ));
16014 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16015 assert_eq!(
16016 UNRETIRED_RELEASE_WARNING_COUNT.load(std::sync::atomic::Ordering::Relaxed),
16017 warnings_before + 1,
16018 "the pending exact allocation must emit the fail-closed warning"
16019 );
16020 assert_eq!(
16021 coordinator.resident_allocation_ids(&schema.id),
16022 vec![allocation_id]
16023 );
16024 assert!(offload.resident_models().await.is_empty());
16025 assert_eq!(
16026 *offload.calls.lock().unwrap(),
16027 vec![format!("release:{}", schema.id)]
16028 );
16029 }
16030
16031 struct MissingAllocationIdOffload {
16032 calls: std::sync::Mutex<Vec<String>>,
16033 model_id: String,
16034 }
16035
16036 #[async_trait::async_trait]
16037 impl crate::offload::LocalGenerationOffload for MissingAllocationIdOffload {
16038 async fn generate(
16039 &self,
16040 _request: GenerateRequest,
16041 ) -> Result<InferenceResult, InferenceError> {
16042 unreachable!("missing allocation-id fixture")
16043 }
16044
16045 async fn stream(
16046 &self,
16047 _request: GenerateRequest,
16048 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16049 unreachable!("missing allocation-id fixture")
16050 }
16051
16052 async fn resident_models(&self) -> Vec<String> {
16053 vec![self.model_id.clone()]
16054 }
16055
16056 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16057 self.calls
16058 .lock()
16059 .unwrap()
16060 .push(format!("release:{model_id}"));
16061 Ok(true)
16062 }
16063 }
16064
16065 #[tokio::test]
16066 async fn release_and_retry_without_scoped_allocation_id_fails_closed() {
16067 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16068 let root = tempfile::tempdir().unwrap();
16069 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16070 let schema = crate::registry::builtin_catalog()
16071 .into_iter()
16072 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16073 .unwrap();
16074 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16075 engine.local_admission = Arc::new(
16076 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16077 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16078 metal_mac_for_fit(32),
16079 Arc::new(SequencedRetryProbe {
16080 calls: probe_calls.clone(),
16081 first_available_mb: 1_000,
16082 subsequent_available_mb: 8_000,
16083 }),
16084 ),
16085 );
16086 let offload = Arc::new(MissingAllocationIdOffload {
16087 calls: std::sync::Mutex::new(Vec::new()),
16088 model_id: schema.id.clone(),
16089 });
16090 crate::offload::set_local_offload(Some(offload.clone()));
16091
16092 let result = engine
16093 .reserve_local_request_with_worker_retry(&schema, 9_000)
16094 .await;
16095 crate::offload::set_local_offload(None);
16096
16097 assert!(matches!(
16098 result,
16099 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16100 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16101 ));
16102 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16103 assert!(offload.calls.lock().unwrap().is_empty());
16104 }
16105
16106 struct ReplacingResidentOffload {
16107 calls: std::sync::Mutex<Vec<String>>,
16108 model_id: String,
16109 allocation_id: String,
16110 coordinator: Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16111 }
16112
16113 #[async_trait::async_trait]
16114 impl crate::offload::LocalGenerationOffload for ReplacingResidentOffload {
16115 async fn generate(
16116 &self,
16117 _request: GenerateRequest,
16118 ) -> Result<InferenceResult, InferenceError> {
16119 unreachable!("replacement-resident fixture")
16120 }
16121
16122 async fn stream(
16123 &self,
16124 _request: GenerateRequest,
16125 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16126 unreachable!("replacement-resident fixture")
16127 }
16128
16129 async fn resident_models(&self) -> Vec<String> {
16130 vec![self.model_id.clone()]
16131 }
16132
16133 fn resident_allocation_id(&self, _model_id: &str) -> Option<String> {
16134 Some(self.allocation_id.clone())
16135 }
16136
16137 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16138 self.calls
16139 .lock()
16140 .unwrap()
16141 .push(format!("release:{model_id}"));
16142 self.coordinator
16146 .mark_teardown_pending_allocation(model_id, &self.allocation_id);
16147 self.coordinator
16148 .finish_teardown_allocation(model_id, &self.allocation_id);
16149 self.coordinator
16150 .mark_resident_allocation(model_id, &self.allocation_id, 4_800);
16151 Ok(true)
16152 }
16153 }
16154
16155 #[tokio::test]
16156 async fn release_and_retry_same_id_replacement_blocks_retry_without_erasing_accounting() {
16157 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16158 let root = tempfile::tempdir().unwrap();
16159 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16160 let schema = crate::registry::builtin_catalog()
16161 .into_iter()
16162 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16163 .unwrap();
16164 let allocation_id = format!("worker:reused-scope:{}", schema.id);
16165 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16166 let coordinator = Arc::new(
16167 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16168 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16169 metal_mac_for_fit(32),
16170 Arc::new(SequencedRetryProbe {
16171 calls: probe_calls.clone(),
16172 first_available_mb: 1_000,
16173 subsequent_available_mb: 8_000,
16174 }),
16175 ),
16176 );
16177 coordinator.mark_resident_allocation(&schema.id, &allocation_id, 4_800);
16178 engine.local_admission = coordinator.clone();
16179 let offload = Arc::new(ReplacingResidentOffload {
16180 calls: std::sync::Mutex::new(Vec::new()),
16181 model_id: schema.id.clone(),
16182 allocation_id,
16183 coordinator,
16184 });
16185 crate::offload::set_local_offload(Some(offload.clone()));
16186
16187 let result = engine
16188 .reserve_local_request_with_worker_retry(&schema, 9_000)
16189 .await;
16190 crate::offload::set_local_offload(None);
16191
16192 assert!(matches!(
16193 result,
16194 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16195 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16196 ));
16197 assert!(engine.local_admission.is_resident(&schema.id));
16198 assert_eq!(
16199 *offload.calls.lock().unwrap(),
16200 vec![format!("release:{}", schema.id)]
16201 );
16202 assert_eq!(
16203 probe_calls.load(std::sync::atomic::Ordering::Relaxed),
16204 1,
16205 "replacement residency must stop the retry before a second probe"
16206 );
16207 }
16208
16209 #[tokio::test]
16210 async fn release_and_retry_nonresident_worker_returns_original_refusal() {
16211 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16212 let root = tempfile::tempdir().unwrap();
16213 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16214 let schema = crate::registry::builtin_catalog()
16215 .into_iter()
16216 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16217 .unwrap();
16218 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16219 engine.local_admission = Arc::new(
16220 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16221 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16222 metal_mac_for_fit(32),
16223 Arc::new(SequencedRetryProbe {
16224 calls: probe_calls.clone(),
16225 first_available_mb: 1_000,
16226 subsequent_available_mb: 1_048_576,
16227 }),
16228 ),
16229 );
16230 let offload = Arc::new(ReleaseRetryOffload {
16231 calls: std::sync::Mutex::new(Vec::new()),
16232 model_id: schema.id.clone(),
16233 resident: false,
16234 release_acknowledged: true,
16235 });
16236 crate::offload::set_local_offload(Some(offload.clone()));
16237
16238 let result = engine
16239 .reserve_local_request_with_worker_retry(&schema, 9_000)
16240 .await;
16241 crate::offload::set_local_offload(None);
16242
16243 assert!(matches!(
16244 result,
16245 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16246 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16247 ));
16248 assert!(offload.calls.lock().unwrap().is_empty());
16249 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16250 }
16251
16252 #[tokio::test]
16253 async fn release_and_retry_unacknowledged_release_does_not_reprobe() {
16254 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16255 let root = tempfile::tempdir().unwrap();
16256 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16257 let schema = crate::registry::builtin_catalog()
16258 .into_iter()
16259 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16260 .unwrap();
16261 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16262 engine.local_admission = Arc::new(
16263 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16264 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16265 metal_mac_for_fit(32),
16266 Arc::new(SequencedRetryProbe {
16267 calls: probe_calls.clone(),
16268 first_available_mb: 1_000,
16269 subsequent_available_mb: 1_048_576,
16270 }),
16271 ),
16272 );
16273 let offload = Arc::new(ReleaseRetryOffload {
16274 calls: std::sync::Mutex::new(Vec::new()),
16275 model_id: schema.id.clone(),
16276 resident: true,
16277 release_acknowledged: false,
16278 });
16279 crate::offload::set_local_offload(Some(offload.clone()));
16280
16281 let result = engine
16282 .reserve_local_request_with_worker_retry(&schema, 9_000)
16283 .await;
16284 crate::offload::set_local_offload(None);
16285
16286 assert!(matches!(
16287 result,
16288 Err(InferenceError::LocalResourceBlocked { preflight, .. })
16289 if preflight.verdict == resource_policy::LocalLoadVerdict::InsufficientLiveMemory
16290 ));
16291 assert_eq!(
16292 *offload.calls.lock().unwrap(),
16293 vec![format!("release:{}", schema.id)]
16294 );
16295 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16296 }
16297
16298 #[tokio::test]
16299 async fn release_and_retry_allowed_first_verdict_never_releases() {
16300 let _offload_guard = crate::offload::test_offload_lock().lock().await;
16301 let root = tempfile::tempdir().unwrap();
16302 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
16303 let schema = crate::registry::builtin_catalog()
16304 .into_iter()
16305 .find(|model| model.id == "mlx/qwen3-8b:4bit")
16306 .unwrap();
16307 let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16308 engine.local_admission = Arc::new(
16309 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16310 crate::resource_policy::ResourcePolicy::local_focused(),
16311 metal_mac_for_fit(64),
16312 Arc::new(SequencedRetryProbe {
16313 calls: probe_calls.clone(),
16314 first_available_mb: 1_048_576,
16315 subsequent_available_mb: 1_048_576,
16316 }),
16317 ),
16318 );
16319 let offload = Arc::new(ReleaseRetryOffload {
16320 calls: std::sync::Mutex::new(Vec::new()),
16321 model_id: schema.id.clone(),
16322 resident: true,
16323 release_acknowledged: true,
16324 });
16325 crate::offload::set_local_offload(Some(offload.clone()));
16326
16327 let result = engine
16328 .reserve_local_request_with_worker_retry(&schema, 9_000)
16329 .await;
16330 crate::offload::set_local_offload(None);
16331
16332 assert!(result.is_ok());
16333 assert!(offload.calls.lock().unwrap().is_empty());
16334 assert_eq!(probe_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
16335 }
16336
16337 struct ReconcileOffload {
16338 calls: std::sync::Mutex<Vec<String>>,
16339 release_acknowledged: bool,
16340 }
16341
16342 #[async_trait::async_trait]
16343 impl crate::offload::LocalGenerationOffload for ReconcileOffload {
16344 async fn generate(
16345 &self,
16346 _request: GenerateRequest,
16347 ) -> Result<InferenceResult, InferenceError> {
16348 unreachable!("residency reconciliation fixture")
16349 }
16350
16351 async fn stream(
16352 &self,
16353 _request: GenerateRequest,
16354 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16355 unreachable!("residency reconciliation fixture")
16356 }
16357
16358 fn resident_allocation_id(&self, model_id: &str) -> Option<String> {
16359 self.calls
16360 .lock()
16361 .unwrap()
16362 .push(format!("allocation:{model_id}"));
16363 Some(format!("worker:{model_id}:generation-7"))
16364 }
16365
16366 async fn release_model(&self, model_id: &str) -> Result<bool, InferenceError> {
16367 self.calls
16368 .lock()
16369 .unwrap()
16370 .push(format!("release:{model_id}"));
16371 Ok(self.release_acknowledged)
16372 }
16373 }
16374
16375 fn reconciliation_reservation() -> (
16376 Arc<crate::resource_policy::LocalAdmissionCoordinator>,
16377 crate::resource_policy::LocalLoadReservation,
16378 ) {
16379 struct FixedProbe;
16380 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
16381 fn available_memory_mb(
16382 &self,
16383 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
16384 Ok(Some(24_000))
16385 }
16386 }
16387 let coordinator = Arc::new(
16388 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
16389 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
16390 crate::hardware::HardwareInfo {
16391 total_ram_mb: 32 * 1024,
16392 ..crate::hardware::HardwareInfo::detect()
16393 },
16394 Arc::new(FixedProbe),
16395 ),
16396 );
16397 let reservation = coordinator
16398 .reserve_measured_host_allocation(
16399 "expected/model",
16400 "expected/model#request",
16401 1024 * 1024,
16402 0,
16403 )
16404 .unwrap();
16405 (coordinator, reservation)
16406 }
16407
16408 #[tokio::test]
16409 async fn worker_residency_reconciliation_releases_mismatched_ack_before_returning_error() {
16410 let offload = ReconcileOffload {
16411 calls: std::sync::Mutex::new(Vec::new()),
16412 release_acknowledged: true,
16413 };
16414 let (_coordinator, mut reservation) = reconciliation_reservation();
16415 let error = InferenceEngine::reconcile_worker_residency(
16416 &offload,
16417 "expected/model",
16418 &crate::offload::LocalWorkerResidency {
16419 model_id: "wrong/model".into(),
16420 measured_weights_bytes: 2 * 1024 * 1024,
16421 },
16422 backend_cache::BackendRetention::Resident,
16423 &mut reservation,
16424 )
16425 .await
16426 .unwrap_err();
16427 assert!(error.to_string().contains("wrong/model"));
16428 assert_eq!(
16429 *offload.calls.lock().unwrap(),
16430 vec!["release:wrong/model".to_string()]
16431 );
16432 }
16433
16434 #[tokio::test]
16435 async fn worker_residency_reconciliation_publishes_exact_owner_for_matching_ack() {
16436 let offload = ReconcileOffload {
16437 calls: std::sync::Mutex::new(Vec::new()),
16438 release_acknowledged: false,
16439 };
16440 let (coordinator, mut reservation) = reconciliation_reservation();
16441 InferenceEngine::reconcile_worker_residency(
16442 &offload,
16443 "expected/model",
16444 &crate::offload::LocalWorkerResidency {
16445 model_id: "expected/model".into(),
16446 measured_weights_bytes: 2 * 1024 * 1024,
16447 },
16448 backend_cache::BackendRetention::Resident,
16449 &mut reservation,
16450 )
16451 .await
16452 .unwrap();
16453 drop(reservation);
16454 assert_eq!(
16455 *offload.calls.lock().unwrap(),
16456 vec!["allocation:expected/model".to_string()]
16457 );
16458 assert!(coordinator.is_resident("expected/model"));
16459 }
16460
16461 #[async_trait::async_trait]
16462 impl crate::offload::LocalGenerationOffload for FixtureLocalOffload {
16463 async fn generate(
16464 &self,
16465 _request: GenerateRequest,
16466 ) -> Result<InferenceResult, InferenceError> {
16467 unreachable!("streaming fixture")
16468 }
16469
16470 async fn stream(
16471 &self,
16472 _request: GenerateRequest,
16473 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
16474 unreachable!("admission-aware streaming fixture")
16475 }
16476
16477 async fn stream_admitted(
16478 &self,
16479 request: GenerateRequest,
16480 admission: crate::offload::LocalWorkerAdmission,
16481 ) -> Result<crate::offload::LocalOffloadStream, InferenceError> {
16482 let (tx, rx) = tokio::sync::mpsc::channel(4);
16483 let emit_done = self.emit_done;
16484 tokio::spawn(async move {
16485 let _ = tx.send(StreamEvent::TextDelta("local answer".into())).await;
16486 if emit_done {
16487 let _ = tx
16488 .send(StreamEvent::Done {
16489 text: "local answer".into(),
16490 tool_calls: vec![],
16491 })
16492 .await;
16493 }
16494 });
16495 Ok(crate::offload::LocalOffloadStream {
16496 events: rx,
16497 residency: crate::offload::LocalWorkerResidency {
16498 model_id: request.model.unwrap_or_else(|| "fixture/local".into()),
16499 measured_weights_bytes: admission.measured_weights_bytes,
16500 },
16501 retention: backend_cache::BackendRetention::Resident,
16502 })
16503 }
16504 }
16505
16506 fn pin_test_local_admission_memory(engine: &mut InferenceEngine) {
16514 let policy = engine.local_admission.policy();
16515 let mut hardware = HardwareInfo::detect();
16516 hardware.total_ram_mb = hardware.total_ram_mb.max(16 * 1024);
16517 hardware.max_model_mb = hardware.max_model_mb.max(16 * 1024);
16518 engine.local_admission = Arc::new(resource_policy::LocalAdmissionCoordinator::with_probe(
16519 policy,
16520 hardware,
16521 Arc::new(resource_policy::FixedLiveMemoryProbe::known(1_048_576)),
16522 ));
16523 }
16524
16525 fn install_small_local_fixture(engine: &InferenceEngine) -> String {
16526 let schema = engine
16527 .unified_registry()
16528 .find_by_name("Qwen3-0.6B")
16529 .expect("small built-in local model")
16530 .clone();
16531 let model_dir = engine.config.models_dir.join(&schema.name);
16532 std::fs::create_dir_all(&model_dir).unwrap();
16533 std::fs::write(model_dir.join("model.gguf"), b"fixture").unwrap();
16534 std::fs::write(model_dir.join("tokenizer.json"), b"{}").unwrap();
16535 schema.id
16536 }
16537
16538 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16539 fn install_exact_pin_equivalent_fixture(engine: &InferenceEngine) -> String {
16540 let gguf = engine
16541 .unified_registry()
16542 .get("qwen/qwen3-0.6b:q8_0")
16543 .expect("GGUF fixture row")
16544 .clone();
16545 let mlx = engine
16546 .unified_registry()
16547 .get("mlx/qwen3-0.6b:6bit")
16548 .expect("MLX equivalent fixture row")
16549 .clone();
16550
16551 let gguf_dir = engine.config.models_dir.join(&gguf.name);
16552 std::fs::create_dir_all(&gguf_dir).unwrap();
16553 std::fs::write(gguf_dir.join("model.gguf"), b"fixture").unwrap();
16554 std::fs::write(gguf_dir.join("tokenizer.json"), b"{}").unwrap();
16555
16556 let mlx_dir = engine.config.models_dir.join(&mlx.name);
16557 std::fs::create_dir_all(&mlx_dir).unwrap();
16558 std::fs::write(mlx_dir.join("config.json"), b"{}").unwrap();
16559 std::fs::write(mlx_dir.join("model.safetensors"), b"fixture").unwrap();
16560
16561 gguf.id
16562 }
16563
16564 fn remote_stream_fixture_schema(
16565 id: &str,
16566 endpoint: String,
16567 protocol: schema::ApiProtocol,
16568 api_key_env: &str,
16569 ) -> ModelSchema {
16570 ModelSchema {
16571 id: id.into(),
16572 name: "gemini-test".into(),
16573 provider: "test".into(),
16574 family: "test".into(),
16575 version: "1".into(),
16576 capabilities: vec![ModelCapability::Generate],
16577 context_length: 128_000,
16578 max_output_tokens: Some(8_192),
16579 param_count: String::new(),
16580 quantization: None,
16581 performance: Default::default(),
16582 cost: Default::default(),
16583 source: ModelSource::RemoteApi {
16584 endpoint,
16585 api_key_env: api_key_env.into(),
16586 api_key_envs: vec![],
16587 api_version: None,
16588 protocol,
16589 },
16590 tags: vec!["test".into()],
16591 supported_params: vec![],
16592 public_benchmarks: vec![],
16593 trust_tier: TrustTier::Community,
16594 deprecated: false,
16595 available: true,
16596 weights_ready: true,
16597 }
16598 }
16599
16600 async fn assert_remote_model_identity_contract(
16601 protocol: schema::ApiProtocol,
16602 model_id: &str,
16603 provider_model: &str,
16604 api_key_env: &str,
16605 ) {
16606 use wiremock::matchers::{method, path};
16607 use wiremock::{Mock, MockServer, ResponseTemplate};
16608
16609 let server = MockServer::start().await;
16610 let (endpoint_path, response) = match protocol {
16611 schema::ApiProtocol::OpenAiCompat => (
16612 "/v1/chat/completions",
16613 serde_json::json!({
16614 "choices": [{
16615 "message": {"content": "openai exact"},
16616 "finish_reason": "stop"
16617 }],
16618 "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}
16619 }),
16620 ),
16621 schema::ApiProtocol::Anthropic => (
16622 "/v1/messages",
16623 serde_json::json!({
16624 "content": [{"type": "text", "text": "anthropic exact"}],
16625 "stop_reason": "end_turn",
16626 "usage": {"input_tokens": 2, "output_tokens": 2}
16627 }),
16628 ),
16629 _ => unreachable!("identity regression covers the newsroom's two remote providers"),
16630 };
16631 Mock::given(method("POST"))
16632 .and(path(endpoint_path))
16633 .respond_with(ResponseTemplate::new(200).set_body_json(response))
16634 .mount(&server)
16635 .await;
16636 unsafe { std::env::set_var(api_key_env, "fixture") };
16637
16638 let tmp = TempDir::new().unwrap();
16639 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
16640 let mut schema =
16641 remote_stream_fixture_schema(model_id, server.uri(), protocol, api_key_env);
16642 schema.name = provider_model.into();
16643 engine.register_model(schema);
16644
16645 let mut exact_request = GenerateRequest {
16646 prompt: "return the exact identity".into(),
16647 ..Default::default()
16648 };
16649 pin_exact_model_id(&mut exact_request, model_id.into()).unwrap();
16650 let exact = engine.generate_tracked(exact_request).await.unwrap();
16651 assert_eq!(exact.model_used, model_id);
16652 assert_eq!(
16653 exact.model_identity.requested_model_id.as_deref(),
16654 Some(model_id)
16655 );
16656 assert_eq!(exact.model_identity.resolved_model_id, model_id);
16657 let exact_envelope = serde_json::to_value(&exact).unwrap();
16658 assert_eq!(exact_envelope["model_used"], model_id);
16659 assert_eq!(exact_envelope["requested_model_id"], model_id);
16660 assert_eq!(exact_envelope["resolved_model_id"], model_id);
16661 assert!(exact_envelope["row_digest"].is_string());
16662 assert!(exact_envelope["catalog_revision"].is_string());
16663
16664 let loose = engine
16665 .generate_tracked(GenerateRequest {
16666 prompt: "keep legacy display-name routing".into(),
16667 model: Some(provider_model.into()),
16668 params: GenerateParams {
16669 strict_model: true,
16670 ..Default::default()
16671 },
16672 ..Default::default()
16673 })
16674 .await
16675 .unwrap();
16676 assert_eq!(loose.model_used, provider_model);
16677 assert_eq!(loose.model_identity.requested_model_id, None);
16678 assert_eq!(loose.model_identity.resolved_model_id, model_id);
16679
16680 let requests = server.received_requests().await.unwrap();
16681 assert_eq!(requests.len(), 2);
16682 for request in requests {
16683 let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
16684 assert_eq!(body["model"], provider_model);
16685 }
16686
16687 unsafe { std::env::remove_var(api_key_env) };
16688 }
16689
16690 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
16696 #[tokio::test]
16697 async fn mlx_device_lock_is_singleton_and_serializes() {
16698 use std::sync::atomic::{AtomicUsize, Ordering};
16699 use std::sync::Arc;
16700 assert!(
16702 Arc::ptr_eq(
16703 &InferenceEngine::mlx_device_lock(),
16704 &InferenceEngine::mlx_device_lock()
16705 ),
16706 "device lock must be a process-wide singleton"
16707 );
16708 let inside = Arc::new(AtomicUsize::new(0));
16710 let peak = Arc::new(AtomicUsize::new(0));
16711 let mut handles = Vec::new();
16712 for _ in 0..8 {
16713 let inside = inside.clone();
16714 let peak = peak.clone();
16715 handles.push(tokio::spawn(async move {
16716 let _g = InferenceEngine::mlx_device_lock().lock_owned().await;
16717 let n = inside.fetch_add(1, Ordering::SeqCst) + 1;
16718 peak.fetch_max(n, Ordering::SeqCst);
16719 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
16720 inside.fetch_sub(1, Ordering::SeqCst);
16721 }));
16722 }
16723 for h in handles {
16724 h.await.unwrap();
16725 }
16726 assert_eq!(
16727 peak.load(Ordering::SeqCst),
16728 1,
16729 "at most one MLX device holder at a time"
16730 );
16731 }
16732
16733 #[test]
16738 fn auto_thinking_budget_gates_code_complex_and_capability() {
16739 assert_eq!(auto_thinking_budget(true, false, true), Some(24_000));
16741 assert_eq!(auto_thinking_budget(false, true, true), Some(8_000));
16743 assert_eq!(auto_thinking_budget(true, true, true), Some(24_000));
16745 assert_eq!(auto_thinking_budget(true, true, false), None);
16747 assert_eq!(auto_thinking_budget(false, false, true), None);
16749 }
16750
16751 #[test]
16752 fn is_explicit_code_intent_keys_on_caller_intent_not_keyword_classifier() {
16753 use crate::intent::{IntentHint, TaskHint};
16754 let code = IntentHint {
16756 task: Some(TaskHint::Code),
16757 ..Default::default()
16758 };
16759 assert!(is_explicit_code_intent(Some(&code)));
16760 assert!(!is_explicit_code_intent(None));
16765 let reasoning = IntentHint {
16767 task: Some(TaskHint::Reasoning),
16768 ..Default::default()
16769 };
16770 assert!(!is_explicit_code_intent(Some(&reasoning)));
16771 let unset = IntentHint {
16773 task: None,
16774 ..Default::default()
16775 };
16776 assert!(!is_explicit_code_intent(Some(&unset)));
16777 }
16778
16779 #[test]
16780 fn strict_model_suppresses_the_local_last_resort_append() {
16781 assert!(should_append_local_last_resort(false, false));
16783 assert!(!should_append_local_last_resort(false, true));
16786 assert!(!should_append_local_last_resort(true, false));
16789 assert!(!should_append_local_last_resort(true, true));
16790 }
16791
16792 #[test]
16793 fn last_resort_fallback_never_returns_an_unrunnable_apple_foundation() {
16794 let tmp = TempDir::new().unwrap();
16809 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
16810 if let Some(name) = engine.first_installed_local_model(false) {
16811 let runnable = engine
16812 .unified_registry()
16813 .find_by_name(&name)
16814 .map(|s| !s.is_foundation_models() || s.available)
16815 .unwrap_or(false);
16816 assert!(
16817 runnable,
16818 "last-resort fallback returned a non-runnable model: {name}"
16819 );
16820 }
16821 }
16822
16823 #[test]
16824 fn empty_tool_catalog_is_no_tools() {
16825 let mut req = GenerateRequest {
16832 prompt: "p".into(),
16833 ..Default::default()
16834 };
16835 assert!(!InferenceEngine::request_has_tools(&req));
16836 req.tools = Some(vec![]);
16837 assert!(!InferenceEngine::request_has_tools(&req));
16838 req.tools = Some(vec![serde_json::json!({
16839 "name": "t", "description": "d", "parameters": {"type": "object"}
16840 })]);
16841 assert!(InferenceEngine::request_has_tools(&req));
16842 }
16843
16844 #[test]
16845 fn top_k_keeps_only_k_highest() {
16846 let mut probs = vec![0.1, 0.4, 0.2, 0.25, 0.05];
16848 InferenceEngine::apply_top_k_top_p(&mut probs, 2, 1.0);
16849 assert!(probs[0] == 0.0 && probs[2] == 0.0 && probs[4] == 0.0);
16851 assert!(probs[1] > 0.0 && probs[3] > 0.0);
16852 let sum: f32 = probs.iter().sum();
16853 assert!((sum - 1.0).abs() < 1e-5, "renormalized to 1.0, got {sum}");
16854 }
16855
16856 #[test]
16857 fn top_k_zero_is_a_noop() {
16858 let mut probs = vec![0.1, 0.4, 0.2, 0.3];
16859 let before = probs.clone();
16860 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 1.0);
16861 assert_eq!(probs, before);
16862 }
16863
16864 #[test]
16865 fn top_p_nucleus_truncates_tail() {
16866 let mut probs = vec![0.6, 0.3, 0.07, 0.03];
16867 InferenceEngine::apply_top_k_top_p(&mut probs, 0, 0.9);
16868 assert!(probs[2] == 0.0 && probs[3] == 0.0);
16870 assert!(probs[0] > 0.0 && probs[1] > 0.0);
16871 }
16872
16873 #[test]
16874 fn truncate_at_stop_excludes_stop_sequence() {
16875 let stops = vec!["<|end|>".to_string(), "STOP".to_string()];
16876 assert_eq!(
16877 tasks::generate::truncate_at_stop("hello world<|end|>extra", &stops),
16878 "hello world"
16879 );
16880 assert_eq!(
16882 tasks::generate::truncate_at_stop("aSTOPb<|end|>c", &stops),
16883 "a"
16884 );
16885 assert_eq!(
16887 tasks::generate::truncate_at_stop("clean output", &stops),
16888 "clean output"
16889 );
16890 assert_eq!(
16892 tasks::generate::truncate_at_stop("text", &["".to_string()]),
16893 "text"
16894 );
16895 }
16896
16897 #[test]
16898 fn no_backend_hint_fires_on_missing_backend_phrases() {
16899 for phrase in [
16901 "no credential for proprietary provider 'parslee'",
16902 "model not found",
16903 "no models available",
16904 "model declares ModelSource::Delegated but no inference runner is registered",
16905 ] {
16906 let hint = no_backend_recovery_hint(phrase)
16907 .unwrap_or_else(|| panic!("expected a hint for {phrase:?}"));
16908 assert!(hint.contains("car models pull"));
16909 assert!(hint.contains("car auth login"));
16912 assert!(hint.contains(phrase));
16914 }
16915 }
16916
16917 #[test]
16918 fn no_backend_hint_opens_with_the_exported_marker() {
16919 let hint = no_backend_recovery_hint(
16923 "model declares ModelSource::Delegated but no inference runner is registered",
16924 )
16925 .expect("a delegated model with no runner is a no-backend case");
16926 assert!(
16927 hint.starts_with(NO_BACKEND_RECOVERY_MARKER),
16928 "hint must open with the exported marker: {hint}"
16929 );
16930 assert!(hint.contains(NO_BACKEND_RECOVERY_MARKER), "{hint}");
16931 }
16932
16933 #[test]
16934 fn route_failure_context_keeps_the_no_backend_marker_behind_the_credential_summary() {
16935 let credential = parslee_signed_out_route_failure();
16939 let underlying = InferenceError::InferenceFailed(
16940 "model declares ModelSource::Delegated but no inference runner is registered".into(),
16941 );
16942 let augmented = apply_route_failure_context(underlying, Some(&credential));
16943 let InferenceError::InferenceFailed(message) = &augmented else {
16944 panic!("a no-backend exhaustion stays InferenceFailed: {augmented:?}");
16945 };
16946 assert!(
16947 message.starts_with(&credential.summary),
16948 "credential context is prepended: {message}"
16949 );
16950 assert!(
16951 message.contains(NO_BACKEND_RECOVERY_MARKER),
16952 "the marker survives the prepend: {message}"
16953 );
16954 assert!(
16955 !message.starts_with(NO_BACKEND_RECOVERY_MARKER),
16956 "the marker is mid-message here, which is why consumers use contains: {message}"
16957 );
16958 }
16959
16960 #[test]
16968 fn credential_failure_is_matchable_as_data() {
16969 let expired = InferenceError::CredentialUnavailable {
16970 provider: "parslee".into(),
16971 model: "parslee/reasoning".into(),
16972 reason: CredentialFailure::Expired {
16973 expires_at: 1_754_257_929,
16974 },
16975 detail: "the Parslee token expired at unix 1754257929 and could not be refreshed"
16976 .into(),
16977 };
16978 let InferenceError::CredentialUnavailable { reason, .. } = &expired else {
16979 panic!("expected CredentialUnavailable");
16980 };
16981 assert_eq!(
16982 *reason,
16983 CredentialFailure::Expired {
16984 expires_at: 1_754_257_929
16985 },
16986 "a consumer must be able to branch on the expiry without parsing prose"
16987 );
16988 assert_ne!(
16992 CredentialFailure::SignedOut,
16993 CredentialFailure::StoreUnreadable
16994 );
16995 assert_ne!(
16996 CredentialFailure::SignedOut,
16997 CredentialFailure::Expired { expires_at: 0 }
16998 );
16999 assert_ne!(
17000 CredentialFailure::StoreUnreadable,
17001 CredentialFailure::RaceRetryable
17002 );
17003 }
17004
17005 #[test]
17015 fn typed_credential_error_still_satisfies_the_substring_classifiers() {
17016 for reason in [
17017 CredentialFailure::Expired { expires_at: 1 },
17018 CredentialFailure::SignedOut,
17019 CredentialFailure::StoreUnreadable,
17020 CredentialFailure::RaceRetryable,
17021 CredentialFailure::EnvVarMissing {
17022 env_var: "OPENAI_API_KEY".into(),
17023 },
17024 ] {
17025 let rendered = InferenceError::CredentialUnavailable {
17026 provider: "parslee".into(),
17027 model: "parslee/reasoning".into(),
17028 reason: reason.clone(),
17029 detail: "detail text".into(),
17030 }
17031 .to_string();
17032 assert!(
17034 rendered
17035 .to_ascii_lowercase()
17036 .contains("no credential for proprietary"),
17037 "classifier substring lost for {reason:?}: {rendered}"
17038 );
17039 assert!(rendered.contains("parslee/reasoning"), "{rendered}");
17041 assert!(rendered.contains("detail text"), "{rendered}");
17043 }
17044 }
17045
17046 #[test]
17047 fn auth_expired_hint_fires_on_auth_rejection_but_not_transient() {
17048 for phrase in [
17050 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17051 "HTTP 403: forbidden",
17052 "invalid_grant: The refresh token is invalid or expired",
17053 "token expired",
17054 ] {
17055 let hint = auth_expired_recovery_hint(phrase)
17056 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
17057 assert!(hint.contains("car auth login"));
17058 assert!(hint.contains(phrase));
17059 }
17060 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
17062 assert!(auth_expired_recovery_hint("request timed out").is_none());
17063 }
17064
17065 #[test]
17083 fn auth_expired_hint_never_carries_the_no_backend_marker() {
17084 for phrase in [
17087 "Parslee org lookup failed: the account service could not be reached",
17088 "Authentication required",
17089 "HTTP 401 Unauthorized",
17090 "HTTP 403 Forbidden",
17091 "invalid_grant: the refresh token is invalid or expired",
17092 "token expired",
17093 ] {
17094 let hint = auth_expired_recovery_hint(phrase)
17095 .unwrap_or_else(|| panic!("expected an auth hint for {phrase:?}"));
17096 assert!(
17097 !hint.contains(NO_BACKEND_RECOVERY_MARKER),
17098 "the auth-expired hint now carries the no-backend marker, so \
17099 car infer would auto-pull for {phrase:?}: {hint}"
17100 );
17101 assert!(hint.contains("car models pull"), "{hint}");
17105 }
17106 assert!(auth_expired_recovery_hint("API returned 503: service unavailable").is_none());
17109 }
17110
17111 #[test]
17116 fn auth_rejection_classifier_matches_the_real_expired_token_error() {
17117 assert!(is_auth_rejection_message(
17118 "inference failed: Parslee org lookup failed: HTTP 401 Unauthorized: \
17119 Authentication required"
17120 ));
17121 for transient in [
17124 "API returned 503: service unavailable",
17125 "request timed out",
17126 "connection reset by peer",
17127 ] {
17128 assert!(
17129 !is_auth_rejection_message(transient),
17130 "transient wrongly classified as an auth rejection: {transient:?}"
17131 );
17132 }
17133 }
17134
17135 #[test]
17143 fn typed_errors_classify_by_their_structure_not_their_prose() {
17144 use FallbackReason as R;
17145 let cases: Vec<(InferenceError, R)> = vec![
17146 (
17149 InferenceError::ProviderAccount {
17150 provider: "openai".into(),
17151 status: 401,
17152 message: "provider rejected the API key — check the configured credential"
17153 .into(),
17154 },
17155 R::CredentialRejected,
17156 ),
17157 (
17159 InferenceError::ProviderAccount {
17160 provider: "openrouter".into(),
17161 status: 402,
17162 message: "OpenRouter account is out of credits".into(),
17163 },
17164 R::QuotaExhausted,
17166 ),
17167 (
17168 InferenceError::CredentialUnavailable {
17169 provider: "parslee".into(),
17170 model: "parslee/reasoning".into(),
17171 reason: CredentialFailure::Expired { expires_at: 0 },
17172 detail: "session expired".into(),
17173 },
17174 R::CredentialRejected,
17175 ),
17176 (
17177 InferenceError::CredentialUnavailable {
17178 provider: "openai".into(),
17179 model: "openai/gpt-5.6".into(),
17180 reason: CredentialFailure::EnvVarMissing {
17181 env_var: "OPENAI_API_KEY".into(),
17182 },
17183 detail: "not set".into(),
17184 },
17185 R::CredentialAbsent,
17186 ),
17187 (
17190 InferenceError::CredentialUnavailable {
17191 provider: "parslee".into(),
17192 model: "parslee/reasoning".into(),
17193 reason: CredentialFailure::StoreUnreadable,
17194 detail: "the bounded Keychain helper timed out".into(),
17195 },
17196 R::Failed,
17197 ),
17198 (
17199 InferenceError::Transient {
17200 status: Some(429),
17201 message: "slow down".into(),
17202 },
17203 R::RateLimited,
17204 ),
17205 (
17210 InferenceError::Transient {
17211 status: None,
17212 message: "connection reset by peer".into(),
17213 },
17214 R::Failed,
17215 ),
17216 (
17217 InferenceError::Transient {
17218 status: Some(503),
17219 message: "upstream down".into(),
17220 },
17221 R::Failed,
17222 ),
17223 ];
17224 for (err, want) in cases {
17225 assert_eq!(classify_fallback_reason(&err), want, "{err}");
17226 }
17227 }
17228
17229 #[test]
17237 fn a_quoted_status_in_a_provider_body_does_not_decide_the_bucket() {
17238 use FallbackReason as R;
17239 for (msg, want) in [
17240 (
17241 "API returned 400 Bad Request: your last request 401'd upstream and was unauthorized",
17242 R::Failed,
17243 ),
17244 ("API returned 400 Bad Request: timeout param invalid", R::Failed),
17245 ("API returned 429 Too Many Requests: slow down", R::RateLimited),
17246 ("API returned 401 Unauthorized: bad key", R::CredentialRejected),
17247 ] {
17248 assert_eq!(
17249 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
17250 want,
17251 "{msg}"
17252 );
17253 }
17254 }
17255
17256 #[test]
17260 fn a_parslee_org_lookup_failure_is_classified_by_its_status() {
17261 use FallbackReason as R;
17262 for (msg, want) in [
17263 (
17264 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17265 R::CredentialRejected,
17266 ),
17267 (
17268 "Parslee org lookup failed: HTTP 429 Too Many Requests: slow down",
17269 R::RateLimited,
17270 ),
17271 (
17272 "Parslee org lookup failed: HTTP 500 Internal Server Error: boom",
17273 R::Failed,
17274 ),
17275 ] {
17276 assert_eq!(
17277 classify_fallback_reason(&InferenceError::InferenceFailed(msg.into())),
17278 want,
17279 "{msg}"
17280 );
17281 }
17282 }
17283
17284 #[test]
17288 fn every_skipped_lane_is_recorded_in_order() {
17289 let mut hops = Vec::new();
17290 for (cand, err) in [
17291 (
17292 "lane-one",
17293 InferenceError::Transient {
17294 status: Some(429),
17295 message: "x".into(),
17296 },
17297 ),
17298 (
17301 "lane-two",
17302 InferenceError::Transient {
17303 status: None,
17304 message: "connection refused".into(),
17305 },
17306 ),
17307 (
17308 "lane-three",
17309 InferenceError::ProviderAccount {
17310 provider: "openai".into(),
17311 status: 401,
17312 message: "rejected".into(),
17313 },
17314 ),
17315 ] {
17316 record_fallback_from(&mut hops, cand, &err);
17317 }
17318 assert_eq!(
17319 hops.iter()
17320 .map(|h| h.candidate.as_str())
17321 .collect::<Vec<_>>(),
17322 ["lane-one", "lane-two", "lane-three"]
17323 );
17324 assert_eq!(
17325 hops.iter().map(|h| h.reason).collect::<Vec<_>>(),
17326 [
17327 FallbackReason::RateLimited,
17328 FallbackReason::Failed,
17329 FallbackReason::CredentialRejected
17330 ]
17331 );
17332 }
17333
17334 #[test]
17342 fn a_refused_api_key_is_journaled_but_does_not_claim_sign_in_fixes_it() {
17343 let refused = InferenceError::ProviderAccount {
17344 provider: "openai".into(),
17345 status: 401,
17346 message: "provider rejected the API key — check the configured credential".into(),
17347 };
17348 let mut hops = Vec::new();
17349 record_fallback_from(&mut hops, "openai/gpt-5.6", &refused);
17350 assert_eq!(hops[0].reason, FallbackReason::CredentialRejected);
17351
17352 let mut auth = None;
17354 record_auth_dead_lane(&mut auth, "openai/gpt-5.6", &refused.to_string());
17355 assert_eq!(auth, None, "car auth login does not fix a bad OpenAI key");
17356 }
17357
17358 #[test]
17359 fn auth_dead_lane_records_first_rejected_candidate_only() {
17360 let mut slot: Option<String> = None;
17362 record_auth_dead_lane(
17363 &mut slot,
17364 "parslee/reasoning",
17365 "API returned 503: unavailable",
17366 );
17367 record_auth_dead_lane(&mut slot, "openai/gpt-5.6", "request timed out");
17368 assert_eq!(slot, None);
17369
17370 record_auth_dead_lane(
17372 &mut slot,
17373 "parslee/reasoning",
17374 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17375 );
17376 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
17377
17378 record_auth_dead_lane(&mut slot, "anthropic/claude", "HTTP 403: forbidden");
17381 assert_eq!(slot.as_deref(), Some("parslee/reasoning"));
17382 }
17383
17384 #[test]
17385 fn configured_provider_with_expired_token_is_named_before_a_local_oom() {
17386 let expired = InferenceError::CredentialUnavailable {
17387 provider: "parslee".into(),
17388 model: "parslee/reasoning".into(),
17389 reason: CredentialFailure::Expired { expires_at: 42 },
17390 detail: "access token expired".into(),
17391 };
17392 let mut credential = None;
17393 record_route_credential_failure(&mut credential, "parslee/reasoning", &expired, false);
17394
17395 let error = apply_route_failure_context(
17396 InferenceError::InferenceFailed(
17397 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17398 .into(),
17399 ),
17400 credential.as_ref(),
17401 )
17402 .to_string();
17403
17404 let auth_pos = error
17405 .find("Parslee login expired")
17406 .expect("expired login must be named");
17407 let remedy_pos = error
17408 .find("car auth login")
17409 .expect("credential remedy must be named");
17410 let oom_pos = error
17411 .find("9059 MB")
17412 .expect("fallback error must remain as secondary detail");
17413 assert!(auth_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
17414 assert!(is_auth_failure_message(&error), "{error}");
17415 }
17416
17417 #[test]
17418 fn absent_login_is_named_before_a_local_oom() {
17419 let failure = parslee_signed_out_route_failure();
17420 let error = apply_route_failure_context(
17421 InferenceError::InferenceFailed("local fallback needs 9059 MB".into()),
17422 Some(&failure),
17423 )
17424 .to_string();
17425
17426 let absent_pos = error
17427 .find("Parslee login is absent")
17428 .expect("missing login must be named");
17429 let remedy_pos = error
17430 .find("car auth login")
17431 .expect("credential remedy must be named");
17432 let oom_pos = error
17433 .find("9059 MB")
17434 .expect("fallback OOM must remain as secondary detail");
17435 assert!(absent_pos < remedy_pos && remedy_pos < oom_pos, "{error}");
17436 }
17437
17438 #[test]
17439 fn genuine_local_oom_is_not_reclassified_as_auth() {
17440 let oom = InferenceError::InferenceFailed(
17441 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17442 .into(),
17443 );
17444 let error = apply_route_failure_context(oom, None);
17445 assert!(matches!(error, InferenceError::InferenceFailed(ref message)
17446 if message.starts_with("This model needs about 9059 MB")));
17447 }
17448
17449 #[test]
17450 fn unconfigured_provider_is_not_surfaced_over_the_terminal_failure() {
17451 let mut credential = None;
17452 record_route_credential_failure(
17453 &mut credential,
17454 "openai/gpt-5.6",
17455 &InferenceError::CredentialUnavailable {
17456 provider: "openai".into(),
17457 model: "openai/gpt-5.6".into(),
17458 reason: CredentialFailure::EnvVarMissing {
17459 env_var: "OPENAI_API_KEY".into(),
17460 },
17461 detail: "set OPENAI_API_KEY".into(),
17462 },
17463 false,
17464 );
17465
17466 let terminal = InferenceError::InferenceFailed(
17467 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17468 .into(),
17469 );
17470 let error = apply_route_failure_context(terminal, credential.as_ref()).to_string();
17471 assert!(
17472 credential.is_none(),
17473 "an unconfigured fallback is ambient noise"
17474 );
17475 assert!(error.contains("9059 MB"), "{error}");
17476 assert!(!error.contains("OPENAI_API_KEY"), "{error}");
17477 assert!(!is_auth_failure_message(&error), "{error}");
17478 }
17479
17480 fn workspace_required() -> InferenceError {
17484 InferenceError::WorkspaceRequired {
17485 provider: "Parslee".into(),
17486 detail: "finish setting up at https://parslee.ai, then try again".into(),
17487 }
17488 }
17489
17490 #[test]
17499 fn workspace_required_reads_as_configuration_not_sign_in() {
17500 let rendered = workspace_required().to_string();
17501 for marker in AUTH_FAILURE_MESSAGE_MARKERS {
17502 assert!(
17503 !rendered
17504 .to_ascii_lowercase()
17505 .contains(&marker.to_ascii_lowercase()),
17506 "`{marker}` must not appear in the no-workspace text: {rendered}"
17507 );
17508 }
17509 assert!(!is_auth_failure_message(&rendered), "{rendered}");
17510 assert!(!is_auth_rejection_message(&rendered), "{rendered}");
17511 assert!(rendered.contains("https://parslee.ai"), "{rendered}");
17512 }
17513
17514 #[test]
17521 fn workspace_required_survives_a_mixed_chain() {
17522 let mut credential = None;
17523 record_route_credential_failure(
17524 &mut credential,
17525 "openai/gpt-5.6",
17526 &InferenceError::CredentialUnavailable {
17527 provider: "openai".into(),
17528 model: "openai/gpt-5.6".into(),
17529 reason: CredentialFailure::SignedOut,
17530 detail: "no account is signed in".into(),
17531 },
17532 true,
17533 );
17534 assert!(
17535 credential.is_some(),
17536 "the fixture must actually record a credential failure"
17537 );
17538
17539 let final_error = apply_route_failure_context(workspace_required(), credential.as_ref());
17540 assert!(
17541 matches!(final_error, InferenceError::WorkspaceRequired { .. }),
17542 "the type must survive the route context: {final_error}"
17543 );
17544 assert!(!is_auth_failure_message(&final_error.to_string()));
17545 assert!(
17546 error_ends_fallback_chain(&final_error),
17547 "no later candidate on that account can succeed"
17548 );
17549 }
17550
17551 #[test]
17558 fn repeated_workspace_required_leaves_model_health_untouched() {
17559 assert!(
17560 !error_counts_against_circuit_breaker(&workspace_required()),
17561 "the breaker must not count a workspace gap"
17562 );
17563
17564 let mut tracker = OutcomeTracker::new();
17565 for _ in 0..5 {
17566 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
17567 record_dispatch_failure(&mut tracker, &trace, &workspace_required());
17568 }
17569 let failed = tracker
17570 .profile("parslee/advisor")
17571 .map(|p| p.fail_count)
17572 .unwrap_or(0);
17573 assert_eq!(
17574 failed, 0,
17575 "five workspace refusals must not bench the model"
17576 );
17577
17578 let trace = tracker.record_start("parslee/advisor", InferenceTask::Generate, "test");
17579 record_dispatch_failure(
17580 &mut tracker,
17581 &trace,
17582 &InferenceError::InferenceFailed("decoder fell over".into()),
17583 );
17584 assert_eq!(
17585 tracker
17586 .profile("parslee/advisor")
17587 .map(|p| p.fail_count)
17588 .unwrap_or(0),
17589 1,
17590 "a genuine failure must still count, or the test above proves nothing"
17591 );
17592 }
17593
17594 #[test]
17595 fn explicitly_requested_missing_credential_uses_the_shared_auth_table() {
17596 let mut credential = None;
17597 record_route_credential_failure(
17598 &mut credential,
17599 "openai/gpt-5.6",
17600 &InferenceError::CredentialUnavailable {
17601 provider: "openai".into(),
17602 model: "openai/gpt-5.6".into(),
17603 reason: CredentialFailure::EnvVarMissing {
17604 env_var: "OPENAI_API_KEY".into(),
17605 },
17606 detail: "set OPENAI_API_KEY".into(),
17607 },
17608 true,
17609 );
17610 let summary = credential
17611 .expect("an explicitly requested provider must surface its missing key")
17612 .summary;
17613 assert!(summary.contains(AUTH_ENV_MISSING_MARKER), "{summary}");
17614 assert!(is_auth_failure_message(&summary), "{summary}");
17615 }
17616
17617 #[test]
17618 fn latest_actionable_credential_failure_wins() {
17619 let mut credential = None;
17620 record_route_credential_failure(
17621 &mut credential,
17622 "parslee/reasoning",
17623 &InferenceError::ProviderAccount {
17624 provider: "parslee".into(),
17625 status: 401,
17626 message: "Unauthorized".into(),
17627 },
17628 false,
17629 );
17630 record_route_credential_failure(
17631 &mut credential,
17632 "anthropic/claude",
17633 &InferenceError::CredentialUnavailable {
17634 provider: "anthropic".into(),
17635 model: "anthropic/claude".into(),
17636 reason: CredentialFailure::Expired { expires_at: 43 },
17637 detail: "configured token expired".into(),
17638 },
17639 false,
17640 );
17641
17642 assert_eq!(
17643 credential
17644 .expect("latest actionable credential cause must be retained")
17645 .summary,
17646 "anthropic login expired for `anthropic/claude` — run `car auth login`"
17647 );
17648 }
17649
17650 #[test]
17651 fn store_unreadable_summary_uses_the_shared_auth_table() {
17652 let summary = route_credential_failure(
17653 "parslee/reasoning",
17654 &InferenceError::CredentialUnavailable {
17655 provider: "parslee".into(),
17656 model: "parslee/reasoning".into(),
17657 reason: CredentialFailure::StoreUnreadable,
17658 detail: "keychain helper timed out".into(),
17659 },
17660 false,
17661 )
17662 .expect("an unreadable configured credential store is actionable");
17663 assert!(summary.contains(AUTH_STORE_UNREADABLE_MARKER), "{summary}");
17664 assert!(is_auth_failure_message(&summary), "{summary}");
17665 }
17666
17667 fn chain_gate_fixture_schema(id: &str, provider: &str, source: ModelSource) -> ModelSchema {
17668 ModelSchema {
17669 id: id.into(),
17670 name: id.into(),
17671 provider: provider.into(),
17672 family: "test".into(),
17673 version: "1".into(),
17674 capabilities: vec![ModelCapability::Generate],
17675 context_length: 32_768,
17676 max_output_tokens: Some(4_096),
17677 param_count: String::new(),
17678 quantization: None,
17679 performance: Default::default(),
17680 cost: Default::default(),
17681 source,
17682 tags: vec![],
17683 supported_params: vec![],
17684 public_benchmarks: vec![],
17685 trust_tier: TrustTier::Community,
17686 deprecated: false,
17687 available: true,
17688 weights_ready: true,
17689 }
17690 }
17691
17692 #[test]
17693 fn signed_out_pre_seed_is_gated_on_a_parslee_route_in_the_chain() {
17694 let local = chain_gate_fixture_schema(
17695 "qwen/qwen3-4b:q4_k_m",
17696 "qwen",
17697 ModelSource::Mlx {
17698 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
17699 hf_weight_file: None,
17700 },
17701 );
17702 let ollama = chain_gate_fixture_schema(
17703 "ollama/llama3",
17704 "ollama",
17705 ModelSource::Ollama {
17706 model_tag: "llama3".into(),
17707 host: "http://localhost:11434".into(),
17708 },
17709 );
17710 let parslee = chain_gate_fixture_schema(
17711 "parslee/reasoning",
17712 "parslee",
17713 ModelSource::Proprietary {
17714 provider: "parslee".into(),
17715 endpoint: "https://api.parslee.ai".into(),
17716 auth: ProprietaryAuth::OAuth2Pkce {
17717 authority: "https://login.example".into(),
17718 client_id: "client".into(),
17719 scopes: vec![],
17720 },
17721 protocol: Default::default(),
17722 },
17723 );
17724 let cloud = chain_gate_fixture_schema(
17725 "openai/gpt-5.6",
17726 "openai",
17727 ModelSource::RemoteApi {
17728 endpoint: "https://api.openai.com".into(),
17729 api_key_env: "OPENAI_API_KEY".into(),
17730 api_key_envs: vec![],
17731 api_version: None,
17732 protocol: schema::ApiProtocol::OpenAiCompat,
17733 },
17734 );
17735 let schemas: std::collections::HashMap<&str, &ModelSchema> = [
17736 ("qwen/qwen3-4b:q4_k_m", &local),
17737 ("ollama/llama3", &ollama),
17738 ("parslee/reasoning", &parslee),
17739 ("openai/gpt-5.6", &cloud),
17740 ]
17741 .into_iter()
17742 .collect();
17743 let resolve = |m: &str| schemas.get(m).copied();
17744
17745 let local_only = vec![
17747 "qwen/qwen3-4b:q4_k_m".to_string(),
17748 "ollama/llama3".to_string(),
17749 ];
17750 assert!(!chain_includes_parslee_route(resolve, &local_only));
17751
17752 let with_parslee = vec![
17754 "parslee/reasoning".to_string(),
17755 "qwen/qwen3-4b:q4_k_m".to_string(),
17756 ];
17757 assert!(chain_includes_parslee_route(resolve, &with_parslee));
17758 let with_cloud = vec![
17759 "openai/gpt-5.6".to_string(),
17760 "qwen/qwen3-4b:q4_k_m".to_string(),
17761 ];
17762 assert!(!chain_includes_parslee_route(resolve, &with_cloud));
17766 let mut credential = chain_includes_parslee_route(resolve, &with_cloud)
17767 .then(parslee_signed_out_route_failure);
17768 let outage = InferenceError::Transient {
17769 status: Some(500),
17770 message: "OpenAI HTTP 500 Internal Server Error".into(),
17771 };
17772 record_route_credential_failure(&mut credential, &with_cloud[0], &outage, false);
17773 let terminal = apply_route_failure_context(outage, credential.as_ref());
17774 assert!(matches!(terminal, InferenceError::Transient { .. }));
17775 assert!(!is_auth_failure_message(&terminal.to_string()));
17776 let oom = InferenceError::InferenceFailed("local model out of memory".into());
17777 record_route_credential_failure(&mut credential, &with_cloud[1], &oom, false);
17778 let terminal = apply_route_failure_context(oom, credential.as_ref()).to_string();
17779 assert!(!terminal.contains("Parslee"), "{terminal}");
17780 assert!(!is_auth_failure_message(&terminal), "{terminal}");
17781
17782 record_route_credential_failure(
17784 &mut credential,
17785 &with_cloud[0],
17786 &InferenceError::ProviderAccount {
17787 provider: "openai".into(),
17788 status: 401,
17789 message: "invalid API key".into(),
17790 },
17791 false,
17792 );
17793 let terminal = apply_route_failure_context(
17794 InferenceError::InferenceFailed("local model out of memory".into()),
17795 credential.as_ref(),
17796 )
17797 .to_string();
17798 assert!(terminal.contains("openai"), "{terminal}");
17799 assert!(!terminal.contains("Parslee"), "{terminal}");
17800 assert!(is_auth_failure_message(&terminal), "{terminal}");
17801
17802 let unknown = vec!["missing/model".to_string()];
17804 assert!(!chain_includes_parslee_route(resolve, &unknown));
17805 }
17806
17807 #[test]
17811 fn local_only_terminal_failures_are_not_relabeled_as_auth() {
17812 let cases = [
17813 InferenceError::InferenceFailed(
17815 "This model needs about 9059 MB, beyond the configured 6553 MB local-model allocation"
17816 .into(),
17817 ),
17818 InferenceError::Transient {
17820 status: Some(500),
17821 message: "HTTP 500 Internal Server Error".into(),
17822 },
17823 InferenceError::InferenceFailed("llama runner process has terminated".into()),
17825 ];
17826 for case in cases {
17827 let error = apply_route_failure_context(case, None).to_string();
17828 assert!(!error.contains("Parslee login is absent"), "{error}");
17829 assert!(!is_auth_failure_message(&error), "{error}");
17830 }
17831
17832 let error = apply_route_failure_context(
17836 InferenceError::ModelNotFound("qwen/qwen3-4b:q4_k_m".into()),
17837 None,
17838 )
17839 .to_string();
17840 assert!(!error.contains("Parslee login is absent"), "{error}");
17841 assert!(error.contains("car models pull"), "{error}");
17842 }
17843
17844 #[test]
17847 fn credential_context_preserves_the_typed_terminal_variant() {
17848 let credential = parslee_signed_out_route_failure();
17849
17850 let transient = apply_route_failure_context(
17851 InferenceError::Transient {
17852 status: Some(500),
17853 message: "HTTP 500 Internal Server Error".into(),
17854 },
17855 Some(&credential),
17856 );
17857 match &transient {
17858 InferenceError::Transient { status, message } => {
17859 assert_eq!(*status, Some(500));
17860 assert!(message.starts_with("Parslee login is absent"), "{message}");
17861 assert!(message.contains("HTTP 500"), "{message}");
17862 }
17863 other => panic!("Transient must stay Transient, got {other:?}"),
17864 }
17865
17866 let account = apply_route_failure_context(
17867 InferenceError::ProviderAccount {
17868 provider: "openai".into(),
17869 status: 402,
17870 message: "insufficient credits".into(),
17871 },
17872 Some(&credential),
17873 );
17874 assert!(
17875 matches!(
17876 &account,
17877 InferenceError::ProviderAccount {
17878 provider,
17879 status: 402,
17880 ..
17881 } if provider == "openai"
17882 ),
17883 "ProviderAccount must stay ProviderAccount, got {account:?}"
17884 );
17885
17886 let unavailable = apply_route_failure_context(
17887 InferenceError::CredentialUnavailable {
17888 provider: "parslee".into(),
17889 model: "parslee/reasoning".into(),
17890 reason: CredentialFailure::Expired { expires_at: 42 },
17891 detail: "access token expired".into(),
17892 },
17893 Some(&credential),
17894 );
17895 match &unavailable {
17896 InferenceError::CredentialUnavailable { reason, detail, .. } => {
17897 assert_eq!(*reason, CredentialFailure::Expired { expires_at: 42 });
17898 assert!(detail.contains("Parslee login is absent"), "{detail}");
17899 }
17900 other => panic!("CredentialUnavailable must keep its reason data, got {other:?}"),
17901 }
17902 assert!(is_auth_failure_message(&unavailable.to_string()));
17903 }
17904
17905 #[test]
17908 fn non_parslee_rejection_summaries_match_the_shared_classifier() {
17909 let rejected = route_credential_failure(
17910 "openai/gpt-5.6",
17911 &InferenceError::ProviderAccount {
17912 provider: "openai".into(),
17913 status: 403,
17914 message: "key revoked".into(),
17915 },
17916 false,
17917 )
17918 .expect("a 403 from a configured provider is actionable");
17919 assert!(rejected.contains("credential was rejected"), "{rejected}");
17920 assert!(is_auth_failure_message(&rejected), "{rejected}");
17921
17922 let generic = route_credential_failure(
17923 "openai/gpt-5.6",
17924 &InferenceError::InferenceFailed("upstream said: token expired".into()),
17925 false,
17926 )
17927 .expect("an auth-rejection message from a non-Parslee route is actionable");
17928 assert!(generic.contains("repair its provider login"), "{generic}");
17929 assert!(is_auth_failure_message(&generic), "{generic}");
17930 }
17931
17932 #[test]
17937 fn the_recovery_hints_lead_with_parslee_sign_in() {
17938 for hint in [
17939 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
17940 .expect("a missing credential is a no-backend case"),
17941 auth_expired_recovery_hint(
17942 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required",
17943 )
17944 .expect("a rejected session is an auth case"),
17945 ] {
17946 let login = hint
17947 .find("car auth login")
17948 .expect("sign-in must be offered");
17949 let pull = hint
17950 .find("car models pull")
17951 .expect("the local path must still be offered");
17952 assert!(login < pull, "sign-in must be read first:\n{hint}");
17953 }
17954 assert!(
17957 no_backend_recovery_hint("no credential for proprietary provider 'parslee'")
17958 .expect("hint")
17959 .starts_with(NO_BACKEND_RECOVERY_MARKER),
17960 "the marker opening is load-bearing"
17961 );
17962 }
17963
17964 #[test]
17967 fn fresh_install_exhaustion_keeps_the_models_pull_guidance() {
17968 let credential = parslee_signed_out_route_failure();
17969 let error = apply_route_failure_context(
17970 InferenceError::InferenceFailed("no models available for generate".into()),
17971 Some(&credential),
17972 )
17973 .to_string();
17974 assert!(error.contains("Parslee login is absent"), "{error}");
17975 assert!(
17976 error.contains(&format!("car models pull {LOCAL_TOOL_MODEL_PULL_ID}")),
17977 "{error}"
17978 );
17979 assert!(error.contains("car auth login"), "{error}");
17980 }
17981
17982 #[test]
17985 fn auth_fallback_from_round_trips_and_defaults_to_none() {
17986 let mut result: InferenceResult = serde_json::from_value(serde_json::json!({
17987 "text": "hi",
17988 "tool_calls": [],
17989 "trace_id": "t",
17990 "model_used": "openai/gpt-5.6",
17991 "latency_ms": 1,
17992 }))
17993 .expect("a payload without the field still deserializes");
17994 assert_eq!(result.auth_fallback_from, None);
17995 let json = serde_json::to_value(&result).unwrap();
17996 assert!(json.get("auth_fallback_from").is_none());
17997
17998 result.auth_fallback_from = Some("parslee/reasoning".to_string());
17999 let json = serde_json::to_value(&result).unwrap();
18000 assert_eq!(json["auth_fallback_from"], "parslee/reasoning");
18001 let back: InferenceResult = serde_json::from_value(json).unwrap();
18002 assert_eq!(
18003 back.auth_fallback_from.as_deref(),
18004 Some("parslee/reasoning")
18005 );
18006 }
18007
18008 #[test]
18013 fn local_last_resort_turn_carries_model_id_and_flag() {
18014 let ordinary: InferenceResult = serde_json::from_value(serde_json::json!({
18015 "text": "hi",
18016 "tool_calls": [],
18017 "trace_id": "ordinary",
18018 "model_used": "anthropic/claude-haiku-4-5:latest",
18019 "latency_ms": 1,
18020 }))
18021 .expect("older payloads default the marker");
18022 assert!(!ordinary.local_last_resort);
18023
18024 let fallback: InferenceResult = serde_json::from_value(serde_json::json!({
18025 "text": "offline answer",
18026 "tool_calls": [],
18027 "trace_id": "fallback",
18028 "model_used": "Qwen3 4B MLX",
18029 "resolved_model_id": "mlx/qwen3-4b:4bit",
18030 "latency_ms": 1,
18031 "local_last_resort": true,
18032 }))
18033 .expect("fallback attribution payload");
18034 assert_eq!(fallback.served_model_id(), "mlx/qwen3-4b:4bit");
18035 assert!(fallback.local_last_resort);
18036
18037 assert!(!is_local_last_resort(None, "mlx/qwen3-4b:4bit"));
18038 assert!(report_local_last_resort_served(
18039 Some("mlx/qwen3-4b:4bit"),
18040 "mlx/qwen3-4b:4bit",
18041 "mlx/qwen3-4b:4bit"
18042 ));
18043 assert!(!is_local_last_resort(
18044 Some("mlx/qwen3-4b:4bit"),
18045 "anthropic/claude-haiku-4-5:latest"
18046 ));
18047 }
18048
18049 #[test]
18050 fn no_backend_hint_passes_through_transient_errors() {
18051 for phrase in [
18054 "API returned 401 Unauthorized",
18055 "API returned 429 Too Many Requests",
18056 "API returned 500 Internal Server Error",
18057 "connection refused",
18058 "request timed out",
18059 "parse response: unexpected end of input",
18060 ] {
18061 assert!(
18062 no_backend_recovery_hint(phrase).is_none(),
18063 "transient error wrongly classified as no-backend: {phrase:?}"
18064 );
18065 }
18066 }
18067
18068 static ENV_MUTEX: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
18071
18072 #[cfg(unix)]
18073 #[tokio::test]
18074 async fn exact_codex_subscription_row_dispatches_without_an_openai_key() {
18075 use std::os::unix::fs::PermissionsExt;
18076
18077 let _environment = ENV_MUTEX.lock().await;
18078 let _restore = RestoredEnvironment::capture(&["CAR_CODEX_BIN", "OPENAI_API_KEY"]);
18079 let tmp = TempDir::new().unwrap();
18080 let fixture = tmp.path().join("codex-fixture.sh");
18081 std::fs::write(
18082 &fixture,
18083 r#"#!/bin/sh
18084if [ -n "${OPENAI_API_KEY-}" ]; then
18085 echo 'OPENAI_API_KEY leaked' >&2
18086 exit 91
18087fi
18088cat >/dev/null
18089printf '%s\n' '{"type":"turn.started"}'
18090printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"fixture newsroom answer"}}'
18091printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":17,"output_tokens":5}}'
18092"#,
18093 )
18094 .unwrap();
18095 std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o700)).unwrap();
18096 unsafe {
18097 std::env::set_var("CAR_CODEX_BIN", &fixture);
18098 std::env::set_var("OPENAI_API_KEY", "must-not-reach-codex");
18099 }
18100
18101 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
18102 let mut request = GenerateRequest {
18103 prompt: "write a brief".into(),
18104 params: GenerateParams {
18105 max_tokens: 1056,
18106 strict_model: true,
18107 ..Default::default()
18108 },
18109 ..Default::default()
18110 };
18111 pin_exact_model_id(&mut request, "openai/gpt-5.6-sol:high".into()).unwrap();
18112 let result = engine.generate_tracked(request).await.unwrap();
18113
18114 assert_eq!(result.model_used, "openai/gpt-5.6-sol:high");
18115 assert_eq!(result.text, "fixture newsroom answer");
18116 assert_eq!(result.usage.as_ref().unwrap().total_tokens, 22);
18117 assert_eq!(result.stop_reason, None, "Codex reports no finish reason");
18118 assert!(result.tool_calls.is_empty());
18119 }
18120
18121 #[tokio::test]
18122 async fn isolated_test_child_removes_parent_auth_lock_override() {
18123 const CHILD_SENTINEL: &str = "CAR_AUTH_LOCK_SANITIZER_CHILD";
18124 if std::env::var_os(CHILD_SENTINEL).is_some() {
18125 assert!(
18126 std::env::var_os("CAR_AUTH_LOCK_PATH").is_none(),
18127 "an isolated credential test must not inherit the invoking process's auth lock"
18128 );
18129 return;
18130 }
18131
18132 let _environment = ENV_MUTEX.lock().await;
18133 let _restore = RestoredEnvironment::capture(&["CAR_AUTH_LOCK_PATH"]);
18134 unsafe {
18135 std::env::set_var(
18136 "CAR_AUTH_LOCK_PATH",
18137 "/sentinel/must-not-reach-isolated-child.lock",
18138 );
18139 }
18140 assert!(!crate::run_in_isolated_test_process(
18141 "tests::isolated_test_child_removes_parent_auth_lock_override",
18142 CHILD_SENTINEL,
18143 ));
18144 }
18145
18146 fn test_config(models_dir: PathBuf) -> InferenceConfig {
18147 let state_root = models_dir
18151 .parent()
18152 .map(Path::to_path_buf)
18153 .unwrap_or_else(|| models_dir.clone());
18154 InferenceConfig {
18155 models_dir,
18156 state_root,
18157 device: None,
18158 generation_model: "Qwen3-0.6B".into(),
18159 preferred_generation_model: None,
18160 embedding_model: "Qwen3-Embedding-0.6B".into(),
18161 preferred_embedding_model: None,
18162 classification_model: "Qwen3-0.6B".into(),
18163 preferred_classification_model: None,
18164 }
18165 }
18166
18167 fn metal_mac_for_fit(ram_gb: u64) -> HardwareInfo {
18168 HardwareInfo {
18169 os: "macos".into(),
18170 arch: "aarch64".into(),
18171 cpu_cores: 8,
18172 total_ram_mb: ram_gb * 1024,
18173 gpu_backend: crate::hardware::GpuBackend::Metal,
18174 gpu_memory_mb: None,
18175 gpu_devices: vec![],
18176 recommended_model: String::new(),
18177 recommended_context: 8_192,
18178 max_model_mb: 0,
18179 }
18180 }
18181
18182 #[test]
18189 fn unified_rows_carry_fit_per_machine_and_keep_deprecated_rows() {
18190 if !crate::run_in_isolated_test_process(
18191 "tests::unified_rows_carry_fit_per_machine_and_keep_deprecated_rows",
18192 "CAR_UNIFIED_ROWS_FIT_TEST_CHILD",
18193 ) {
18194 return;
18195 }
18196
18197 let _credential_scope = crate::openrouter::test_credential_scope();
18204 crate::openrouter::set_test_credential(Some("unified-rows-fit-test-key"));
18205 let secret_activity_before = car_secrets::secret_store_activity();
18206 let root = tempfile::tempdir().unwrap();
18207 let config = test_config(root.path().join("weights"));
18208 let mut engine = InferenceEngine::new(config);
18209 let mut retired = engine
18210 .list_schemas()
18211 .into_iter()
18212 .find(|schema| schema.id == "mlx/qwen3-4b:4bit")
18213 .expect("builtin 4B MLX row");
18214 retired.id = "test/retired-4b:4bit".into();
18215 retired.name = "Retired 4B".into();
18216 retired.deprecated = true;
18217 engine.register_model(retired);
18218 let policy = resource_policy::ResourcePolicy::everyday();
18219
18220 let at_8 = engine.list_models_unified_for(&metal_mac_for_fit(8), &policy);
18221 let at_32 = engine.list_models_unified_for(&metal_mac_for_fit(32), &policy);
18222 let ids = |rows: &[ModelInfo]| rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
18223 assert_eq!(
18224 ids(&at_8),
18225 ids(&at_32),
18226 "the machine never removes or reorders a row"
18227 );
18228 let row = |rows: &[ModelInfo], id: &str| {
18229 rows.iter()
18230 .find(|row| row.id == id)
18231 .unwrap_or_else(|| panic!("{id} missing"))
18232 .clone()
18233 };
18234
18235 let eight_b_small = row(&at_8, "mlx/qwen3-8b:4bit");
18236 let eight_b_large = row(&at_32, "mlx/qwen3-8b:4bit");
18237 assert_eq!(eight_b_small.fit, ModelFitStatus::TooBig);
18238 assert_eq!(eight_b_large.fit, ModelFitStatus::Fits);
18239 assert!(eight_b_small.platform_compatible && eight_b_large.platform_compatible);
18240 assert_eq!(
18241 eight_b_small.estimated_peak_mb, eight_b_large.estimated_peak_mb,
18242 "the estimate is the model's; only the budget differs"
18243 );
18244 assert!(eight_b_small.estimated_peak_mb.is_some_and(|mb| mb > 4_800));
18245 assert_eq!(eight_b_small.family.as_deref(), Some("qwen3"));
18246 assert!(eight_b_small.version.is_some());
18247
18248 let retired = row(&at_8, "test/retired-4b:4bit");
18249 assert!(retired.deprecated, "deprecated rows stay listed, flagged");
18250 assert!(!eight_b_small.deprecated);
18251
18252 let mut saw_remote = false;
18253 for remote in at_8.iter().filter(|row| !row.is_local) {
18254 saw_remote = true;
18255 assert_eq!(remote.fit, ModelFitStatus::Fits, "{}", remote.id);
18256 assert!(remote.platform_compatible, "{}", remote.id);
18257 assert_eq!(remote.estimated_peak_mb, None, "{}", remote.id);
18258 assert_eq!(
18259 remote.family, None,
18260 "{}: no upstream identifier here",
18261 remote.id
18262 );
18263 assert_eq!(remote.version, None, "{}", remote.id);
18264 }
18265 assert!(saw_remote);
18266 assert_eq!(
18267 car_secrets::secret_store_activity(),
18268 secret_activity_before,
18269 "catalog snapshots must use the injected credential without consulting host secrets"
18270 );
18271
18272 for (small, large) in at_8.iter().zip(&at_32) {
18275 let strip = |row: &ModelInfo| {
18276 let mut value = serde_json::to_value(row).unwrap();
18277 let object = value.as_object_mut().unwrap();
18278 for key in ["fit", "estimated_peak_mb", "platform_compatible"] {
18279 object.remove(key);
18280 }
18281 value
18282 };
18283 assert_eq!(strip(small), strip(large), "{}", small.id);
18284 }
18285 }
18286
18287 #[tokio::test]
18288 async fn explicit_model_blocked_by_zero_budget_does_not_substitute_or_download() {
18289 let root = tempfile::tempdir().unwrap();
18290 let models_dir = root.path().join("weights");
18291 let config = test_config(models_dir.clone());
18292 let repository =
18293 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
18294 crate::resource_policy::ResourcePolicyRepository::save(
18295 &repository,
18296 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
18297 )
18298 .unwrap();
18299 let engine = InferenceEngine::new(config);
18300
18301 let error = engine
18302 .generate_tracked(GenerateRequest {
18303 prompt: "hello".into(),
18304 model: Some("mlx/qwen3-4b:4bit".into()),
18305 ..Default::default()
18306 })
18307 .await
18308 .unwrap_err();
18309
18310 assert!(matches!(
18311 error,
18312 InferenceError::LocalResourceBlocked {
18313 preflight: crate::resource_policy::LocalLoadPreflight {
18314 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
18315 ..
18316 },
18317 ..
18318 }
18319 ));
18320 assert!(
18321 !models_dir.exists() || std::fs::read_dir(models_dir).unwrap().next().is_none(),
18322 "admission must happen before download/load"
18323 );
18324 }
18325
18326 #[tokio::test]
18327 async fn external_vllm_mlx_bypasses_local_admission_while_managed_artifact_does_not() {
18328 let schema = crate::vllm_mlx::to_model_schema(
18329 &crate::vllm_mlx::DiscoveredModel {
18330 id: "mlx-community/Qwen3-4B-4bit".into(),
18331 owned_by: None,
18332 },
18333 "http://localhost:8000",
18334 );
18335 assert!(!InferenceEngine::requires_local_admission(&schema));
18336
18337 let root = tempfile::tempdir().unwrap();
18338 let config = test_config(root.path().join("weights"));
18339 let repository =
18340 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
18341 crate::resource_policy::ResourcePolicyRepository::save(
18342 &repository,
18343 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
18344 )
18345 .unwrap();
18346 let engine = InferenceEngine::new(config);
18347 let (external, reservation) = engine
18348 .vllm_live_schema(schema.clone(), None, 0)
18349 .await
18350 .unwrap();
18351 assert!(matches!(
18352 external.source,
18353 ModelSource::VllmMlx { ref endpoint, .. } if endpoint == "http://localhost:8000"
18354 ));
18355 assert!(reservation.is_none());
18356
18357 let mut managed = schema;
18358 managed.source = ModelSource::ManagedVllmMlx {
18359 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
18360 hf_weight_file: None,
18361 };
18362 assert!(InferenceEngine::requires_local_admission(&managed));
18363 let error = engine
18364 .reserve_local_request(&managed, 0)
18365 .expect_err("zero-GB policy must reject before vllm runtime/download/spawn");
18366 assert!(matches!(
18367 error,
18368 InferenceError::LocalResourceBlocked {
18369 preflight: crate::resource_policy::LocalLoadPreflight {
18370 verdict: crate::resource_policy::LocalLoadVerdict::DisabledByPolicy,
18371 ..
18372 },
18373 ..
18374 }
18375 ));
18376 }
18377
18378 #[cfg(unix)]
18379 async fn assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(
18380 streaming: bool,
18381 cancel_first: bool,
18382 ) {
18383 struct FixedProbe;
18384 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
18385 fn available_memory_mb(
18386 &self,
18387 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
18388 Ok(Some(24_000))
18389 }
18390 }
18391
18392 let Some(python) = crate::vllm_runtime::test_python_interpreter() else {
18393 panic!("a real Python interpreter is required for the managed-vllm dispatch fixture");
18394 };
18395 let root = tempfile::tempdir().unwrap();
18396 let script = root.path().join("fake-vllm-mlx");
18397 let spawned = root.path().join("spawned");
18398 let release = root.path().join("release");
18399 std::fs::write(
18400 &script,
18401 format!(
18402 "#!{}\n\
18403 import http.server, os, sys, time\n\
18404 import socket as _s; _s.getfqdn = lambda n='': n\n\
18405 marker = {:?}\n\
18406 release = {:?}\n\
18407 open(marker, 'w').close()\n\
18408 while not os.path.exists(release): time.sleep(0.01)\n\
18409 port = int(sys.argv[sys.argv.index('--port') + 1])\n\
18410 class H(http.server.BaseHTTPRequestHandler):\n\
18411 \x20 def do_GET(self):\n\
18412 \x20 self.send_response(200); self.end_headers(); self.wfile.write(b'ok')\n\
18413 \x20 def do_POST(self):\n\
18414 \x20 length = int(self.headers.get('content-length', '0'))\n\
18415 \x20 request = self.rfile.read(length).replace(b' ', b'')\n\
18416 \x20 if b'\"stream\":true' in request:\n\
18417 \x20 body = b'data: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"round13-stream\"}},\"finish_reason\":null}}]}}\\n\\ndata: {{\"id\":\"fixture\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"stop\"}}]}}\\n\\ndata: [DONE]\\n\\n'\n\
18418 \x20 content_type = 'text/event-stream'\n\
18419 \x20 else:\n\
18420 \x20 body = b'{{\"id\":\"fixture\",\"object\":\"chat.completion\",\"created\":0,\"model\":\"fixture\",\"choices\":[{{\"index\":0,\"message\":{{\"role\":\"assistant\",\"content\":\"round13-ok\"}},\"finish_reason\":\"stop\"}}],\"usage\":{{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}}}'\n\
18421 \x20 content_type = 'application/json'\n\
18422 \x20 self.send_response(200); self.send_header('content-type', content_type); self.send_header('content-length', str(len(body))); self.end_headers(); self.wfile.write(body)\n\
18423 \x20 def log_message(self, *args): pass\n\
18424 http.server.HTTPServer(('127.0.0.1', port), H).serve_forever()\n",
18425 python.display(),
18426 spawned.display().to_string(),
18427 release.display().to_string(),
18428 ),
18429 )
18430 .unwrap();
18431 use std::os::unix::fs::PermissionsExt;
18432 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
18433
18434 let models_dir = root.path().join("models");
18435 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
18436 let mut schema = crate::registry::builtin_catalog()
18437 .into_iter()
18438 .find(ModelSchema::is_car_managed_vllm_mlx)
18439 .expect("managed vllm fixture schema");
18440 schema.id = "vllm-mlx/round12-singleflight".into();
18441 schema.name = "round12-singleflight".into();
18442 schema.cost.size_mb = Some(1);
18443 schema.cost.ram_mb = Some(1);
18444 schema.source = ModelSource::ManagedVllmMlx {
18445 hf_repo: "fixture/round12-singleflight".into(),
18446 hf_weight_file: None,
18447 };
18448 let model_dir = models_dir.join(&schema.name);
18449 std::fs::create_dir_all(&model_dir).unwrap();
18450 std::fs::write(model_dir.join("config.json"), b"{}").unwrap();
18451 std::fs::write(model_dir.join("model.safetensors"), b"fixture").unwrap();
18452 engine
18453 .unified_registry_mut()
18454 .register_project_model(schema.clone());
18455
18456 let coordinator = Arc::new(
18457 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
18458 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
18463 crate::hardware::HardwareInfo::detect(),
18464 Arc::new(FixedProbe),
18465 ),
18466 );
18467 engine.local_admission = coordinator.clone();
18468 engine.vllm_pool = Arc::new(crate::vllm_pool::VllmServerPool::with_test_runtime(
18469 std::time::Duration::from_secs(300),
18470 coordinator,
18471 script,
18472 ));
18473 let engine = Arc::new(engine);
18474 let model_id = schema.id.clone();
18475 let mut request = GenerateRequest {
18476 prompt: "hello from round 13".into(),
18477 model: Some(model_id.clone()),
18478 ..Default::default()
18479 };
18480 request.params.strict_model = true;
18485 let (estimated_input, _, _) = engine.estimated_tokens(&request, Some(&model_id));
18486 let expected_footprint = estimated_input.saturating_add(request.params.max_tokens);
18487 let expected_estimate = engine
18488 .local_model_preflight(&model_id, expected_footprint)
18489 .unwrap()
18490 .estimate;
18491 let expected_request_overhead_mb = expected_estimate
18492 .runtime_overhead_mb
18493 .saturating_add(expected_estimate.context_overhead_mb)
18494 .saturating_add(expected_estimate.transient_margin_mb);
18495
18496 async fn execute(
18497 engine: Arc<InferenceEngine>,
18498 request: GenerateRequest,
18499 streaming: bool,
18500 direct_vllm: bool,
18501 ) -> Result<String, InferenceError> {
18502 if direct_vllm {
18503 let model_id = request.model.as_deref().expect("explicit fixture model");
18504 let schema = engine
18505 .unified_registry()
18506 .get(model_id)
18507 .cloned()
18508 .expect("fixture schema");
18509 let (schema, _) = engine.vllm_live_schema(schema, None, 0).await?;
18510 return match schema.source {
18511 ModelSource::VllmMlx { endpoint, .. } => Ok(endpoint),
18512 source => Err(InferenceError::InferenceFailed(format!(
18513 "fixture did not resolve to a vllm endpoint: {source:?}"
18514 ))),
18515 };
18516 }
18517 if !streaming {
18518 return engine
18519 .generate_tracked(request)
18520 .await
18521 .map(|result| result.text);
18522 }
18523 let mut tracked = engine.generate_tracked_stream(request).await?;
18524 let mut accumulator = crate::stream::StreamAccumulator::default();
18525 while let Some(event) = tracked.events.recv().await {
18526 let done = matches!(event, crate::stream::StreamEvent::Done { .. });
18527 accumulator.push(&event);
18528 if done {
18529 break;
18530 }
18531 }
18532 Ok(accumulator.finish().0)
18533 }
18534
18535 let first_engine = engine.clone();
18536 let first_request = request.clone();
18537 let mut first = tokio::spawn(async move {
18538 execute(first_engine, first_request, streaming, cancel_first).await
18539 });
18540 tokio::time::timeout(std::time::Duration::from_secs(5), async {
18541 while !spawned.exists() {
18542 if first.is_finished() {
18543 let result = (&mut first).await;
18544 panic!("first dispatch ended before spawn: {result:?}");
18545 }
18546 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
18547 }
18548 })
18549 .await
18550 .expect("first request must spawn before publication");
18551 if !cancel_first {
18552 let during_startup = engine.local_model_preflight(&model_id, 0).unwrap();
18553 assert_eq!(
18554 during_startup.active_reservations_mb, expected_request_overhead_mb,
18555 "the cold-to-pending transfer must retain all request context/KV overhead"
18556 );
18557 assert_eq!(
18558 during_startup.resident_model_mb, expected_estimate.weights_mb,
18559 "the pending allocation must replace the exact cold-weight charge"
18560 );
18561 assert_eq!(
18562 during_startup
18563 .active_reservations_mb
18564 .saturating_add(during_startup.resident_model_mb),
18565 expected_estimate.estimated_peak_mb,
18566 "active request overhead plus pending weights must preserve the full admitted footprint"
18567 );
18568 }
18569
18570 let second_engine = engine.clone();
18571 let mut second =
18572 tokio::spawn(
18573 async move { execute(second_engine, request, streaming, cancel_first).await },
18574 );
18575 assert!(
18576 tokio::time::timeout(std::time::Duration::from_millis(100), &mut second)
18577 .await
18578 .is_err(),
18579 "request 2 must wait for request 1 to publish instead of failing on startup state"
18580 );
18581
18582 let first_result = if cancel_first {
18583 std::fs::remove_file(&spawned).unwrap();
18584 first.abort();
18585 let _ = first.await;
18586 tokio::time::timeout(std::time::Duration::from_secs(5), async {
18587 while !spawned.exists() {
18588 if second.is_finished() {
18589 let result = (&mut second).await;
18590 panic!("waiter ended before replacement spawn: {result:?}");
18591 }
18592 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
18593 }
18594 })
18595 .await
18596 .expect("waiter must continue after cancelled owner teardown");
18597 None
18598 } else {
18599 std::fs::write(&release, b"release").unwrap();
18600 Some(
18601 tokio::time::timeout(std::time::Duration::from_secs(5), first)
18602 .await
18603 .expect("first dispatch completes")
18604 .unwrap()
18605 .unwrap(),
18606 )
18607 };
18608 std::fs::write(&release, b"release").unwrap();
18609 let second_result = tokio::time::timeout(std::time::Duration::from_secs(5), second)
18610 .await
18611 .expect("second dispatch completes")
18612 .unwrap()
18613 .unwrap();
18614 let expected = if cancel_first {
18615 None
18616 } else if streaming {
18617 Some("round13-stream")
18618 } else {
18619 Some("round13-ok")
18620 };
18621 if let (Some(first_result), Some(expected)) = (first_result, expected) {
18622 assert_eq!(first_result, expected);
18623 }
18624 if let Some(expected) = expected {
18625 assert_eq!(second_result, expected);
18626 } else {
18627 assert!(second_result.starts_with("http://127.0.0.1:"));
18628 }
18629 assert_eq!(engine.vllm_pool.len().await, 1);
18630 assert!(engine
18631 .vllm_pool
18632 .release_model_if_present(&model_id)
18633 .await
18634 .unwrap());
18635 }
18636
18637 #[cfg(unix)]
18638 #[tokio::test]
18639 async fn concurrent_managed_vllm_generate_waits_before_outer_reservation() {
18640 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, false).await;
18641 }
18642
18643 #[cfg(unix)]
18644 #[tokio::test]
18645 async fn concurrent_managed_vllm_stream_waits_before_outer_reservation() {
18646 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(true, false).await;
18647 }
18648
18649 #[cfg(unix)]
18650 #[tokio::test]
18651 async fn managed_vllm_waiter_continues_after_startup_owner_cancellation() {
18652 assert_concurrent_managed_vllm_dispatch_waits_before_outer_reservation(false, true).await;
18653 }
18654
18655 #[tokio::test]
18656 async fn same_state_root_engines_share_runtime_components() {
18657 const CHILD_ENV: &str = "CAR_TWO_ENGINE_RUNTIME_TEST_CHILD";
18658 if std::env::var_os(CHILD_ENV).is_some() {
18659 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
18660 return;
18661 }
18662 let root = tempfile::tempdir().unwrap();
18663 let first = InferenceEngine::new(test_config(root.path().join("weights")));
18664 let second = InferenceEngine::new(test_config(root.path().join("weights")));
18665
18666 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
18667 assert!(Arc::ptr_eq(&first.vllm_pool, &second.vllm_pool));
18668 assert!(Arc::ptr_eq(
18669 &first.resource_policy_generation,
18670 &second.resource_policy_generation
18671 ));
18672 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
18673 {
18674 assert!(Arc::ptr_eq(&first.text_backends, &second.text_backends));
18675 assert!(Arc::ptr_eq(&first.local_backends, &second.local_backends));
18676 }
18677 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18678 {
18679 assert!(Arc::ptr_eq(&first.backend, &second.backend));
18680 assert!(Arc::ptr_eq(
18681 &first.embedding_backend,
18682 &second.embedding_backend
18683 ));
18684 }
18685
18686 let loads = Arc::new(std::sync::atomic::AtomicU64::new(0));
18687 let first_cache = first._runtime_scope.load_probe.clone();
18688 let second_cache = second._runtime_scope.load_probe.clone();
18689 let mut threads = Vec::new();
18690 for cache in [first_cache, second_cache] {
18691 let loads = loads.clone();
18692 threads.push(std::thread::spawn(move || {
18693 cache
18694 .get_or_load::<()>("same/model", 1, || {
18695 loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
18696 std::thread::sleep(std::time::Duration::from_millis(20));
18697 Ok(())
18698 })
18699 .unwrap()
18700 }));
18701 }
18702 let handles = threads
18703 .into_iter()
18704 .map(|thread| thread.join().unwrap())
18705 .collect::<Vec<_>>();
18706 assert_eq!(loads.load(std::sync::atomic::Ordering::SeqCst), 1);
18707 assert!(Arc::ptr_eq(&handles[0], &handles[1]));
18708
18709 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
18710 .arg("--exact")
18711 .arg("tests::same_state_root_engines_share_runtime_components")
18712 .env(CHILD_ENV, "1")
18713 .stdin(std::process::Stdio::null())
18714 .stdout(std::process::Stdio::null())
18715 .stderr(std::process::Stdio::null())
18716 .kill_on_drop(true)
18717 .spawn()
18718 .unwrap();
18719 first
18720 .vllm_pool
18721 .insert_test_process("vllm-mlx/two-engine", child)
18722 .await;
18723 first.local_admission.mark_resident_allocation(
18724 "vllm-mlx/two-engine",
18725 &resource_policy::vllm_process_allocation_id("vllm-mlx/two-engine"),
18726 1,
18727 );
18728 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
18729 drop(first);
18730 assert!(second.vllm_pool.contains("vllm-mlx/two-engine").await);
18731 assert!(second.local_admission.is_resident("vllm-mlx/two-engine"));
18732 assert!(second.vllm_pool.evict_model("vllm-mlx/two-engine").await);
18733 assert!(!second.local_admission.is_resident("vllm-mlx/two-engine"));
18734 }
18735
18736 #[tokio::test]
18737 async fn last_engine_drop_reaps_vllm_before_new_runtime_admission() {
18738 const CHILD_ENV: &str = "CAR_LAST_ENGINE_VLLM_DROP_TEST_CHILD";
18739 if std::env::var_os(CHILD_ENV).is_some() {
18740 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
18741 return;
18742 }
18743 let root = tempfile::tempdir().unwrap();
18744 let config = test_config(root.path().join("weights"));
18745 let engine = InferenceEngine::new(config.clone());
18746 let coordinator = engine.local_admission.clone();
18747 let schema = engine
18748 .unified_registry()
18749 .all()
18750 .find(|schema| schema.is_vllm_mlx())
18751 .cloned()
18752 .expect("supervised vllm schema");
18753 let child = tokio::process::Command::new(std::env::current_exe().unwrap())
18754 .arg("--exact")
18755 .arg("tests::last_engine_drop_reaps_vllm_before_new_runtime_admission")
18756 .env(CHILD_ENV, "1")
18757 .stdin(std::process::Stdio::null())
18758 .stdout(std::process::Stdio::null())
18759 .stderr(std::process::Stdio::null())
18760 .kill_on_drop(true)
18761 .spawn()
18762 .unwrap();
18763 engine
18764 .vllm_pool
18765 .insert_test_process(&schema.id, child)
18766 .await;
18767 coordinator.mark_resident_allocation(
18768 &schema.id,
18769 &resource_policy::vllm_process_allocation_id(&schema.id),
18770 1,
18771 );
18772
18773 drop(engine);
18774
18775 tokio::time::timeout(std::time::Duration::from_secs(2), async {
18776 while coordinator.teardown_pending(&schema.id) {
18777 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
18778 }
18779 })
18780 .await
18781 .expect("last runtime drop must eventually confirm supervised child exit");
18782 assert!(!coordinator.is_resident(&schema.id));
18783 let next = InferenceEngine::new(config);
18784 assert!(Arc::ptr_eq(&coordinator, &next.local_admission));
18785 let preflight = next.local_model_preflight(&schema.id, 0).unwrap();
18786 assert_eq!(preflight.resident_model_mb, 0);
18787 assert!(preflight.estimated_incremental_mb >= preflight.estimate.weights_mb);
18788 }
18789
18790 #[cfg(unix)]
18791 #[test]
18792 fn symlinked_state_roots_share_exact_coordinator_and_runtime() {
18793 use std::os::unix::fs::symlink;
18794
18795 let fixture = tempfile::tempdir().unwrap();
18796 let real = fixture.path().join("real-state");
18797 std::fs::create_dir(&real).unwrap();
18798 let alias = fixture.path().join("state-alias");
18799 symlink(&real, &alias).unwrap();
18800
18801 let first = InferenceEngine::new(test_config(real.join("weights")));
18802 let second = InferenceEngine::new(test_config(alias.join("weights")));
18803 assert!(Arc::ptr_eq(&first.local_admission, &second.local_admission));
18804 assert!(Arc::ptr_eq(&first._runtime_scope, &second._runtime_scope));
18805 assert!(Arc::ptr_eq(&first.model_budget, &second.model_budget));
18806 }
18807
18808 #[tokio::test]
18809 async fn stream_reservation_lives_until_returned_receiver_is_released() {
18810 let root = tempfile::tempdir().unwrap();
18811 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18812 let schema = crate::registry::builtin_catalog()
18813 .into_iter()
18814 .find(|schema| schema.is_local() && !schema.is_vllm_mlx())
18815 .expect("local model schema");
18816 engine.local_admission.mark_resident(&schema.id, 1);
18817 let reservation = engine.reserve_local_request(&schema, 64).unwrap();
18818 assert_eq!(engine.local_admission.active_request_count(&schema.id), 1);
18819 let (source_tx, source_rx) = tokio::sync::mpsc::channel(1);
18820 let returned =
18821 InferenceEngine::hold_optional_reservation_for_stream(source_rx, Some(reservation));
18822
18823 drop(returned);
18824 source_tx
18825 .send(stream::StreamEvent::TextDelta("release".into()))
18826 .await
18827 .unwrap();
18828 for _ in 0..20 {
18829 if engine.local_admission.active_request_count(&schema.id) == 0 {
18830 break;
18831 }
18832 tokio::task::yield_now().await;
18833 }
18834 assert_eq!(engine.local_admission.active_request_count(&schema.id), 0);
18835
18836 let compact_source = include_str!("lib.rs")
18837 .split_whitespace()
18838 .collect::<String>();
18839 let remote_handoff = [
18840 "Self::hold_optional_reservation_for_stream",
18841 "(receiver,",
18842 "candidate_reservation.take(),",
18843 ");",
18844 ]
18845 .concat();
18846 assert!(compact_source.contains(&remote_handoff));
18847 }
18848
18849 #[tokio::test]
18850 async fn residual_voice_allocation_blocks_removal_with_typed_error() {
18851 let root = tempfile::tempdir().unwrap();
18852 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18853 engine
18854 .local_admission
18855 .mark_resident("voice/removal-fixture", 1);
18856
18857 let error = match engine
18858 .prepare_local_model_removal("voice/removal-fixture")
18859 .await
18860 {
18861 Err(error) => error,
18862 Ok(_) => panic!("voice allocation must be released by its owner first"),
18863 };
18864 assert!(matches!(
18865 error,
18866 crate::resource_policy::ModelMaintenanceError::ResidualResidency {
18867 model_id,
18868 allocation_ids,
18869 } if model_id == "voice/removal-fixture"
18870 && allocation_ids == vec!["voice/removal-fixture"]
18871 ));
18872 }
18873
18874 #[tokio::test]
18875 async fn catalog_voice_id_blocks_removal_of_live_provider_alias() {
18876 let root = tempfile::tempdir().unwrap();
18877 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18878 engine.local_admission.register_model_aliases(
18879 "voice/parakeet-tdt-0.6b",
18880 ["mlx/parakeet-tdt-0.6b-v3:default"],
18881 );
18882 let mut reservation = engine
18883 .local_admission
18884 .reserve_measured_host_allocation(
18885 "voice/parakeet-tdt-0.6b",
18886 "voice/parakeet-tdt-0.6b#voice-allocation-0",
18887 1024 * 1024,
18888 0,
18889 )
18890 .unwrap();
18891 reservation.publish_resident_weights(1024 * 1024);
18892 drop(reservation);
18893
18894 let result = engine
18895 .prepare_local_model_removal("mlx/parakeet-tdt-0.6b-v3:default")
18896 .await;
18897 assert!(matches!(
18898 result,
18899 Err(crate::resource_policy::ModelMaintenanceError::ResidualResidency { .. })
18900 ));
18901 }
18902
18903 struct RefusingRemovalOffload;
18904
18905 #[async_trait::async_trait]
18906 impl crate::offload::LocalGenerationOffload for RefusingRemovalOffload {
18907 async fn generate(
18908 &self,
18909 _request: GenerateRequest,
18910 ) -> Result<InferenceResult, InferenceError> {
18911 unreachable!("removal test does not generate")
18912 }
18913
18914 async fn stream(
18915 &self,
18916 _request: GenerateRequest,
18917 ) -> Result<tokio::sync::mpsc::Receiver<StreamEvent>, InferenceError> {
18918 unreachable!("removal test does not stream")
18919 }
18920
18921 async fn resident_models(&self) -> Vec<String> {
18922 vec!["worker/refuses-release".into()]
18923 }
18924
18925 async fn release_model(&self, _model_id: &str) -> Result<bool, InferenceError> {
18926 Ok(false)
18927 }
18928 }
18929
18930 #[tokio::test]
18931 async fn worker_release_boolean_is_required_for_model_removal() {
18932 let _offload_guard = crate::offload::test_offload_lock().lock().await;
18933 crate::offload::set_local_offload(Some(Arc::new(RefusingRemovalOffload)));
18934 let root = tempfile::tempdir().unwrap();
18935 let engine = InferenceEngine::new(test_config(root.path().join("weights")));
18936 let result = engine
18937 .prepare_local_model_removal("worker/refuses-release")
18938 .await;
18939 crate::offload::set_local_offload(None);
18940
18941 assert!(matches!(
18942 result,
18943 Err(crate::resource_policy::ModelMaintenanceError::WorkerReleaseUnacknowledged(
18944 model_id
18945 )) if model_id == "worker/refuses-release"
18946 ));
18947 }
18948
18949 #[test]
18950 fn transient_mlx_vlm_allocation_never_becomes_resident() {
18951 struct FixedProbe;
18952 impl crate::resource_policy::LiveMemoryProbe for FixedProbe {
18953 fn available_memory_mb(
18954 &self,
18955 ) -> Result<Option<u64>, crate::resource_policy::ResourcePolicyError> {
18956 Ok(Some(24_000))
18957 }
18958 }
18959
18960 let root = tempfile::tempdir().unwrap();
18961 let mut engine = InferenceEngine::new(test_config(root.path().join("weights")));
18962 engine.local_admission = Arc::new(
18963 crate::resource_policy::LocalAdmissionCoordinator::with_probe(
18964 crate::resource_policy::ResourcePolicy::custom_gb(8.0).unwrap(),
18965 crate::hardware::HardwareInfo {
18966 total_ram_mb: 32 * 1024,
18967 ..crate::hardware::HardwareInfo::detect()
18968 },
18969 Arc::new(FixedProbe),
18970 ),
18971 );
18972 let mut schema = crate::registry::builtin_catalog()
18973 .into_iter()
18974 .find(|schema| schema.tags.iter().any(|tag| tag == "mlx-vlm-cli"))
18975 .expect("one-shot mlx-vlm schema");
18976 schema.param_count = "1M".into();
18977 schema.quantization = Some(Quantization::parse("Q4"));
18978 let mut reservation = engine.reserve_local_request(&schema, 64).unwrap();
18979 InferenceEngine::reconcile_transient_local_allocation(&mut reservation, 1024 * 1024)
18980 .unwrap();
18981 drop(reservation);
18982 assert!(!engine.local_admission.is_resident(&schema.id));
18983 }
18984
18985 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18986 #[test]
18987 fn non_apple_backend_map_keeps_two_local_model_identities_exact() {
18988 let mut backends = std::collections::HashMap::<String, u32>::new();
18989 backends.insert("local/model-a".into(), 1);
18990 backends.insert("local/model-b".into(), 2);
18991 assert_eq!(backends.get("local/model-a"), Some(&1));
18992 assert_eq!(backends.get("local/model-b"), Some(&2));
18993 assert_eq!(backends.remove("local/model-a"), Some(1));
18994 assert!(!backends.contains_key("local/model-a"));
18995 assert_eq!(backends.get("local/model-b"), Some(&2));
18996 }
18997
18998 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
18999 #[tokio::test]
19000 async fn non_apple_remote_classification_at_zero_gb_routes_without_local_allocation() {
19001 use wiremock::matchers::{method, path};
19002 use wiremock::{Mock, MockServer, ResponseTemplate};
19003
19004 let _credential_scope = crate::openrouter::test_credential_scope();
19005 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
19006 let server = MockServer::start().await;
19007 Mock::given(method("POST"))
19008 .and(path("/v1/chat/completions"))
19009 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
19010 "choices": [{
19011 "message": {"role": "assistant", "content": "positive"},
19012 "finish_reason": "stop"
19013 }],
19014 "usage": {"prompt_tokens": 16, "completion_tokens": 1}
19015 })))
19016 .mount(&server)
19017 .await;
19018
19019 let fixture = tempfile::tempdir().unwrap();
19020 let config = test_config(fixture.path().join("weights"));
19021 let repository =
19022 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19023 crate::resource_policy::ResourcePolicyRepository::save(
19024 &repository,
19025 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19026 )
19027 .unwrap();
19028 let mut engine = InferenceEngine::new(config);
19029 let mut remote = crate::openrouter::curated_schemas()
19030 .into_iter()
19031 .next()
19032 .expect("OpenRouter schema");
19033 remote.id = "openrouter/test/classifier".into();
19034 remote.name = remote.id.clone();
19035 if let ModelSource::RemoteApi { endpoint, .. } = &mut remote.source {
19036 *endpoint = server.uri();
19037 }
19038 let remote_id = remote.id.clone();
19039 engine.unified_registry_mut().register_project_model(remote);
19040
19041 let result = engine
19042 .classify(ClassifyRequest {
19043 work_context: None,
19044 text: "a good outcome".into(),
19045 labels: vec!["positive".into(), "negative".into()],
19046 model: Some(remote_id),
19047 })
19048 .await
19049 .unwrap();
19050 assert_eq!(
19051 result.first().map(|item| item.label.as_str()),
19052 Some("positive")
19053 );
19054 assert_eq!(engine.local_admission.resident_model_mb(), 0);
19055 assert!(
19056 !engine.config.models_dir.exists()
19057 || std::fs::read_dir(&engine.config.models_dir)
19058 .unwrap()
19059 .next()
19060 .is_none(),
19061 "remote classification must not download local weights"
19062 );
19063 }
19064
19065 #[test]
19066 fn adaptive_speech_skips_blocked_local_but_explicit_speech_fails() {
19067 let fixture = TempDir::new().unwrap();
19068 let config = InferenceConfig {
19069 models_dir: fixture.path().join("weights"),
19070 state_root: fixture.path().join("state"),
19071 ..Default::default()
19072 };
19073 let repository =
19074 crate::resource_policy::FileResourcePolicyRepository::new(config.state_root.clone());
19075 crate::resource_policy::ResourcePolicyRepository::save(
19076 &repository,
19077 &crate::resource_policy::ResourcePolicy::custom_gb(0.0).unwrap(),
19078 )
19079 .unwrap();
19080 let engine = InferenceEngine::new(config);
19081 let local = engine
19082 .unified_registry()
19083 .all()
19084 .find(|schema| {
19085 schema.is_local() && schema.has_capability(ModelCapability::SpeechToText)
19086 })
19087 .cloned()
19088 .expect("built-in local STT model");
19089
19090 assert!(matches!(
19091 engine.admit_speech_candidate(&local, false),
19092 SpeechCandidateAdmission::SkipBlocked(_)
19093 ));
19094 assert!(matches!(
19095 engine.admit_speech_candidate(&local, true),
19096 SpeechCandidateAdmission::FailBlocked(InferenceError::LocalResourceBlocked { .. })
19097 ));
19098
19099 let mut os_owned = local;
19100 os_owned.id = "windows/speech-synthesis:test".into();
19101 os_owned.source = ModelSource::WindowsSpeech {};
19102 assert!(matches!(
19103 engine.admit_speech_candidate(&os_owned, true),
19104 SpeechCandidateAdmission::Proceed(None)
19105 ));
19106 }
19107
19108 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19109 async fn cancelled_native_blocking_work_keeps_its_exact_charge_until_completion() {
19110 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
19111 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
19112 crate::hardware::HardwareInfo {
19113 os: "test".into(),
19114 arch: "test".into(),
19115 cpu_cores: 8,
19116 total_ram_mb: 32 * 1024,
19117 gpu_backend: crate::hardware::GpuBackend::Cpu,
19118 gpu_memory_mb: None,
19119 gpu_devices: Vec::new(),
19120 recommended_model: "fixture".into(),
19121 recommended_context: 4096,
19122 max_model_mb: 32 * 1024,
19123 },
19124 ));
19125 let reservation = coordinator
19126 .reserve_measured_host("detached-native-a", 512 * 1024 * 1024, 0)
19127 .unwrap();
19128 let lease = reservation.detached_lease();
19129 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
19130 let release =
19131 std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
19132 let release_worker = release.clone();
19133 let operation = tokio::spawn(async move {
19134 let _request_reservation = reservation;
19135 run_admitted_blocking(lease, move || {
19136 let _ = started_tx.send(());
19137 let (lock, ready) = &*release_worker;
19138 let mut released = lock
19139 .lock()
19140 .unwrap_or_else(std::sync::PoisonError::into_inner);
19141 while !*released {
19142 released = ready
19143 .wait(released)
19144 .unwrap_or_else(std::sync::PoisonError::into_inner);
19145 }
19146 })
19147 .await
19148 });
19149 started_rx.await.unwrap();
19150 operation.abort();
19151 let _ = operation.await;
19152
19153 let blocked = coordinator.reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0);
19154 assert!(
19155 blocked.is_err(),
19156 "cancelling the await must not advertise memory still owned by spawn_blocking"
19157 );
19158
19159 let (lock, ready) = &*release;
19160 *lock
19161 .lock()
19162 .unwrap_or_else(std::sync::PoisonError::into_inner) = true;
19163 ready.notify_one();
19164 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19165 loop {
19166 if coordinator
19167 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
19168 .is_ok()
19169 {
19170 break;
19171 }
19172 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19173 }
19174 })
19175 .await
19176 .expect("the detached charge must clear after native work really exits");
19177 }
19178
19179 #[test]
19180 fn nonstream_native_text_generation_uses_cancellable_blocking_boundary() {
19181 let source = include_str!("lib.rs");
19182 let mlx = source
19183 .split("async fn generate_mlx(")
19184 .nth(1)
19185 .and_then(|tail| tail.split("async fn generate_local(").next())
19186 .expect("generate_mlx source");
19187 let local = source
19188 .split("async fn generate_local(")
19189 .nth(1)
19190 .and_then(|tail| tail.split("fn apply_top_k_top_p").next())
19191 .expect("generate_local source");
19192
19193 assert!(
19194 mlx.contains("run_admitted_blocking"),
19195 "native MLX decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
19196 );
19197 assert!(
19198 local.contains("run_admitted_blocking"),
19199 "polymorphic local decode must yield the Tokio runtime so a WS timeout/cancel can preempt its response waiter"
19200 );
19201 }
19202
19203 #[cfg(unix)]
19204 #[tokio::test]
19205 async fn cancelled_speech_subprocess_is_killed_and_reaped_before_charge_clears() {
19206 use std::os::unix::fs::PermissionsExt;
19207
19208 let fixture = TempDir::new().unwrap();
19209 let runtime = SpeechRuntime::new(fixture.path().join("speech-runtime"));
19210 std::fs::create_dir_all(runtime.stt_program.parent().unwrap()).unwrap();
19211 std::fs::write(
19212 &runtime.stt_program,
19213 b"#!/bin/sh\necho started > \"$1\"\nsleep 1\necho continued > \"$2\"\n",
19214 )
19215 .unwrap();
19216 let mut permissions = std::fs::metadata(&runtime.stt_program)
19217 .unwrap()
19218 .permissions();
19219 permissions.set_mode(0o755);
19220 std::fs::set_permissions(&runtime.stt_program, permissions).unwrap();
19221
19222 let coordinator = std::sync::Arc::new(resource_policy::LocalAdmissionCoordinator::new(
19223 resource_policy::ResourcePolicy::custom_gb(1.0).unwrap(),
19224 crate::hardware::HardwareInfo {
19225 total_ram_mb: 32 * 1024,
19226 ..crate::hardware::HardwareInfo::detect()
19227 },
19228 ));
19229 let reservation = coordinator
19230 .reserve_measured_host("mlx-audio-a", 512 * 1024 * 1024, 0)
19231 .unwrap();
19232 let lease = reservation.detached_lease();
19233 let started = fixture.path().join("started");
19234 let continued = fixture.path().join("continued");
19235 let args = vec![
19236 started.display().to_string(),
19237 continued.display().to_string(),
19238 ];
19239 let command = tokio::spawn(async move {
19240 let _request_reservation = reservation;
19241 run_mlx_audio_command(&runtime, "stt.generate", &args, lease).await
19242 });
19243 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19244 while !started.exists() {
19245 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19246 }
19247 })
19248 .await
19249 .expect("speech fixture child must start");
19250 command.abort();
19251 let _ = command.await;
19252
19253 tokio::time::timeout(std::time::Duration::from_secs(2), async {
19254 loop {
19255 if coordinator
19256 .reserve_measured_host("different-model-b", 768 * 1024 * 1024, 0)
19257 .is_ok()
19258 {
19259 break;
19260 }
19261 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
19262 }
19263 })
19264 .await
19265 .expect("charge clears only after the cancelled child is reaped");
19266 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
19267 assert!(
19268 !continued.exists(),
19269 "the cancelled local speech process must not keep allocating in the background"
19270 );
19271 }
19272
19273 #[test]
19274 fn local_model_eviction_surface_includes_every_in_process_cache() {
19275 let source = include_str!("lib.rs");
19276 for cache in [
19277 "self.text_backends.evict_if_idle(model_id)",
19278 "self.local_backends.evict_if_idle(model_id)",
19279 "self.flux_cache.evict_if_idle(model_id)",
19280 "self.ltx_cache.evict_if_idle(model_id)",
19281 "self.kokoro_cache.evict_if_idle(model_id)",
19282 "self.local_backends.evict_idle()",
19283 ] {
19284 assert!(source.contains(cache), "missing eviction seam: {cache}");
19285 }
19286 }
19287
19288 #[tokio::test]
19294 async fn catalog_refresh_uses_authority_hints_without_secret_reads() {
19295 if !crate::run_in_isolated_test_process(
19296 "tests::catalog_refresh_uses_authority_hints_without_secret_reads",
19297 "CAR_CATALOG_ZERO_SECRET_READ_CHILD",
19298 ) {
19299 return;
19300 }
19301 let _environment = ENV_MUTEX.lock().await;
19302 let fixture = TempDir::new().unwrap();
19303 let credential_envs = [
19304 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19305 car_auth::PARSLEE_API_BASE_KEY,
19306 crate::openrouter::API_KEY_ENV,
19307 "OPENAI_API_KEY",
19308 "ANTHROPIC_API_KEY",
19309 "GOOGLE_API_KEY",
19310 "ELEVENLABS_API_KEY",
19311 ];
19312 let mut restored_names = vec![car_home::ENV_VAR, "CAR_SECRETS_FILE_DIR"];
19313 restored_names.extend(credential_envs);
19314 let _restore = RestoredEnvironment::capture(&restored_names);
19315 unsafe {
19316 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
19317 std::env::set_var(
19318 "CAR_SECRETS_FILE_DIR",
19319 fixture.path().join("isolated-secrets"),
19320 );
19321 for name in credential_envs {
19322 std::env::remove_var(name);
19323 }
19324 }
19325
19326 let before = car_secrets::secret_store_activity();
19327 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
19328
19329 let listed = engine.list_models_unified();
19331 let schemas = engine.list_schemas();
19332 for provider in ["parslee", "openai", "anthropic", "google", "elevenlabs"] {
19333 assert!(
19334 schemas.iter().any(|schema| schema.provider == provider),
19335 "real builtin catalog lost the credential-bearing {provider} rows"
19336 );
19337 }
19338 assert!(
19339 schemas.iter().any(|schema| matches!(
19340 schema.source,
19341 ModelSource::RemoteApi {
19342 protocol: ApiProtocol::OpenRouter,
19343 ..
19344 }
19345 )),
19346 "real builtin catalog lost the reviewed OpenRouter rows"
19347 );
19348 assert_eq!(listed.len(), schemas.len());
19349
19350 let schema_refs: Vec<&ModelSchema> = schemas.iter().collect();
19352 let _setup_plan = crate::recommend(
19353 &schema_refs,
19354 &HardwareInfo::detect(),
19355 UseCase::default(),
19356 QualityTier::default(),
19357 Privacy::OnDevice,
19358 );
19359
19360 let _speech = engine.speech_health();
19362 let _concierge = engine.concierge_status(false).await;
19363 let _health = engine.model_health().await;
19364
19365 let after = car_secrets::secret_store_activity();
19366 assert_eq!(
19367 after, before,
19368 "passive builtin catalog surfaces performed secret-store operations"
19369 );
19370 }
19371
19372 #[tokio::test]
19373 async fn list_models_unified_and_model_health_are_zero_secret_store_probes() {
19374 if !crate::run_in_isolated_test_process(
19375 "tests::list_models_unified_and_model_health_are_zero_secret_store_probes",
19376 "CAR_DIRECT_MODEL_SURFACES_ZERO_SECRET_CHILD",
19377 ) {
19378 return;
19379 }
19380 let _environment = ENV_MUTEX.lock().await;
19381 let fixture = TempDir::new().unwrap();
19382 let _restore = RestoredEnvironment::capture(&[
19383 car_home::ENV_VAR,
19384 "CAR_SECRETS_FILE_DIR",
19385 crate::openrouter::API_KEY_ENV,
19386 ]);
19387 unsafe {
19388 std::env::set_var(car_home::ENV_VAR, fixture.path().join("car-home"));
19389 std::env::set_var(
19390 "CAR_SECRETS_FILE_DIR",
19391 fixture.path().join("isolated-secrets"),
19392 );
19393 std::env::remove_var(crate::openrouter::API_KEY_ENV);
19394 }
19395 let engine = InferenceEngine::new(test_config(fixture.path().join("models")));
19396 let before = car_secrets::secret_store_activity();
19397
19398 let rows = engine.list_models_unified();
19399 let health = engine.model_health().await;
19400
19401 assert!(!rows.is_empty(), "the unified catalog fixture must exist");
19402 assert!(health.total_models > 0, "the health fixture must exist");
19403 let after = car_secrets::secret_store_activity();
19404 assert_eq!(after.status_attempts, before.status_attempts);
19405 assert_eq!(after.get_attempts, before.get_attempts);
19406
19407 assert!(
19417 rows.iter().all(|row| row.credential_required.is_none()),
19418 "a passive catalog read must not populate `credential_required` — its \
19419 availability verdict cannot see the secret store, so any credential it \
19420 named might already be present: {:?}",
19421 rows.iter()
19422 .filter(|row| row.credential_required.is_some())
19423 .map(|row| (&row.id, &row.credential_required))
19424 .collect::<Vec<_>>()
19425 );
19426 }
19427
19428 #[test]
19438 fn car_home_moves_the_catalog_and_discovery_caches_but_never_the_weights() {
19439 let _environment = crate::openrouter::test_environment_scope();
19440 let prior = std::env::var_os(car_home::ENV_VAR);
19441
19442 unsafe { std::env::remove_var(car_home::ENV_VAR) };
19443 let shared = InferenceConfig::default();
19444 let default_catalog = crate::catalog::cache_path(&shared.state_root);
19445 let default_discovery = crate::discovery::cache_path(&shared.state_models_dir());
19446
19447 let alt = Path::new("/tmp/car-home-inference-cache-test");
19448 unsafe { std::env::set_var(car_home::ENV_VAR, alt) };
19449 let relocated = InferenceConfig::default();
19450 let catalog = crate::catalog::cache_path(&relocated.state_root);
19451 let discovery = crate::discovery::cache_path(&relocated.state_models_dir());
19452
19453 match prior {
19454 Some(value) => unsafe { std::env::set_var(car_home::ENV_VAR, value) },
19455 None => unsafe { std::env::remove_var(car_home::ENV_VAR) },
19456 }
19457
19458 assert_eq!(catalog, alt.join(crate::catalog::CATALOG_CACHE_FILE));
19459 assert_eq!(
19460 discovery,
19461 alt.join("models")
19462 .join(crate::discovery::DISCOVERED_MODELS_FILE)
19463 );
19464 assert_ne!(
19465 catalog, default_catalog,
19466 "the catalog cache must not resolve back into the shared root",
19467 );
19468 assert_ne!(
19469 discovery, default_discovery,
19470 "the discovery cache must not resolve back into the shared root",
19471 );
19472
19473 assert_eq!(
19474 relocated.models_dir, shared.models_dir,
19475 "the weights cache is machine-global and must not follow CAR_HOME",
19476 );
19477 assert!(
19478 !relocated.models_dir.starts_with(alt),
19479 "the weights cache must not be dragged under the override",
19480 );
19481 }
19482
19483 #[derive(Clone, Copy)]
19484 enum CacheRoutingSurface {
19485 Generate,
19486 Stream,
19487 }
19488
19489 async fn invoke_cache_routed_openrouter(
19495 surface: CacheRoutingSurface,
19496 cache_read_estimate: usize,
19497 cache_write_estimate: usize,
19498 ) -> String {
19499 use wiremock::matchers::{method, path};
19500 use wiremock::{Mock, MockServer, ResponseTemplate};
19501
19502 let _credential_scope = crate::openrouter::test_credential_scope();
19503 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
19504 let server = MockServer::start().await;
19505 let response = match surface {
19506 CacheRoutingSurface::Generate => ResponseTemplate::new(200).set_body_json(
19507 serde_json::json!({
19508 "choices": [{
19509 "message": {"role": "assistant", "content": "cache-route-ok"},
19510 "finish_reason": "stop"
19511 }],
19512 "usage": {"prompt_tokens": 40_000, "completion_tokens": 8}
19513 }),
19514 ),
19515 CacheRoutingSurface::Stream => ResponseTemplate::new(200).set_body_raw(
19516 concat!(
19517 "data: {\"choices\":[{\"delta\":{\"content\":\"cache-route-ok\"},\"finish_reason\":\"stop\"}]}\n\n",
19518 "data: [DONE]\n\n"
19519 ),
19520 "text/event-stream",
19521 ),
19522 };
19523 Mock::given(method("POST"))
19524 .and(path("/v1/chat/completions"))
19525 .respond_with(response)
19526 .mount(&server)
19527 .await;
19528
19529 let tmp = TempDir::new().unwrap();
19530 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19531 let mut uncached_cheap = crate::openrouter::curated_schemas()
19532 .into_iter()
19533 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
19534 .unwrap();
19535 uncached_cheap.id = "openrouter/test/uncached-cheap".into();
19536 uncached_cheap.name = uncached_cheap.id.clone();
19537 uncached_cheap.cost = CostModel {
19538 input_per_mtok: Some(1.0),
19539 output_per_mtok: Some(1.0),
19540 cache_read_input_per_mtok: Some(100.0),
19541 cache_write_input_per_mtok: Some(100.0),
19542 ..Default::default()
19543 };
19544 if let ModelSource::RemoteApi { endpoint, .. } = &mut uncached_cheap.source {
19545 *endpoint = server.uri();
19546 }
19547
19548 let mut cached_cheap = uncached_cheap.clone();
19549 cached_cheap.id = "openrouter/test/cached-cheap".into();
19550 cached_cheap.name = cached_cheap.id.clone();
19551 cached_cheap.cost = CostModel {
19552 input_per_mtok: Some(80.0),
19553 output_per_mtok: Some(1.0),
19554 cache_read_input_per_mtok: Some(0.001),
19555 cache_write_input_per_mtok: Some(0.001),
19556 ..Default::default()
19557 };
19558
19559 let uncached_id = uncached_cheap.id.clone();
19560 let cached_id = cached_cheap.id.clone();
19561 engine
19562 .unified_registry_mut()
19563 .register_project_model(uncached_cheap);
19564 engine
19565 .unified_registry_mut()
19566 .register_project_model(cached_cheap);
19567 let exclude_models = engine
19568 .list_schemas()
19569 .into_iter()
19570 .map(|schema| schema.id)
19571 .filter(|id| id != &uncached_id && id != &cached_id)
19572 .collect();
19573
19574 let mut params = GenerateParams {
19578 max_tokens: 8,
19579 ..Default::default()
19580 };
19581 assert_eq!(params.estimated_cache_read_input_tokens, 0);
19582 assert_eq!(params.estimated_cache_write_input_tokens, 0);
19583 params.estimated_cache_read_input_tokens = cache_read_estimate;
19584 params.estimated_cache_write_input_tokens = cache_write_estimate;
19585 let req = GenerateRequest {
19586 prompt: "x".repeat(160_000),
19587 params,
19588 cache_control: true,
19589 intent: Some(IntentHint {
19590 prefer_quality: true,
19591 exclude_models,
19592 ..Default::default()
19593 }),
19594 ..Default::default()
19595 };
19596
19597 match surface {
19598 CacheRoutingSurface::Generate => {
19599 engine
19600 .generate_tracked(req)
19601 .await
19602 .expect("mocked OpenRouter generation should succeed")
19603 .model_used
19604 }
19605 CacheRoutingSurface::Stream => {
19606 let mut handle = engine
19607 .generate_tracked_stream(req)
19608 .await
19609 .expect("mocked OpenRouter stream should start");
19610 let selected = handle.model_used.clone();
19611 while handle.events.recv().await.is_some() {}
19612 selected
19613 }
19614 }
19615 }
19616
19617 #[tokio::test(flavor = "current_thread")]
19618 async fn tracked_generate_uses_explicit_cache_estimate_and_defaults_to_zero() {
19619 let without_estimate =
19620 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 0, 0).await;
19621 let with_estimate =
19622 invoke_cache_routed_openrouter(CacheRoutingSurface::Generate, 40_000, 0).await;
19623 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
19624 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
19625 }
19626
19627 #[tokio::test(flavor = "current_thread")]
19628 async fn tracked_stream_uses_explicit_cache_estimate_and_defaults_to_zero() {
19629 let without_estimate =
19630 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 0).await;
19631 let with_estimate =
19632 invoke_cache_routed_openrouter(CacheRoutingSurface::Stream, 0, 40_000).await;
19633 assert_eq!(without_estimate, "openrouter/test/uncached-cheap");
19634 assert_eq!(with_estimate, "openrouter/test/cached-cheap");
19635 }
19636
19637 #[tokio::test(flavor = "current_thread")]
19638 async fn authenticated_openrouter_registry_stays_static_and_rejects_unknown_ids() {
19639 let _credential_scope = crate::openrouter::test_credential_scope();
19640 crate::openrouter::set_test_credential(Some("static-key"));
19641 let tmp = TempDir::new().unwrap();
19642 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19643 let personal: Vec<_> = engine
19644 .list_schemas()
19645 .into_iter()
19646 .filter(|schema| schema.id.starts_with("openrouter/"))
19647 .collect();
19648 assert_eq!(personal.len(), crate::openrouter::curated_model_count());
19649 assert!(personal.iter().all(ModelSchema::available_now));
19650 assert!(personal
19651 .iter()
19652 .all(|schema| schema.trust_tier == TrustTier::Curated));
19653 assert!(personal
19654 .iter()
19655 .all(|schema| !schema.tags.iter().any(|tag| tag == "dynamic")));
19656
19657 for unknown in [
19658 "openrouter/vendor/brand-new-model",
19659 "openrouter/openai/gpt-5.4-typo",
19660 ] {
19661 assert!(engine
19662 .list_schemas()
19663 .iter()
19664 .all(|schema| schema.id != unknown));
19665 assert_eq!(engine.model_context_window(unknown), 0);
19666 let error = engine
19667 .generate_tracked(GenerateRequest {
19668 prompt: "must fail before transport".into(),
19669 model: Some(unknown.into()),
19670 params: GenerateParams {
19671 strict_model: true,
19672 ..Default::default()
19673 },
19674 ..Default::default()
19675 })
19676 .await
19677 .expect_err("unregistered personal OpenRouter ids must not reach inference");
19678 assert!(
19679 matches!(&error, InferenceError::ModelNotFound(id) if id == unknown),
19680 "{unknown}: {error}"
19681 );
19682 let stream_error = match engine
19683 .generate_tracked_stream(GenerateRequest {
19684 prompt: "must fail before stream transport".into(),
19685 model: Some(unknown.into()),
19686 ..Default::default()
19687 })
19688 .await
19689 {
19690 Ok(_) => panic!("unregistered ids must also fail before streaming"),
19691 Err(error) => error,
19692 };
19693 assert!(
19694 matches!(&stream_error, InferenceError::ModelNotFound(id) if id == unknown),
19695 "{unknown}: {stream_error}"
19696 );
19697 }
19698 }
19699
19700 #[tokio::test(flavor = "current_thread")]
19701 async fn static_openrouter_rows_participate_in_adaptive_routing_only_with_a_key() {
19702 let (_credential_scope, _provider_env) =
19703 crate::openrouter::test_credential_and_environment_scope_async().await;
19704 let tmp = TempDir::new().unwrap();
19705 crate::openrouter::set_test_credential(Some("static-key"));
19706 unsafe {
19707 std::env::set_var("CAR_STATIC_ROUTING_PEER_KEY", "peer-key");
19708 }
19709 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
19710 let mut peer = remote_stream_fixture_schema(
19711 "test/openai-routing-peer",
19712 "http://127.0.0.1:9".into(),
19713 schema::ApiProtocol::OpenAiCompat,
19714 "CAR_STATIC_ROUTING_PEER_KEY",
19715 );
19716 peer.provider = "openai".into();
19717 peer.trust_tier = TrustTier::Curated;
19718 engine.unified_registry_mut().register_project_model(peer);
19719 let reviewed: std::collections::HashSet<_> = engine
19720 .list_schemas()
19721 .into_iter()
19722 .filter(|schema| schema.id.starts_with("openrouter/"))
19723 .map(|schema| schema.id)
19724 .collect();
19725 assert_eq!(reviewed.len(), crate::openrouter::curated_model_count());
19726
19727 let with_key = engine
19728 .route_adaptive_with_intent(
19729 "Answer this simple question cheaply.",
19730 Some(IntentHint::default()),
19731 )
19732 .await;
19733 let openrouter_candidates: Vec<_> = std::iter::once(with_key.model_id.as_str())
19734 .chain(
19735 with_key
19736 .candidates
19737 .iter()
19738 .map(|candidate| candidate.model_id.as_str()),
19739 )
19740 .chain(with_key.fallbacks.iter().map(String::as_str))
19741 .filter(|id| id.starts_with("openrouter/"))
19742 .collect();
19743 assert!(
19744 !openrouter_candidates.is_empty(),
19745 "keyed adaptive decision must include a reviewed OpenRouter row: {with_key:?}"
19746 );
19747 assert!(openrouter_candidates
19748 .iter()
19749 .all(|id| reviewed.contains(*id)));
19750 assert!(
19751 std::iter::once(with_key.model_id.as_str())
19752 .chain(with_key.fallbacks.iter().map(String::as_str),)
19753 .any(|id| !id.starts_with("openrouter/")),
19754 "fallback chain must retain cross-provider alternatives: {with_key:?}"
19755 );
19756
19757 crate::openrouter::set_test_credential(None);
19758 let without_key = engine
19759 .route_adaptive_with_intent(
19760 "Answer this simple question cheaply.",
19761 Some(IntentHint::default()),
19762 )
19763 .await;
19764 assert!(!std::iter::once(without_key.model_id.as_str())
19765 .chain(
19766 without_key
19767 .candidates
19768 .iter()
19769 .map(|candidate| candidate.model_id.as_str()),
19770 )
19771 .chain(without_key.fallbacks.iter().map(String::as_str))
19772 .any(|id| id.starts_with("openrouter/")));
19773 unsafe {
19774 std::env::remove_var("CAR_STATIC_ROUTING_PEER_KEY");
19775 }
19776 }
19777
19778 #[tokio::test(flavor = "current_thread")]
19779 async fn v2_parslee_auth_drives_managed_registration_routing_lane_and_logout() {
19780 if !crate::run_in_isolated_test_process(
19781 "tests::v2_parslee_auth_drives_managed_registration_routing_lane_and_logout",
19782 "CAR_V2_ROUTING_AUTH_CHILD",
19783 ) {
19784 return;
19785 }
19786 let tmp = TempDir::new().unwrap();
19787 let (_credential_scope, _provider_env) =
19788 crate::openrouter::test_credential_and_environment_scope_async().await;
19789 let _restore = RestoredEnvironment::capture(&[
19790 "CAR_SECRETS_FILE_DIR",
19791 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19792 ]);
19793 let _home = crate::openrouter::StateRootScope::new();
19800 crate::openrouter::clear_gateway_unconfigured();
19801 crate::parslee_credential::clear_credential_rejected();
19802 let secrets_dir = tmp.path().join("secrets");
19803 unsafe {
19804 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
19805 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
19806 }
19807 crate::openrouter::set_test_credential(Some("personal-openrouter-key"));
19808
19809 let store = car_secrets::SecretStore::new();
19810 let state_ref =
19811 car_secrets::SecretRef::with_default_service(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
19812 store
19813 .publish(
19814 &state_ref,
19815 &serde_json::json!({
19816 "schema": 2,
19817 "revision": 7,
19818 "generation": 3,
19819 "active": {
19820 "account_id": "account-v2",
19821 "access_token": "v2-access",
19822 "refresh_token": "v2-refresh",
19823 "expires_at": 9_999_999_999_u64,
19824 "api_base": "https://api.parslee.ai"
19825 },
19826 "accounts": [{
19827 "account_id": "account-v2",
19828 "access_token": "v2-access",
19829 "refresh_token": "v2-refresh",
19830 "expires_at": 9_999_999_999_u64,
19831 "api_base": "https://api.parslee.ai"
19832 }],
19833 "tombstone": false
19834 })
19835 .to_string(),
19836 )
19837 .unwrap();
19838 assert_eq!(car_auth::access_token().as_deref(), Some("v2-access"));
19839 assert!(
19840 !store
19841 .status(&car_secrets::SecretRef::with_default_service(
19842 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
19843 ))
19844 .unwrap()
19845 .exists,
19846 "the regression must exercise V2 state without the import-only legacy slot"
19847 );
19848
19849 let managed_ids = [
19850 "parslee/openrouter/frontier-general",
19851 "parslee/openrouter/balanced-general",
19852 ];
19853 let personal_id = "openrouter/deepseek/deepseek-v3.2";
19854 let rogue_id = "community/custom-oauth";
19855 let schemas = crate::openrouter::curated_schemas();
19856 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("registry-models"));
19857 for id in managed_ids.into_iter().chain(std::iter::once(personal_id)) {
19858 registry.register_project_model(
19859 schemas
19860 .iter()
19861 .find(|schema| schema.id == id)
19862 .unwrap_or_else(|| panic!("missing curated schema {id}"))
19863 .clone(),
19864 );
19865 }
19866 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
19867 for id in managed_ids {
19868 assert!(
19869 registry.get(id).unwrap().available_now(),
19870 "{id} must be available immediately when registered from V2 auth"
19871 );
19872 }
19873 assert!(registry.get(personal_id).unwrap().available_now());
19874 let mut rogue = schemas
19875 .iter()
19876 .find(|schema| schema.id == managed_ids[0])
19877 .unwrap()
19878 .clone();
19879 rogue.id = rogue_id.into();
19880 rogue.provider = "community".into();
19881 if let ModelSource::Proprietary {
19882 provider, endpoint, ..
19883 } = &mut rogue.source
19884 {
19885 *provider = "community".into();
19886 *endpoint = "https://untrusted.example".into();
19887 } else {
19888 panic!("managed fixture must remain proprietary");
19889 }
19890 registry.register(rogue);
19891 assert!(
19892 !registry.get(rogue_id).unwrap().available_now(),
19893 "a non-Parslee OAuth schema must not inherit Parslee V2 availability"
19894 );
19895
19896 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
19897 for id in managed_ids {
19898 assert!(
19899 registry.get(id).unwrap().available_now(),
19900 "{id} must stay available in a refreshed V2-auth snapshot"
19901 );
19902 }
19903 assert!(
19904 !registry.get(rogue_id).unwrap().available_now(),
19905 "refresh must keep non-Parslee OAuth schemas unavailable"
19906 );
19907
19908 let router = AdaptiveRouter::new(
19909 crate::hardware::HardwareInfo::detect(),
19910 RoutingConfig {
19911 prefer_local: false,
19912 prior_strength: 1_000_000.0,
19913 quality_first_cold_start: false,
19914 ..RoutingConfig::default()
19915 },
19916 );
19917 let tracker = OutcomeTracker::new();
19918 let intent = IntentHint {
19919 task: Some(crate::intent::TaskHint::Chat),
19920 exclude_models: vec![personal_id.into()],
19921 ..Default::default()
19922 };
19923 let decision = router.route_with(crate::adaptive_router::RouteRequest {
19924 intent: Some(&intent),
19925 ..crate::adaptive_router::RouteRequest::new(
19926 "Explain this architecture.",
19927 ®istry,
19928 &tracker,
19929 )
19930 });
19931 assert!(
19932 managed_ids.contains(&decision.model_id.as_str()),
19933 "managed V2-auth alias must be selectable: {decision:?}"
19934 );
19935 assert!(
19936 decision
19937 .candidates
19938 .iter()
19939 .any(|candidate| managed_ids.contains(&candidate.model_id.as_str())),
19940 "managed V2-auth alias must appear in adaptive candidates: {decision:?}"
19941 );
19942 assert!(
19943 decision
19944 .fallbacks
19945 .iter()
19946 .any(|id| managed_ids.contains(&id.as_str())),
19947 "managed V2-auth alias must appear in adaptive fallbacks: {decision:?}"
19948 );
19949 assert!(
19950 !std::iter::once(decision.model_id.as_str())
19951 .chain(
19952 decision
19953 .candidates
19954 .iter()
19955 .map(|candidate| candidate.model_id.as_str())
19956 )
19957 .chain(decision.fallbacks.iter().map(String::as_str))
19958 .any(|id| id == rogue_id),
19959 "a non-Parslee OAuth schema must never enter adaptive selection, candidates, or fallbacks: {decision:?}"
19960 );
19961
19962 let engine = InferenceEngine::new(test_config(tmp.path().join("engine-models")));
19963 let managed_lane_id = managed_ids[0];
19964 engine.lane_defaults_cache.write().unwrap().set(
19965 None,
19966 crate::intent::UseCase::Assistant,
19967 managed_lane_id.into(),
19968 1,
19969 );
19970 let request = GenerateRequest {
19971 prompt: "lane default".into(),
19972 intent: Some(IntentHint {
19973 task: Some(crate::intent::TaskHint::Chat),
19974 ..Default::default()
19975 }),
19976 ..Default::default()
19977 };
19978 assert_eq!(
19979 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
19980 Some(managed_lane_id.to_string()),
19981 "a managed lane default must resolve from authoritative V2 auth"
19982 );
19983
19984 car_auth::logout()
19985 .await
19986 .expect("production logout must publish a signed-out tombstone");
19987 let persisted_logout: serde_json::Value =
19988 serde_json::from_str(&store.get(&state_ref).unwrap()).unwrap();
19989 assert_eq!(persisted_logout["tombstone"], true);
19990 assert_eq!(persisted_logout["accounts"], serde_json::json!([]));
19991 assert!(persisted_logout.get("active").is_none());
19992 assert_eq!(car_auth::access_token(), None);
19993
19994 registry.refresh_routing_availability(None, true);
19995 for id in managed_ids {
19996 assert!(
19997 !registry.get(id).unwrap().available_now(),
19998 "{id} must disappear from routing after the signed-out tombstone"
19999 );
20000 }
20001 assert!(
20002 registry.get(personal_id).unwrap().available_now(),
20003 "personal rows must continue to follow their independent personal-key seam"
20004 );
20005
20006 let signed_out = engine.routing_registry_snapshot().await;
20007 assert!(
20008 signed_out
20009 .list()
20010 .into_iter()
20011 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
20012 .all(|schema| !schema.available_now()),
20013 "the next engine snapshot must exclude every managed alias after logout"
20014 );
20015 assert!(
20016 signed_out.get(personal_id).unwrap().available_now(),
20017 "logout must not disable a still-keyed personal OpenRouter row"
20018 );
20019 assert_eq!(
20020 engine.lane_pin_for(&request, &signed_out),
20021 None,
20022 "a signed-out managed lane default must stop resolving"
20023 );
20024
20025 crate::openrouter::set_test_credential(None);
20026 registry.refresh_availability();
20027 assert!(
20028 !registry.get(personal_id).unwrap().available_now(),
20029 "personal row availability must still turn off with the personal-key seam"
20030 );
20031 }
20032
20033 #[tokio::test(flavor = "current_thread")]
20034 async fn legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates() {
20035 if !crate::run_in_isolated_test_process(
20036 "tests::legacy_parslee_auth_is_routable_only_until_v2_tombstone_then_migrates",
20037 "CAR_LEGACY_V2_MIGRATION_CHILD",
20038 ) {
20039 return;
20040 }
20041 let tmp = TempDir::new().unwrap();
20042 let _provider_env = crate::openrouter::test_environment_scope_async().await;
20043 let _restore = RestoredEnvironment::capture(&[
20044 "CAR_SECRETS_FILE_DIR",
20045 car_home::ENV_VAR,
20046 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20047 ]);
20048 let _home = crate::openrouter::StateRootScope::new();
20063 crate::openrouter::clear_gateway_unconfigured();
20064 crate::parslee_credential::clear_credential_rejected();
20065 unsafe {
20066 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
20067 std::env::set_var(car_home::ENV_VAR, tmp.path().join("car-home"));
20068 }
20069
20070 let secret = |key| car_secrets::SecretRef::with_default_service(key);
20071 let seed_legacy = |store: &car_secrets::SecretStore| {
20072 store
20073 .put(
20074 &secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY),
20075 "legacy-access",
20076 )
20077 .unwrap();
20078 store
20079 .put(
20080 &secret(car_secrets::PARSLEE_ACTIVE_ACCOUNT_ID_KEY),
20081 "legacy-account",
20082 )
20083 .unwrap();
20084 store.put(
20085 &secret(car_secrets::PARSLEE_ACCOUNTS_KEY),
20086 r#"{"active":"legacy-account","accounts":[{"id":"legacy-account","email":"legacy@example.test"}]}"#,
20087 )
20088 .unwrap();
20089 };
20090 let state_ref = secret(car_secrets::PARSLEE_AUTH_STATE_V2_KEY);
20091 let managed = crate::openrouter::curated_schemas()
20092 .into_iter()
20093 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
20094 .unwrap();
20095
20096 unsafe {
20097 std::env::set_var("CAR_SECRETS_FILE_DIR", tmp.path().join("tombstone-secrets"));
20098 }
20099 let tombstoned_store = car_secrets::SecretStore::new();
20100 seed_legacy(&tombstoned_store);
20101 tombstoned_store
20102 .publish(
20103 &state_ref,
20104 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
20105 )
20106 .unwrap();
20107 let mut tombstoned_registry =
20108 UnifiedRegistry::new_empty(tmp.path().join("tombstoned-models"));
20109 tombstoned_registry.register_project_model(managed.clone());
20110 assert!(
20111 !tombstoned_registry
20112 .get(&managed.id)
20113 .unwrap()
20114 .available_now(),
20115 "passive catalog registration must remain disabled without a configured hint"
20116 );
20117 let tombstoned = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
20118 .await
20119 .unwrap();
20120 assert!(tombstoned.is_none(), "the V2 tombstone is authoritative");
20121 tombstoned_registry.refresh_routing_availability(None, true);
20122 assert!(!tombstoned_registry
20123 .get(&managed.id)
20124 .unwrap()
20125 .available_now());
20126
20127 unsafe {
20128 std::env::set_var(
20129 "CAR_SECRETS_FILE_DIR",
20130 tmp.path().join("legacy-only-secrets"),
20131 );
20132 }
20133 let legacy_store = car_secrets::SecretStore::new();
20134 seed_legacy(&legacy_store);
20135 assert!(!legacy_store.status(&state_ref).unwrap().exists);
20136 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("legacy-models"));
20137 registry.register_project_model(managed.clone());
20138 assert!(
20139 !registry.get(&managed.id).unwrap().available_now(),
20140 "passive registration must not inspect attributable legacy slots"
20141 );
20142 registry.refresh_availability();
20143 assert!(
20144 !registry.get(&managed.id).unwrap().available_now(),
20145 "passive refresh must remain secret-store free until request-time migration"
20146 );
20147 let resolved = car_auth::resolve_credential(car_auth::CredentialReadMode::Use)
20148 .await
20149 .unwrap()
20150 .expect("request-time auth reconciliation must migrate attributable legacy state");
20151 registry.refresh_routing_availability(Some(&resolved.api_base), false);
20152 assert!(registry.get(&managed.id).unwrap().available_now());
20153 assert!(
20154 legacy_store.status(&state_ref).unwrap().exists,
20155 "request-time auth reconciliation must publish the migrated V2 record"
20156 );
20157 assert!(
20158 !legacy_store
20159 .status(&secret(car_secrets::PARSLEE_ACCESS_TOKEN_KEY))
20160 .unwrap()
20161 .exists,
20162 "successful V2 migration must clean the legacy access slot"
20163 );
20164 }
20165
20166 #[tokio::test]
20167 async fn reviewed_openrouter_lane_default_tracks_live_credential_availability() {
20168 let _credential_scope = crate::openrouter::test_credential_scope();
20169 crate::openrouter::set_test_credential(Some("static-key"));
20170 let tmp = TempDir::new().unwrap();
20171 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20172 let reviewed_id = "openrouter/deepseek/deepseek-v3.2";
20173 engine.lane_defaults_cache.write().unwrap().set(
20174 None,
20175 crate::intent::UseCase::Assistant,
20176 reviewed_id.into(),
20177 1,
20178 );
20179 let request = GenerateRequest {
20180 prompt: "lane default".into(),
20181 intent: Some(IntentHint {
20182 task: Some(crate::intent::TaskHint::Chat),
20183 ..Default::default()
20184 }),
20185 ..Default::default()
20186 };
20187 assert_eq!(
20188 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
20189 Some(reviewed_id.to_string()),
20190 "a reviewed keyed row must be eligible as a lane default"
20191 );
20192 crate::openrouter::set_test_credential(None);
20193 assert_eq!(
20194 engine.lane_pin_for(&request, &engine.routing_registry_snapshot().await),
20195 None,
20196 "the same static lane default must stop being eligible immediately after key removal"
20197 );
20198 }
20199
20200 #[tokio::test(flavor = "current_thread")]
20201 async fn personal_openrouter_baseline_remains_visible_but_disabled_without_a_credential() {
20202 let _credential_scope = crate::openrouter::test_credential_scope();
20203 crate::openrouter::set_test_credential(None);
20204 let tmp = TempDir::new().unwrap();
20205 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20206
20207 let personal: Vec<_> = engine
20208 .list_schemas()
20209 .into_iter()
20210 .filter(|schema| schema.id.starts_with("openrouter/"))
20211 .collect();
20212 assert_eq!(
20213 personal.len(),
20214 crate::openrouter::curated_model_count(),
20215 "the vetted personal rows stay discoverable as a disabled baseline"
20216 );
20217 assert!(
20218 personal.iter().all(|schema| !schema.available_now()),
20219 "no-key baseline rows must never become routing candidates"
20220 );
20221
20222 let error = engine
20223 .generate_tracked(GenerateRequest {
20224 prompt: "hello".into(),
20225 model: Some("openrouter/openai/gpt-5.4".into()),
20226 params: GenerateParams {
20227 strict_model: true,
20228 ..Default::default()
20229 },
20230 ..Default::default()
20231 })
20232 .await
20233 .expect_err("explicitly selecting a disabled baseline row must be actionable");
20234 let message = error.to_string();
20235 assert!(
20236 message.contains("car keys set openrouter") && message.contains("CarHost"),
20237 "disabled personal row must explain how to connect OpenRouter: {message}"
20238 );
20239 }
20240
20241 #[tokio::test]
20242 async fn reviewed_openrouter_metadata_stays_static_across_credential_changes() {
20243 let _credential_scope = crate::openrouter::test_credential_scope();
20244 crate::openrouter::set_test_credential(Some("static-key"));
20245
20246 let tmp = TempDir::new().unwrap();
20247 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
20248 let live = engine.routing_registry_snapshot().await;
20249 let vetted = live.get("openrouter/openai/gpt-5.4").unwrap();
20250 assert!(vetted.available_now());
20251 assert_eq!(vetted.context_length, 1_050_000);
20252 assert_eq!(vetted.max_output_tokens, Some(128_000));
20253 assert_eq!(vetted.cost.input_per_mtok, Some(2.5));
20254 assert_eq!(vetted.cost.output_per_mtok, Some(15.0));
20255 assert!(
20256 !vetted.cost.pricing_tiers.is_empty(),
20257 "reviewed high-context pricing tiers must remain in the static row"
20258 );
20259 assert_eq!(vetted.trust_tier, TrustTier::Curated);
20260 assert!(vetted.tags.iter().any(|tag| tag == "frontier"));
20261 assert!(vetted.has_capability(ModelCapability::Code));
20262 assert!(vetted
20263 .supported_params
20264 .contains(&schema::GenerateParam::ExtendedThinking));
20265
20266 assert!(live.get("openrouter/vendor/unreviewed-model").is_none());
20267
20268 crate::openrouter::set_test_credential(None);
20269 let reverted = engine.routing_registry_snapshot().await;
20270 assert!(reverted.get("openrouter/vendor/unreviewed-model").is_none());
20271 let baseline = reverted.get("openrouter/openai/gpt-5.4").unwrap();
20272 assert!(
20273 !baseline.available_now(),
20274 "credential removal must disable the reviewed row without removing it"
20275 );
20276 }
20277
20278 #[test]
20297 fn model_pulled_at_runtime_is_ready_without_a_daemon_restart() {
20298 let tmp = TempDir::new().unwrap();
20299 let models_dir = tmp.path().join("models");
20300 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20301 engine.register_model(ModelSchema {
20302 id: "mlx/pulled-later".into(),
20303 name: "PulledLater".into(),
20304 provider: "local".into(),
20305 family: "qwen3".into(),
20306 version: "test".into(),
20307 capabilities: vec![ModelCapability::Generate, ModelCapability::Code],
20308 context_length: 4096,
20309 max_output_tokens: None,
20310 param_count: String::new(),
20311 quantization: None,
20312 performance: schema::PerformanceEnvelope::default(),
20313 cost: schema::CostModel::default(),
20314 source: ModelSource::Mlx {
20315 hf_repo: "example/pulled-later".into(),
20316 hf_weight_file: None,
20317 },
20318 tags: vec![],
20319 supported_params: vec![],
20320 public_benchmarks: vec![],
20321 trust_tier: TrustTier::Curated,
20322 deprecated: false,
20323 available: true,
20324 weights_ready: false,
20325 });
20326
20327 let engine = &engine;
20329 let is_ready = || {
20330 engine
20331 .list_schemas()
20332 .into_iter()
20333 .find(|s| s.id == "mlx/pulled-later")
20334 .expect("registered model must be listed")
20335 .weights_ready
20336 };
20337
20338 assert!(!is_ready(), "precondition: no weights on disk yet");
20339
20340 let dir = models_dir.join("PulledLater");
20343 std::fs::create_dir_all(&dir).unwrap();
20344 std::fs::write(dir.join("config.json"), "{}").unwrap();
20345 std::fs::write(dir.join("model.safetensors"), b"weights").unwrap();
20346
20347 assert!(
20348 is_ready(),
20349 "a model pulled at runtime must be ready without restarting the daemon"
20350 );
20351
20352 std::fs::remove_file(dir.join("model.safetensors")).unwrap();
20355 assert!(
20356 !is_ready(),
20357 "readiness must track the disk in both directions, not latch"
20358 );
20359 }
20360
20361 #[tokio::test(flavor = "current_thread")]
20362 async fn missing_openrouter_key_is_typed_without_changing_its_string_contract() {
20363 let (_credential_scope, _provider_environment) =
20367 crate::openrouter::test_credential_and_environment_scope_async().await;
20368 let home = crate::openrouter::StateRootScope::new();
20369 let _restore = RestoredEnvironment::capture(&["CAR_SECRETS_FILE_DIR"]);
20370 unsafe {
20371 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20372 }
20373
20374 crate::openrouter::set_test_credential(None);
20378
20379 let models_dir = home.path().join("models");
20380 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20381 let schema = crate::openrouter::curated_schemas()
20382 .into_iter()
20383 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
20384 .expect("personal OpenRouter fixture");
20385 let model_id = schema.id.clone();
20386 engine.unified_registry_mut().register_project_model(schema);
20387
20388 let error = engine
20389 .generate_tracked(GenerateRequest {
20390 prompt: "must stop before provider dispatch".into(),
20391 model: Some(model_id.clone()),
20392 params: GenerateParams {
20393 strict_model: true,
20394 ..Default::default()
20395 },
20396 ..Default::default()
20397 })
20398 .await
20399 .expect_err("an OpenRouter route without a key must fail before dispatch");
20400
20401 let tracker = engine.outcome_tracker();
20402 assert!(
20403 tracker.read().await.profile(&model_id).is_none(),
20404 "a missing OpenRouter key must not create model health"
20405 );
20406 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20407 receipts.extend(tracker.write().await.drain_ledger());
20408 let receipts = receipts
20409 .iter()
20410 .filter(|receipt| receipt.model_id == model_id)
20411 .collect::<Vec<_>>();
20412 assert_eq!(receipts.len(), 1);
20413 assert_eq!(receipts[0].success, None);
20414 assert_eq!(receipts[0].quality, None);
20415 assert!(
20416 engine
20417 .adaptive_router
20418 .circuit_breakers
20419 .lock()
20420 .unwrap()
20421 .state(&model_id)
20422 .is_none(),
20423 "a missing OpenRouter key must not create breaker state"
20424 );
20425
20426 let message = "OpenRouter requires a key — run `car keys set openrouter` or connect your OpenRouter account in CarHost";
20427 assert_eq!(
20428 error.to_string(),
20429 format!("inference failed: {message}"),
20430 "guard: the pre-existing Display contract must stay byte-identical"
20431 );
20432 assert!(
20433 !is_auth_failure_message(&error.to_string()),
20434 "guard: an OpenRouter key is not repaired by Parslee sign-in"
20435 );
20436 assert_eq!(
20437 classify_fallback_reason(&error),
20438 classify_fallback_reason(&InferenceError::InferenceFailed(message.into())),
20439 "retyping must preserve prior fallback classification"
20440 );
20441 match error {
20442 InferenceError::ProviderKeyMissing {
20443 provider,
20444 model,
20445 env_vars,
20446 message: actual_message,
20447 } => {
20448 assert_eq!(provider, "openrouter");
20449 assert_eq!(model, model_id);
20450 assert_eq!(env_vars, vec![crate::openrouter::API_KEY_ENV]);
20451 assert_eq!(actual_message, message);
20452 }
20453 other => panic!("expected ProviderKeyMissing, got {other:?}"),
20454 }
20455 }
20456
20457 #[tokio::test(flavor = "current_thread")]
20458 async fn missing_generic_remote_key_is_typed_without_changing_its_string_contract() {
20459 const KEY_ENV: &str = "CAR_TEST_A47_GENERIC_PROVIDER_KEY_DO_NOT_SET";
20460 const EXTRA_KEY_ENV_1: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_1_DO_NOT_SET";
20461 const EXTRA_KEY_ENV_2: &str = "CAR_TEST_A47_GENERIC_EXTRA_KEY_2_DO_NOT_SET";
20462 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
20463 let home = crate::openrouter::StateRootScope::new();
20464 let _restore = RestoredEnvironment::capture(&[
20465 "CAR_SECRETS_FILE_DIR",
20466 KEY_ENV,
20467 EXTRA_KEY_ENV_1,
20468 EXTRA_KEY_ENV_2,
20469 ]);
20470 unsafe {
20471 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20472 std::env::remove_var(KEY_ENV);
20473 std::env::remove_var(EXTRA_KEY_ENV_1);
20474 std::env::remove_var(EXTRA_KEY_ENV_2);
20475 }
20476
20477 let endpoint = "http://127.0.0.1:9/a47-missing-key";
20478 let model_id = "test/a47-generic-missing-key";
20479 let models_dir = home.path().join("models");
20480 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20481 let mut schema = remote_stream_fixture_schema(
20482 model_id,
20483 endpoint.into(),
20484 schema::ApiProtocol::OpenAiCompat,
20485 KEY_ENV,
20486 );
20487 let ModelSource::RemoteApi { api_key_envs, .. } = &mut schema.source else {
20488 panic!("generic remote fixture must use RemoteApi");
20489 };
20490 *api_key_envs = vec![EXTRA_KEY_ENV_1.into(), EXTRA_KEY_ENV_2.into()];
20491 engine.unified_registry_mut().register_project_model(schema);
20492
20493 let error = engine
20494 .generate_tracked(GenerateRequest {
20495 prompt: "must stop before provider dispatch".into(),
20496 model: Some(model_id.into()),
20497 params: GenerateParams {
20498 strict_model: true,
20499 ..Default::default()
20500 },
20501 ..Default::default()
20502 })
20503 .await
20504 .expect_err("a generic remote route without a key must fail before dispatch");
20505
20506 let tracker = engine.outcome_tracker();
20507 assert!(
20508 tracker.read().await.profile(model_id).is_none(),
20509 "a missing generic provider key must not create model health"
20510 );
20511 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20512 receipts.extend(tracker.write().await.drain_ledger());
20513 let receipts = receipts
20514 .iter()
20515 .filter(|receipt| receipt.model_id == model_id)
20516 .collect::<Vec<_>>();
20517 assert_eq!(receipts.len(), 1);
20518 assert_eq!(receipts[0].success, None);
20519 assert_eq!(receipts[0].quality, None);
20520 assert!(
20521 engine
20522 .adaptive_router
20523 .circuit_breakers
20524 .lock()
20525 .unwrap()
20526 .state(model_id)
20527 .is_none(),
20528 "a missing generic provider key must not create breaker state"
20529 );
20530
20531 let message = format!(
20532 "no API keys available for endpoint {endpoint} (checked env vars: [\"{KEY_ENV}\", \"{EXTRA_KEY_ENV_1}\", \"{EXTRA_KEY_ENV_2}\"])"
20533 );
20534 assert_eq!(
20535 error.to_string(),
20536 format!("inference failed: {message}"),
20537 "guard: the pre-existing Display contract must stay byte-identical"
20538 );
20539 assert!(
20540 !is_auth_failure_message(&error.to_string()),
20541 "guard: a generic provider key is not repaired by Parslee sign-in"
20542 );
20543 assert_eq!(
20544 classify_fallback_reason(&error),
20545 classify_fallback_reason(&InferenceError::InferenceFailed(message.clone())),
20546 "retyping must preserve prior fallback classification"
20547 );
20548 match error {
20549 InferenceError::ProviderKeyMissing {
20550 provider,
20551 model,
20552 env_vars,
20553 message: actual_message,
20554 } => {
20555 assert_eq!(provider, "test");
20556 assert_eq!(model, model_id);
20557 assert_eq!(env_vars, vec![KEY_ENV, EXTRA_KEY_ENV_1, EXTRA_KEY_ENV_2]);
20558 assert_eq!(actual_message, message);
20559 }
20560 other => panic!("expected ProviderKeyMissing, got {other:?}"),
20561 }
20562 }
20563
20564 #[test]
20568 fn credential_unavailable_outcomes_do_not_degrade_models_or_open_breakers() {
20569 for reason in [
20570 CredentialFailure::Expired { expires_at: 42 },
20571 CredentialFailure::SignedOut,
20572 CredentialFailure::StoreUnreadable,
20573 CredentialFailure::EnvVarMissing {
20574 env_var: "TEST_PROVIDER_KEY".into(),
20575 },
20576 CredentialFailure::RaceRetryable,
20577 ] {
20578 let model_id = "parslee/test-credential-outcome";
20579 let error = InferenceError::CredentialUnavailable {
20580 provider: "parslee".into(),
20581 model: model_id.into(),
20582 reason: reason.clone(),
20583 detail: "credential resolution stopped before dispatch".into(),
20584 };
20585 let mut tracker = OutcomeTracker::new();
20586 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
20587 record_dispatch_failure(&mut tracker, &trace, &error);
20588
20589 assert!(
20590 tracker.profile(model_id).is_none(),
20591 "{reason:?} changed the model profile: {:?}",
20592 tracker.profile(model_id)
20593 );
20594 let receipts = tracker.drain_ledger();
20595 assert_eq!(receipts.len(), 1, "{reason:?}");
20596 assert_eq!(receipts[0].success, None, "{reason:?}");
20597 assert_eq!(receipts[0].quality, None, "{reason:?}");
20598
20599 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
20600 if error_counts_against_circuit_breaker(&error) {
20601 breakers.record_failure(model_id);
20602 }
20603 assert!(
20604 breakers.state(model_id).is_none(),
20605 "{reason:?} created per-model breaker state"
20606 );
20607 }
20608 }
20609
20610 #[tokio::test(flavor = "current_thread")]
20611 async fn signed_out_managed_attempts_leave_health_and_breaker_untouched() {
20612 if !crate::run_in_isolated_test_process(
20613 "tests::signed_out_managed_attempts_leave_health_and_breaker_untouched",
20614 "CAR_SIGNED_OUT_MODEL_HEALTH_CHILD",
20615 ) {
20616 return;
20617 }
20618
20619 let _provider_environment = crate::openrouter::test_environment_scope_async().await;
20620 let home = crate::openrouter::StateRootScope::new();
20621 let _restore = RestoredEnvironment::capture(&[
20622 "CAR_SECRETS_FILE_DIR",
20623 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
20624 car_auth::PARSLEE_API_BASE_KEY,
20625 ]);
20626 unsafe {
20627 std::env::set_var("CAR_SECRETS_FILE_DIR", home.path().join("secrets"));
20628 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
20629 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
20630 }
20631 crate::openrouter::clear_gateway_unconfigured();
20632 crate::parslee_credential::clear_credential_rejected();
20633
20634 let store = car_secrets::SecretStore::new();
20635 store
20636 .publish(
20637 &car_secrets::SecretRef::with_default_service(
20638 car_secrets::PARSLEE_AUTH_STATE_V2_KEY,
20639 ),
20640 r#"{"schema":2,"revision":1,"generation":1,"accounts":[],"tombstone":true}"#,
20641 )
20642 .unwrap();
20643
20644 let models_dir = home.path().join("models");
20645 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
20646 let schema = crate::openrouter::curated_schemas()
20647 .into_iter()
20648 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
20649 .expect("managed alias fixture");
20650 let model_id = schema.id.clone();
20651 engine.unified_registry_mut().register_project_model(schema);
20652
20653 for _ in 0..3 {
20657 let error = engine
20658 .generate_tracked(GenerateRequest {
20659 prompt: "must stop before provider dispatch".into(),
20660 model: Some(model_id.clone()),
20661 params: GenerateParams {
20662 strict_model: true,
20663 ..Default::default()
20664 },
20665 ..Default::default()
20666 })
20667 .await
20668 .expect_err("signed-out managed inference must be rejected");
20669 assert!(
20670 matches!(
20671 &error,
20672 InferenceError::CredentialUnavailable {
20673 reason: CredentialFailure::SignedOut,
20674 ..
20675 }
20676 ),
20677 "unexpected signed-out error: {error:?}"
20678 );
20679 }
20680
20681 let tracker = engine.outcome_tracker();
20682 assert!(
20683 tracker.read().await.profile(&model_id).is_none(),
20684 "signed-out attempts must not create a model-health profile"
20685 );
20686 let mut receipts = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
20690 receipts.extend(tracker.write().await.drain_ledger());
20691 let signed_out_receipts = receipts
20692 .iter()
20693 .filter(|entry| entry.model_id == model_id)
20694 .collect::<Vec<_>>();
20695 assert_eq!(signed_out_receipts.len(), 3);
20696 assert!(signed_out_receipts
20697 .iter()
20698 .all(|entry| entry.success.is_none() && entry.quality.is_none()));
20699 assert!(
20700 engine
20701 .adaptive_router
20702 .circuit_breakers
20703 .lock()
20704 .unwrap()
20705 .state(&model_id)
20706 .is_none(),
20707 "signed-out attempts must not create per-model breaker state"
20708 );
20709 }
20710
20711 fn admission_refusals() -> Vec<InferenceError> {
20715 use resource_policy::{LocalLoadVerdict, ModelMemoryEstimate, ModelResourceEvidence};
20716 [
20717 LocalLoadVerdict::LiveMemoryUnknown,
20718 LocalLoadVerdict::DisabledByPolicy,
20719 LocalLoadVerdict::ExceedsConfiguredCeiling,
20720 LocalLoadVerdict::InsufficientLiveMemory,
20721 LocalLoadVerdict::ModelMaintenance,
20722 LocalLoadVerdict::PendingTeardown,
20723 ]
20724 .into_iter()
20725 .map(|verdict| {
20726 InferenceError::from(resource_policy::LocalAdmissionError {
20727 preflight: resource_policy::LocalLoadPreflight {
20728 model_id: "vllm-mlx/big:4bit".into(),
20729 estimate: ModelMemoryEstimate {
20730 weights_mb: 16_000,
20731 runtime_overhead_mb: 1_000,
20732 context_overhead_mb: 2_000,
20733 transient_margin_mb: 1_000,
20734 estimated_peak_mb: 20_000,
20735 evidence: ModelResourceEvidence::CatalogExact,
20736 },
20737 configured_ceiling_mb: 26_214,
20738 resident_model_mb: 0,
20739 active_reservations_mb: 0,
20740 estimated_incremental_mb: 20_000,
20741 accelerator_total_mb: None,
20742 accelerator_resident_mb: None,
20743 accelerator_incremental_mb: None,
20744 live_available_mb: Some(4_000),
20745 emergency_reserve_mb: 6_553,
20746 verdict,
20747 },
20748 })
20749 })
20750 .collect()
20751 }
20752
20753 #[test]
20757 fn local_admission_refusal_is_not_a_model_failure() {
20758 let model_id = "vllm-mlx/big:4bit";
20759 for refusal in admission_refusals() {
20760 assert!(matches!(
20761 refusal,
20762 InferenceError::LocalResourceBlocked { .. }
20763 ));
20764 let mut tracker = OutcomeTracker::new();
20765 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
20766 record_dispatch_failure(&mut tracker, &trace, &refusal);
20767 assert!(
20768 tracker.profile(model_id).is_none(),
20769 "admission refusal changed the model profile: {refusal}"
20770 );
20771 let receipt = tracker.drain_ledger().pop().expect("receipt kept");
20772 assert_eq!(receipt.success, None, "{refusal}");
20773 assert_eq!(receipt.quality, None, "{refusal}");
20774 assert!(!error_counts_against_circuit_breaker(&refusal), "{refusal}");
20775 }
20776 }
20777
20778 #[test]
20779 fn content_refusal_closes_its_parenthetical_in_every_shape() {
20780 let render = |kind: Option<&str>, code: Option<&str>| {
20781 InferenceError::ContentRefused {
20782 provider: "p".into(),
20783 kind: kind.map(Into::into),
20784 code: code.map(Into::into),
20785 message: "m".into(),
20786 }
20787 .to_string()
20788 };
20789 assert_eq!(
20790 render(Some("k"), Some("c")),
20791 "p refused this request on content grounds (type=k, code=c): m"
20792 );
20793 assert_eq!(
20794 render(Some("k"), None),
20795 "p refused this request on content grounds (type=k): m"
20796 );
20797 assert_eq!(
20798 render(None, Some("c")),
20799 "p refused this request on content grounds (code=c): m"
20800 );
20801 assert_eq!(
20802 render(None, None),
20803 "p refused this request on content grounds: m"
20804 );
20805 }
20806
20807 fn legacy_unattributed_errors() -> Vec<String> {
20812 let mut legacy: Vec<String> = admission_refusals()
20813 .iter()
20814 .map(ToString::to_string)
20815 .collect();
20816 legacy.push(
20817 InferenceError::ProviderKeyMissing {
20818 provider: "anthropic".into(),
20819 model: "m".into(),
20820 env_vars: vec!["ANTHROPIC_API_KEY".into()],
20821 message: "no API keys available for endpoint https://api.anthropic.com (checked env vars: [\"ANTHROPIC_API_KEY\"])".into(),
20822 }
20823 .to_string(),
20824 );
20825 legacy.push(
20826 InferenceError::InferenceFailed(
20827 "Parslee org lookup failed: HTTP 401 Unauthorized: Authentication required".into(),
20828 )
20829 .to_string(),
20830 );
20831 legacy.push(
20832 InferenceError::InferenceFailed(
20833 "FoundationModels declined the request (guardrailViolation): unsafe".into(),
20834 )
20835 .to_string(),
20836 );
20837 legacy.push(
20838 InferenceError::InferenceFailed(
20839 InferenceError::UnsupportedMode {
20840 mode: "apple-foundation-models",
20841 backend: "remote",
20842 reason: "no",
20843 }
20844 .to_string(),
20845 )
20846 .to_string(),
20847 );
20848 legacy
20849 }
20850
20851 fn write_legacy_ledger(
20855 ledger: &std::path::Path,
20856 real: &str,
20857 ghost: &str,
20858 ) -> Vec<outcome::ModelProfile> {
20859 let mut writer = OutcomeTracker::new();
20860 for error in legacy_unattributed_errors() {
20861 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
20862 writer.record_failure(&t, &error);
20863 }
20864 let t = writer.record_start(real, InferenceTask::Generate, "explicit model");
20865 writer.record_failure(&t, GENUINE_FAILURE);
20866 let t = writer.record_start(ghost, InferenceTask::Generate, "explicit model");
20867 writer.record_failure(
20868 &t,
20869 &InferenceError::InferenceFailed(
20870 InferenceError::ModelNotFound(ghost.into()).to_string(),
20871 )
20872 .to_string(),
20873 );
20874 outcome::append_ledger_entries(ledger, &writer.drain_ledger()).unwrap();
20875 writer.export_profiles()
20876 }
20877
20878 fn reconcile_for_test(
20881 ledger: &std::path::Path,
20882 tracker: &mut OutcomeTracker,
20883 is_registered: &dyn Fn(&str) -> bool,
20884 ) -> std::io::Result<outcome::LedgerReconciliation> {
20885 let plan = outcome::plan_reconcile(
20886 ledger,
20887 tracker,
20888 is_registered,
20889 &std::collections::HashMap::new(),
20890 );
20891 if let Some(entries) = &plan.ledger {
20892 outcome::write_reconciled_ledger(ledger, entries)?;
20893 }
20894 if let Some(profiles) = plan.profiles {
20895 tracker.commit_reconciled(profiles);
20896 }
20897 Ok(plan.report)
20898 }
20899
20900 const GENUINE_FAILURE: &str =
20901 "inference failed: forward: \"[metal::malloc] Attempting to allocate 41846087808 bytes\"";
20902
20903 fn profile_json(tracker: &OutcomeTracker, id: &str) -> serde_json::Value {
20904 let mut v = serde_json::to_value(tracker.profile(id).unwrap()).unwrap();
20905 v.as_object_mut().unwrap().remove("updated_at");
20906 v
20907 }
20908
20909 #[test]
20914 fn reconcile_takes_back_misattributed_failures_and_their_quality_evidence() {
20915 let dir = tempfile::tempdir().unwrap();
20916 let ledger = dir.path().join("outcome_ledger.jsonl");
20917 let (real, ghost) = ("mlx/real:4bit", "nonexistent-model-id-xyz");
20918 let profiles = write_legacy_ledger(&ledger, real, ghost);
20919 let legacy = legacy_unattributed_errors().len();
20920
20921 let mut tracker = OutcomeTracker::new();
20922 tracker.import_profiles(profiles);
20923 let before = tracker.profile(real).unwrap().clone();
20924 assert_eq!(before.fail_count, legacy as u64 + 1);
20925 assert!(
20926 before.quality_observations > 1,
20927 "the legacy rows moved the EMA"
20928 );
20929 assert!(
20930 before.ema_quality < 0.4 - 1e-9,
20931 "buried below one failure's worth"
20932 );
20933
20934 let is_registered = |id: &str| id == real;
20935 let report = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
20936 assert_eq!(report.reclassified, legacy);
20937 assert_eq!(report.unregistered, 1);
20938 assert_eq!(report.profiles_rebuilt, 1);
20939 assert_eq!(report.profiles_removed, 1);
20940
20941 let p = tracker.profile(real).unwrap();
20942 assert_eq!((p.fail_count, p.total_calls, p.success_count), (1, 1, 0));
20943 assert_eq!(p.quality_observations, 1);
20944 assert!(
20945 (p.ema_quality - 0.4).abs() < 1e-12,
20946 "0.5 stepped once toward 0: {}",
20947 p.ema_quality
20948 );
20949 let ts = &p.task_stats["generate"];
20950 assert_eq!((ts.failures, ts.calls, ts.quality_observations), (1, 0, 1));
20952 assert!(tracker.profile(ghost).is_none());
20953
20954 let rows = outcome::read_ledger(&ledger, 0);
20955 assert_eq!(rows.len(), legacy + 2, "nothing is deleted");
20956 for row in &rows {
20957 match row.error.as_deref() {
20958 Some(GENUINE_FAILURE) => {
20959 assert_eq!(row.success, Some(false));
20960 assert_eq!(row.attribution, None);
20961 }
20962 _ if row.model_id == ghost => {
20963 assert_eq!(row.success, None);
20964 assert_eq!(
20965 row.attribution,
20966 Some(outcome::UnattributedReason::Unregistered)
20967 );
20968 }
20969 Some(error) => {
20970 use outcome::UnattributedReason as R;
20971 assert_eq!(row.success, None);
20972 assert_eq!(row.quality, None);
20973 let expected = if error.contains("memory")
20975 || error.contains("allocation")
20976 || error.contains("maintained")
20977 || error.contains("previous local model")
20978 {
20979 R::ResourceBlocked
20980 } else if error.contains("API keys") || error.contains("org lookup") {
20981 R::Account
20982 } else if error.contains("declined") {
20983 R::ContentRefused
20984 } else {
20985 R::Capability
20986 };
20987 assert_eq!(row.attribution, Some(expected), "{error}");
20988 }
20989 None => panic!("every legacy row carries its error"),
20990 }
20991 }
20992 #[cfg(unix)]
20993 {
20994 use std::os::unix::fs::PermissionsExt;
20995 let mode = std::fs::metadata(&ledger).unwrap().permissions().mode() & 0o777;
20996 assert_eq!(mode, 0o600, "a rewritten ledger stays owner-only");
20997 }
20998
20999 let rebuilt = profile_json(&tracker, real);
21001 let again = reconcile_for_test(&ledger, &mut tracker, &is_registered).unwrap();
21002 assert_eq!(
21003 again.reclassified + again.unregistered + again.profiles_removed,
21004 0
21005 );
21006 assert_eq!(profile_json(&tracker, real), rebuilt);
21007 }
21008
21009 #[test]
21015 fn reconcile_repairs_a_ledger_the_first_migration_already_rewrote() {
21016 let dir = tempfile::tempdir().unwrap();
21017 let ledger = dir.path().join("outcome_ledger.jsonl");
21018 let real = "mlx/real:4bit";
21019 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
21020
21021 let mut clean = OutcomeTracker::new();
21023 clean.import_profiles(profiles.clone());
21024 let clean_ledger = dir.path().join("clean.jsonl");
21025 std::fs::copy(&ledger, &clean_ledger).unwrap();
21026 reconcile_for_test(&clean_ledger, &mut clean, &|id| id == real).unwrap();
21027 let expected = profile_json(&clean, real);
21028
21029 let rows: Vec<_> = outcome::read_ledger(&ledger, 0)
21032 .into_iter()
21033 .map(|mut r| {
21034 if r.model_id == real && r.error.as_deref() != Some(GENUINE_FAILURE) {
21035 r.success = None;
21036 r.quality = None;
21037 }
21038 r
21039 })
21040 .collect();
21041 std::fs::remove_file(&ledger).unwrap();
21042 outcome::append_ledger_entries(&ledger, &rows).unwrap();
21043 let mut v2 = profiles;
21044 for p in &mut v2 {
21045 if p.model_id == real {
21046 p.fail_count = 1;
21047 p.total_calls = 1;
21048 p.task_stats.get_mut("generate").unwrap().failures = 1;
21049 }
21050 }
21051 let mut tracker = OutcomeTracker::new();
21052 tracker.import_profiles(v2);
21053 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21054 assert_eq!(profile_json(&tracker, real), expected);
21055 }
21056
21057 #[test]
21062 fn reconcile_clamps_a_broken_profile_and_keeps_ids_that_ran() {
21063 let dir = tempfile::tempdir().unwrap();
21064 let ledger = dir.path().join("outcome_ledger.jsonl");
21065 let (model, renamed) = ("parslee/reasoning", "openai/gpt-5.4");
21066 let mut writer = OutcomeTracker::new();
21067 for id in [model, renamed] {
21068 for _ in 0..4 {
21069 let t = writer.record_start(id, InferenceTask::Generate, "r");
21070 writer.record_failure(
21071 &t,
21072 "inference failed: Parslee inference returned no content",
21073 );
21074 }
21075 }
21076 let mut rows = writer.drain_ledger();
21077 let mut ok = rows[0].clone();
21078 ok.model_id = renamed.into();
21079 ok.success = Some(true);
21080 ok.error = None;
21081 rows.push(ok);
21082 outcome::append_ledger_entries(&ledger, &rows).unwrap();
21083
21084 let mut profiles = writer.export_profiles();
21085 for p in &mut profiles {
21086 if p.model_id == model {
21087 p.success_count = 1522;
21088 p.total_calls = 1519;
21089 p.fail_count = 0;
21090 p.task_stats.get_mut("generate").unwrap().failures = 0;
21091 }
21092 }
21093 let mut tracker = OutcomeTracker::new();
21094 tracker.import_profiles(profiles);
21095 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == model).unwrap();
21096 assert_eq!(report.unregistered, 0, "an id that ran is not unregistered");
21097 let p = tracker.profile(model).unwrap();
21098 assert_eq!(
21099 (p.success_count, p.fail_count, p.total_calls),
21100 (1522, 0, 1522)
21101 );
21102 assert_eq!(tracker.profile(renamed).unwrap().fail_count, 4);
21103 assert!(outcome::read_ledger(&ledger, 0)
21104 .iter()
21105 .filter(|r| r.model_id == renamed && r.success != Some(true))
21106 .all(|r| r.success == Some(false)));
21107 }
21108
21109 #[test]
21115 fn reconcile_keeps_successes_the_ledger_never_saw() {
21116 let dir = tempfile::tempdir().unwrap();
21117 let ledger = dir.path().join("outcome_ledger.jsonl");
21118 let model = "mlx/qwen3-30b-a3b:4bit";
21119 let mut writer = OutcomeTracker::new();
21120 for _ in 0..20 {
21121 let t = writer.record_start(model, InferenceTask::Generate, "r");
21122 writer.record_complete(&t, 100, 10, 20);
21123 }
21124 for refusal in admission_refusals().iter().take(3) {
21125 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
21126 writer.record_failure(&t, &refusal.to_string());
21127 }
21128 let written = writer.drain_ledger();
21130 assert!(written.iter().all(|r| r.success == Some(false)));
21131 outcome::append_ledger_entries(&ledger, &written).unwrap();
21132 let mut tracker = OutcomeTracker::new();
21133 tracker.import_profiles(writer.export_profiles());
21134 let before = tracker.profile(model).unwrap().clone();
21135 assert_eq!((before.success_count, before.fail_count), (20, 3));
21136
21137 reconcile_for_test(&ledger, &mut tracker, &|_| true).unwrap();
21138 let p = tracker.profile(model).unwrap();
21139 assert_eq!((p.success_count, p.fail_count), (20, 0));
21140 assert_eq!(p.total_calls, before.total_calls - 3);
21141 assert_eq!(p.quality_observations, 0);
21142 assert_eq!(p.ema_quality, 0.5);
21143 }
21144
21145 #[test]
21149 fn reconcile_replays_a_seeded_profile_from_its_benchmark_prior() {
21150 let dir = tempfile::tempdir().unwrap();
21151 let ledger = dir.path().join("outcome_ledger.jsonl");
21152 let model = "mlx/qwen3-4b:4bit";
21153 let mut seeded = outcome::ModelProfile::new(model.into());
21154 seeded.ema_quality = 0.7;
21155 seeded.prior_sample_size = 3;
21156 let mut writer = OutcomeTracker::new();
21157 writer.import_profiles(vec![seeded]);
21158 for refusal in admission_refusals().iter().take(2) {
21159 let t = writer.record_start(model, InferenceTask::Generate, "explicit model");
21160 writer.record_failure(&t, &refusal.to_string());
21161 }
21162 outcome::append_ledger_entries(&ledger, &writer.drain_ledger()).unwrap();
21163 let mut tracker = OutcomeTracker::new();
21164 tracker.import_profiles(writer.export_profiles());
21165 let before = tracker.profile(model).unwrap().clone();
21166 assert_eq!(before.quality_observations, 2);
21167 assert!(before.ema_quality < 0.7 * 0.8 * 0.8 + 1e-9);
21168
21169 let seeds = std::collections::HashMap::from([(
21170 model.to_string(),
21171 outcome::QualitySeed {
21172 overall: Some(0.7),
21173 tasks: std::collections::HashMap::new(),
21174 },
21175 )]);
21176 let plan = outcome::plan_reconcile(&ledger, &tracker, &|_| true, &seeds);
21177 assert_eq!(plan.report.profiles_rebuilt, 1);
21178 tracker.commit_reconciled(plan.profiles.expect("a corrected profile set"));
21179 let p = tracker.profile(model).unwrap();
21180 assert_eq!(p.quality_observations, 0);
21181 assert!(
21182 (p.ema_quality - 0.7).abs() < 1e-12,
21183 "back to the seed: {}",
21184 p.ema_quality
21185 );
21186 assert_eq!(p.prior_sample_size, 3, "the prior's weight is the prior's");
21187 }
21188
21189 #[test]
21193 fn reconcile_is_quiet_in_steady_state() {
21194 let dir = tempfile::tempdir().unwrap();
21195 let ledger = dir.path().join("outcome_ledger.jsonl");
21196 let real = "mlx/real:4bit";
21197 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
21198 let mut tracker = OutcomeTracker::new();
21199 tracker.import_profiles(profiles);
21200 reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21201
21202 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21204 tracker.record_complete(&t, 120, 10, 20);
21205 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21206 tracker.record_complete(&t, 80, 10, 0);
21207 let t = tracker.record_start(real, InferenceTask::Generate, "r");
21208 tracker.record_failure(&t, GENUINE_FAILURE);
21209 outcome::append_ledger_entries(&ledger, &tracker.drain_ledger()).unwrap();
21210 let settled = profile_json(&tracker, real);
21211
21212 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21213 assert_eq!(report, outcome::LedgerReconciliation::default());
21214 assert_eq!(profile_json(&tracker, real), settled);
21215
21216 outcome::prune_ledger(&ledger, 2).unwrap();
21219 let report = reconcile_for_test(&ledger, &mut tracker, &|id| id == real).unwrap();
21220 assert_eq!(report, outcome::LedgerReconciliation::default());
21221 assert_eq!(profile_json(&tracker, real), settled);
21222 }
21223
21224 #[test]
21228 fn neutral_profile_is_not_backfilled_on_load() {
21229 let dir = tempfile::tempdir().unwrap();
21230 let path = dir.path().join("profiles.json");
21231 let mut writer = OutcomeTracker::new();
21232 let t = writer.record_start("m", InferenceTask::Generate, "r");
21233 writer.record_failure(&t, "daemon read timeout on infer after 30s");
21234 let p = writer.profile("m").unwrap();
21235 assert_eq!((p.fail_count, p.quality_observations), (1, 0));
21236 assert_eq!(p.ema_quality, 0.5);
21237 writer.save_to_file(&path).unwrap();
21238 let mut reader = OutcomeTracker::new();
21239 reader.load_from_file(&path).unwrap();
21240 assert_eq!(reader.profile("m").unwrap().quality_observations, 0);
21241 }
21242
21243 #[tokio::test]
21247 async fn reconcile_refuses_when_profiles_could_not_be_read() {
21248 let root = tempfile::tempdir().unwrap();
21249 let state = root.path().join("state");
21250 let models = state.join("models");
21251 std::fs::create_dir_all(&models).unwrap();
21252 let ledger = models.join("outcome_ledger.jsonl");
21253 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
21254 std::fs::write(models.join("outcome_profiles.json"), b"{ not json").unwrap();
21255 let before = std::fs::read(&ledger).unwrap();
21256 let engine = InferenceEngine::new(InferenceConfig {
21257 state_root: state.clone(),
21258 models_dir: root.path().join("weights"),
21259 ..InferenceConfig::default()
21260 });
21261 let report = engine.reconcile_outcome_ledger().await.unwrap();
21262 assert_eq!(report, outcome::LedgerReconciliation::default());
21263 assert_eq!(std::fs::read(&ledger).unwrap(), before, "ledger untouched");
21264 assert_eq!(
21265 std::fs::read(models.join("outcome_profiles.json")).unwrap(),
21266 b"{ not json",
21267 "profiles untouched"
21268 );
21269 }
21270
21271 #[cfg(unix)]
21274 #[tokio::test]
21275 async fn retire_plan_includes_the_projection_and_engine_refusals() {
21276 let _serial = RETIRE_LOCKS.lock().await;
21277 let root = tempfile::tempdir().unwrap();
21278 let root_path = root.path().canonicalize().unwrap();
21279 let hub = root_path.join("hub");
21280 let models = root_path.join("weights");
21281 let engine = InferenceEngine::new(InferenceConfig {
21282 state_root: root_path.join("state"),
21283 models_dir: models.clone(),
21284 ..InferenceConfig::default()
21285 });
21286 let schema = engine
21287 .unified_registry()
21288 .all()
21289 .find(|m| m.id == plain_mlx_id(&engine))
21290 .unwrap()
21291 .clone();
21292 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
21293 unreachable!()
21294 };
21295 let repo = hf_cache::repo_dir_in(&hub, hf_repo);
21296 std::fs::create_dir_all(repo.join("snapshots/s")).unwrap();
21297 std::fs::create_dir_all(repo.join("blobs")).unwrap();
21298 std::fs::write(repo.join("blobs/w"), vec![0u8; 64]).unwrap();
21299 std::os::unix::fs::symlink(repo.join("blobs/w"), repo.join("snapshots/s/w")).unwrap();
21300 std::fs::create_dir_all(&models).unwrap();
21301 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
21302
21303 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21304 assert!(!plan.is_refused(), "{plan:?}");
21305 assert_eq!(
21306 plan.freed_bytes, 64,
21307 "the link frees nothing; the repo frees its blob"
21308 );
21309 assert_eq!(
21310 plan.projections,
21311 vec![retire::Projection {
21312 path: models.join(&schema.name),
21313 receipt_for: None,
21314 }]
21315 );
21316 assert!(matches!(
21317 &plan.deletions[..],
21318 [retire::HubDeletion::Repo { .. }]
21319 ));
21320
21321 let guard = crate::download::acquire_model_lock(&schema.id).await;
21322 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21323 assert!(plan.refusals.contains(&retire::RetireRefusal::Downloading));
21324 drop(guard);
21325
21326 engine.lane_defaults_cache.write().unwrap().set(
21327 None,
21328 crate::intent::UseCase::Coding,
21329 schema.id.clone(),
21330 1,
21331 );
21332 let plan = engine.retire_plan_in(&schema.id, &hub).unwrap();
21333 assert!(plan
21334 .refusals
21335 .iter()
21336 .any(|r| matches!(r, retire::RetireRefusal::LaneDefault { .. })));
21337 }
21338
21339 #[cfg(unix)]
21342 #[tokio::test]
21343 async fn retire_plan_contains_projections_and_sees_sibling_downloads() {
21344 let _serial = RETIRE_LOCKS.lock().await;
21345 let root = tempfile::tempdir().unwrap();
21346 let root_path = root.path().canonicalize().unwrap();
21347 let hub = root_path.join("hub");
21348 std::fs::create_dir_all(&hub).unwrap();
21349 let mut engine = InferenceEngine::new(InferenceConfig {
21350 state_root: root_path.join("state"),
21351 models_dir: root_path.join("weights"),
21352 ..InferenceConfig::default()
21353 });
21354 let mut escaping = engine
21355 .unified_registry()
21356 .get("mlx/qwen3-4b:4bit")
21357 .unwrap()
21358 .clone();
21359 escaping.id = "user/escape".into();
21360 escaping.name = "../../outside".into();
21361 std::fs::create_dir_all(root_path.join("outside")).unwrap();
21362 engine
21363 .unified_registry_mut()
21364 .register_project_model(escaping);
21365 let plan = engine.retire_plan_in("user/escape", &hub).unwrap();
21366 assert!(plan.projections.is_empty(), "{plan:?}");
21367 assert!(plan
21368 .refusals
21369 .iter()
21370 .any(|r| matches!(r, retire::RetireRefusal::UnsafePath { .. })));
21371
21372 let guard = crate::download::acquire_model_lock("vllm-mlx/gemma-4-12b-it").await;
21374 let plan = engine
21375 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
21376 .unwrap();
21377 assert!(
21378 plan.refusals.contains(&retire::RetireRefusal::Downloading),
21379 "{plan:?}"
21380 );
21381 assert_eq!(
21383 plan.also_retires,
21384 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
21385 );
21386 drop(guard);
21387 let plan = engine
21388 .retire_plan_in("mlx/gemma-4-12b-it:4bit", &hub)
21389 .unwrap();
21390 assert!(!plan.refusals.contains(&retire::RetireRefusal::Downloading));
21391 }
21392
21393 #[cfg(unix)]
21396 #[test]
21397 fn discovered_rows_are_retired_alone_and_their_files_are_kept() {
21398 let root = tempfile::tempdir().unwrap();
21399 let root_path = root.path().canonicalize().unwrap();
21400 let hub = root_path.join("hub");
21401 let models = root_path.join("weights");
21402 std::fs::create_dir_all(&hub).unwrap();
21403 let mut engine = InferenceEngine::new(InferenceConfig {
21404 state_root: root_path.join("state"),
21405 models_dir: models.clone(),
21406 ..InferenceConfig::default()
21407 });
21408 let base = engine
21409 .unified_registry()
21410 .get("mlx/qwen3-4b:4bit")
21411 .unwrap()
21412 .clone();
21413 for (id, name) in [("disk/one", "One-MLX"), ("disk/two", "Two-MLX")] {
21414 let mut row = base.clone();
21415 row.id = id.into();
21416 row.name = name.into();
21417 row.source = ModelSource::Mlx {
21418 hf_repo: String::new(),
21419 hf_weight_file: None,
21420 };
21421 engine.unified_registry_mut().register_project_model(row);
21422 std::fs::create_dir_all(models.join(name)).unwrap();
21423 std::fs::write(models.join(name).join("model.safetensors"), b"user").unwrap();
21424 }
21425 let plan = engine.retire_plan_in("disk/one", &hub).unwrap();
21426 assert!(plan.also_retires.is_empty(), "{plan:?}");
21427 assert!(plan.projections.is_empty(), "{plan:?}");
21428 assert_eq!(plan.user_content, vec![models.join("One-MLX")]);
21429 assert_eq!(plan.freed_bytes, 0);
21430 }
21431
21432 static RETIRE_LOCKS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
21436
21437 #[cfg(unix)]
21441 fn plain_mlx_id(engine: &InferenceEngine) -> String {
21442 let protected = engine.protected_models(&engine.unified_registry(), true);
21443 [
21444 "mlx/qwen3-1.7b:3bit",
21445 "mlx/qwen3-0.6b:6bit",
21446 "mlx/qwen3-30b-a3b:4bit",
21447 ]
21448 .into_iter()
21449 .find(|id| !protected.contains_key(*id))
21450 .unwrap()
21451 .to_string()
21452 }
21453
21454 #[cfg(unix)]
21456 fn hub_repo(hub: &Path, repo: &str, files: &[(&str, usize)]) -> PathBuf {
21457 let dir = hf_cache::repo_dir_in(hub, repo);
21458 std::fs::create_dir_all(dir.join("snapshots/s")).unwrap();
21459 std::fs::create_dir_all(dir.join("blobs")).unwrap();
21460 for (i, (path, size)) in files.iter().enumerate() {
21461 let blob = dir.join("blobs").join(format!("b{i}"));
21462 std::fs::write(&blob, vec![0u8; *size]).unwrap();
21463 let entry = dir.join("snapshots/s").join(path);
21464 std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
21465 std::os::unix::fs::symlink(&blob, &entry).unwrap();
21466 }
21467 dir
21468 }
21469
21470 fn isolated_engine(root: &Path) -> (InferenceEngine, PathBuf, PathBuf) {
21471 let hub = root.join("hub");
21472 let models = root.join("weights");
21473 std::fs::create_dir_all(&hub).unwrap();
21474 std::fs::create_dir_all(&models).unwrap();
21475 let engine = InferenceEngine::new(InferenceConfig {
21476 state_root: root.join("state"),
21477 models_dir: models.clone(),
21478 ..InferenceConfig::default()
21479 });
21480 (engine, hub, models)
21481 }
21482
21483 #[tokio::test]
21489 async fn a_swapped_signed_catalog_is_one_generation_for_every_reader() {
21490 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21491 let root = tempfile::tempdir().unwrap();
21492 let (mut engine, _hub, _models) = isolated_engine(root.path());
21493 let builtin = crate::registry::builtin_catalog()
21494 .into_iter()
21495 .find(|m| !m.is_local())
21496 .expect("a remote builtin row");
21497 let mut added = builtin.clone();
21498 added.id = "signed/swapped-in".into();
21499 added.name = "swapped-in".into();
21500 added.tags.retain(|t| t != "builtin");
21501 let mut redefined = builtin.clone();
21502 redefined.context_length = builtin.context_length + 1;
21503 let mut kept = added.clone();
21505 kept.id = "project/registered-in-memory".into();
21506 kept.name = "registered-in-memory".into();
21507 assert!(engine
21508 .unified_registry_mut()
21509 .register_project_model(kept.clone()));
21510
21511 let before = engine.unified_registry();
21512 let (first, _key) = crate::catalog::signed_test_catalog(
21513 crate::catalog::CatalogDoc {
21514 revoked: Vec::new(),
21515 version: 2,
21516 models: vec![added.clone(), redefined],
21517 },
21518 7,
21519 );
21520 assert_eq!(engine.swap_signed_catalog(first), 2);
21522 assert!(engine.list_schemas().iter().any(|m| m.id == added.id));
21523 assert!(engine.unified_registry().get(&added.id).is_some());
21524 assert_eq!(
21525 engine.model_schema("swapped-in").map(|m| m.id),
21526 Some(added.id.clone())
21527 );
21528 assert_eq!(
21529 engine
21530 .unified_registry()
21531 .get(&builtin.id)
21532 .unwrap()
21533 .context_length,
21534 builtin.context_length,
21535 "a signed row never redefines a compiled builtin's identity"
21536 );
21537 assert!(
21538 before.get(&added.id).is_none(),
21539 "an older generation is unchanged"
21540 );
21541 assert!(engine.unified_registry().get(&kept.id).is_some());
21542
21543 let (second, _key) = crate::catalog::signed_test_catalog(
21544 crate::catalog::CatalogDoc {
21545 revoked: Vec::new(),
21546 version: 3,
21547 models: vec![],
21548 },
21549 7,
21550 );
21551 assert_eq!(engine.swap_signed_catalog(second), 0);
21552 assert!(
21553 engine.unified_registry().get(&added.id).is_none(),
21554 "withdrawn"
21555 );
21556 assert!(engine.model_schema("swapped-in").is_none());
21557 assert!(engine.unified_registry().get(&builtin.id).is_some());
21558 assert!(engine.unified_registry().get(&kept.id).is_some());
21559 }
21560
21561 #[tokio::test]
21564 async fn a_withdrawn_row_a_lane_defaults_to_is_kept_deprecated() {
21565 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21566 let root = tempfile::tempdir().unwrap();
21567 let (engine, _hub, _models) = isolated_engine(root.path());
21568 let mut row = crate::registry::builtin_catalog()
21569 .into_iter()
21570 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
21571 .unwrap();
21572 row.id = "signed/lane-default".into();
21573 row.name = "lane-default".into();
21574 let catalog = |models, version| {
21575 crate::catalog::signed_test_catalog(
21576 crate::catalog::CatalogDoc {
21577 revoked: Vec::new(),
21578 version,
21579 models,
21580 },
21581 13,
21582 )
21583 .0
21584 };
21585 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
21586 engine
21587 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
21588 .unwrap();
21589 engine.swap_signed_catalog(catalog(vec![], 3));
21590 let kept = engine
21591 .unified_registry()
21592 .get(&row.id)
21593 .cloned()
21594 .expect("kept");
21595 assert!(kept.deprecated);
21596 }
21597
21598 #[tokio::test]
21601 async fn a_row_withdrawn_mid_pull_is_kept_and_recorded() {
21602 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21603 let root = tempfile::tempdir().unwrap();
21604 let (engine, _hub, _models) = isolated_engine(root.path());
21605 let mut row = crate::registry::builtin_catalog()
21606 .into_iter()
21607 .find(|m| !m.is_local())
21608 .unwrap();
21609 row.id = "signed/mid-pull".into();
21610 row.name = "mid-pull".into();
21611 let catalog = |models, version| {
21612 crate::catalog::signed_test_catalog(
21613 crate::catalog::CatalogDoc {
21614 revoked: Vec::new(),
21615 version,
21616 models,
21617 },
21618 17,
21619 )
21620 .0
21621 };
21622 engine.swap_signed_catalog(catalog(vec![row.clone()], 2));
21623 let retained = catalog::retained_path(&root.path().join("state"));
21624 engine
21625 .with_pinned_registry(async {
21626 engine.swap_signed_catalog(catalog(vec![], 3));
21627 assert!(engine.current_registry().get(&row.id).is_none());
21628 assert!(!retained.exists(), "nothing was kept yet");
21629 engine.keep_pulled_row_if_withdrawn("mid-pull");
21630 })
21631 .await;
21632 let kept = engine
21633 .unified_registry()
21634 .get(&row.id)
21635 .cloned()
21636 .expect("kept");
21637 assert!(kept.deprecated);
21638 let recorded = std::fs::read_to_string(&retained).unwrap();
21639 assert!(recorded.contains("signed/mid-pull"), "{recorded}");
21640 }
21641
21642 #[tokio::test]
21646 async fn the_scheduled_refresh_loads_only_a_newer_verified_catalog() {
21647 use std::io::{Read, Write};
21648 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21649 let root = tempfile::tempdir().unwrap();
21650 let (engine, _hub, _models) = isolated_engine(root.path());
21651 let mut row = crate::registry::builtin_catalog()
21652 .into_iter()
21653 .find(|m| !m.is_local())
21654 .unwrap();
21655 row.id = "signed/served".into();
21656 row.name = "served".into();
21657 let (published, key) = crate::catalog::signed_test_catalog(
21658 crate::catalog::CatalogDoc {
21659 revoked: Vec::new(),
21660 version: 5,
21661 models: vec![row],
21662 },
21663 31,
21664 );
21665 let serving = std::sync::Arc::new(std::sync::Mutex::new(published.envelope()));
21666 let served = serving.clone();
21667 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
21668 let url = format!("http://{}/catalog.json", listener.local_addr().unwrap());
21669 std::thread::spawn(move || {
21670 for stream in listener.incoming() {
21671 let Ok(mut stream) = stream else { return };
21672 let mut request = [0u8; 2048];
21673 let n = stream.read(&mut request).unwrap_or(0);
21674 let request = String::from_utf8_lossy(&request[..n]);
21675 let envelope = served.lock().unwrap().clone();
21676 let body = if request.starts_with("GET /catalog.json.sig ") {
21677 envelope.signature.clone()
21678 } else {
21679 envelope.signed_body.clone()
21680 };
21681 let _ = write!(
21682 stream,
21683 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
21684 body.len(),
21685 body
21686 );
21687 }
21688 });
21689 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(Some(1)));
21690 assert!(engine.unified_registry().get("signed/served").is_some());
21691 assert_eq!(
21692 engine.refresh_catalog_from(&url, &key).await,
21693 Ok(None),
21694 "nothing newer is not an error"
21695 );
21696 let status_path = catalog::status_path(&engine.config.state_root);
21698 let status = catalog::load_status(&status_path);
21699 assert_eq!(status.source.as_deref(), Some(url.as_str()));
21700 assert_eq!(status.verified_version, Some(5));
21701 assert!(status.last_verified_at.is_some() && status.failing_since.is_none());
21702 let (_, other_key) = crate::catalog::signed_test_catalog(
21703 crate::catalog::CatalogDoc {
21704 revoked: Vec::new(),
21705 version: 1,
21706 models: vec![],
21707 },
21708 32,
21709 );
21710 assert!(engine.refresh_catalog_from(&url, &other_key).await.is_err());
21711 let status = catalog::load_status(&status_path);
21714 assert!(status.failing_since.is_some(), "{status:?}");
21715 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Rejected));
21716 assert!(matches!(
21717 engine.catalog_freshness(),
21718 catalog::CatalogFreshness::Current { .. }
21719 ));
21720
21721 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
21724 assert!(catalog::load_status(&status_path).failing_since.is_none());
21725 let (older, _) = crate::catalog::signed_test_catalog(
21726 crate::catalog::CatalogDoc {
21727 revoked: Vec::new(),
21728 version: 4,
21729 models: vec![],
21730 },
21731 31,
21732 );
21733 *serving.lock().unwrap() = older.envelope();
21734 assert_eq!(engine.refresh_catalog_from(&url, &key).await, Ok(None));
21735 let status = catalog::load_status(&status_path);
21736 assert_eq!(status.last_failure, Some(catalog::CheckFailure::Older));
21737 assert!(
21738 status
21739 .last_error
21740 .as_deref()
21741 .is_some_and(|e| e.contains("older than the cached v5")),
21742 "{status:?}"
21743 );
21744
21745 let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
21747 let gone = format!("http://{}/catalog.json", closed.local_addr().unwrap());
21748 drop(closed);
21749 assert!(engine.refresh_catalog_from(&gone, &key).await.is_err());
21750 assert_eq!(
21751 catalog::load_status(&status_path).last_failure,
21752 Some(catalog::CheckFailure::Unreachable)
21753 );
21754 }
21755
21756 #[cfg(unix)]
21760 #[tokio::test]
21761 async fn a_revoked_installed_model_is_unlinked_and_not_rediscovered() {
21762 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21763 let root = tempfile::tempdir().unwrap();
21764 let root_path = root.path().canonicalize().unwrap();
21765 let (engine, _hub, models) = isolated_engine(&root_path);
21766 let mut row = crate::registry::builtin_catalog()
21767 .into_iter()
21768 .find(|m| matches!(m.source, ModelSource::Local { .. }))
21769 .expect("a downloadable local row");
21770 row.id = "signed/revoked-local".into();
21771 row.name = "signed-revoked-local".into();
21772 row.tags.retain(|t| t != "builtin");
21773 let state = root_path.join("state");
21774 let path = catalog::cache_path(&state);
21775 let doc = |models, revoked, version| catalog::CatalogDoc {
21776 version,
21777 models,
21778 revoked,
21779 };
21780 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 61);
21781 engine
21782 .adopt_verified_catalog(&path, &v2, &key)
21783 .await
21784 .unwrap();
21785
21786 let shared = root_path.join("hub-snapshot");
21788 std::fs::create_dir_all(&shared).unwrap();
21789 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
21790 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
21791 std::os::unix::fs::symlink(&shared, models.join(&row.name)).unwrap();
21792 engine.adopt_model_into_car(&row.id).await.unwrap();
21793
21794 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 61);
21795 engine
21796 .adopt_verified_catalog(&path, &v3, &key)
21797 .await
21798 .unwrap();
21799 assert!(engine.unified_registry().get(&row.id).is_none());
21800 assert!(
21801 std::fs::symlink_metadata(models.join(&row.name)).is_err(),
21802 "CAR's linkage to revoked weights is gone"
21803 );
21804 assert!(
21805 shared.join("model.gguf").exists(),
21806 "the bytes are left to the orphan rules"
21807 );
21808 let restarted =
21809 crate::registry::UnifiedRegistry::new_with_state_root(state, models.clone());
21810 assert!(
21811 restarted.list().iter().all(|m| m.name != row.name),
21812 "nothing on disk brings it back"
21813 );
21814 }
21815
21816 #[cfg(unix)]
21822 #[tokio::test]
21823 async fn revoked_models_without_receipts_never_come_back_and_keep_their_bytes() {
21824 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21825 let root = tempfile::tempdir().unwrap();
21826 let root_path = root.path().canonicalize().unwrap();
21827 let (engine, _hub, models) = isolated_engine(&root_path);
21828 let template = crate::registry::builtin_catalog()
21829 .into_iter()
21830 .find(|m| matches!(m.source, ModelSource::Local { .. }))
21831 .expect("a downloadable local row");
21832 let signed = |id: &str, name: &str| {
21833 let mut m = template.clone();
21834 m.id = id.into();
21835 m.name = name.into();
21836 m.tags.retain(|t| t != "builtin");
21837 m
21838 };
21839 let lazy = signed("signed/lazy", "signed-lazy");
21840 let copied = signed("signed/copied", "signed-copied");
21841 let state = root_path.join("state");
21842 let path = catalog::cache_path(&state);
21843 let doc = |models, revoked, version| catalog::CatalogDoc {
21844 version,
21845 models,
21846 revoked,
21847 };
21848 let (v2, key) =
21849 catalog::signed_test_catalog(doc(vec![lazy.clone(), copied.clone()], vec![], 2), 62);
21850 engine
21851 .adopt_verified_catalog(&path, &v2, &key)
21852 .await
21853 .unwrap();
21854
21855 let shared = root_path.join("hub-snapshot");
21856 std::fs::create_dir_all(&shared).unwrap();
21857 std::fs::write(shared.join("model.gguf"), b"weights").unwrap();
21858 std::fs::write(shared.join("tokenizer.json"), b"{}").unwrap();
21859 std::os::unix::fs::symlink(&shared, models.join(&lazy.name)).unwrap();
21860 let copy = models.join(&copied.name);
21861 std::fs::create_dir_all(©).unwrap();
21862 std::fs::write(copy.join("model.gguf"), b"copied weights").unwrap();
21863 std::fs::write(copy.join("tokenizer.json"), b"{}").unwrap();
21864
21865 let (v3, _) = catalog::signed_test_catalog(
21866 doc(vec![], vec![lazy.id.clone(), copied.id.clone()], 3),
21867 62,
21868 );
21869 engine
21870 .adopt_verified_catalog(&path, &v3, &key)
21871 .await
21872 .unwrap();
21873 assert!(
21874 std::fs::symlink_metadata(models.join(&lazy.name)).is_err(),
21875 "links unlinked"
21876 );
21877 assert!(
21878 copy.join("model.gguf").exists(),
21879 "a copy's bytes are never deleted"
21880 );
21881 let restarted =
21882 crate::registry::UnifiedRegistry::new_with_state_root(state.clone(), models.clone());
21883 assert!(
21884 restarted
21885 .list()
21886 .iter()
21887 .all(|m| m.name != lazy.name && m.name != copied.name),
21888 "neither is rediscovered"
21889 );
21890
21891 let want = vec![retire::RevokedCopy {
21892 model_id: copied.id.clone(),
21893 path: copy.clone(),
21894 bytes: (b"copied weights".len() + b"{}".len()) as u64,
21895 }];
21896 assert_eq!(retire::revoked_copies(&state, &models), want);
21897 let hub = root_path.join("hub");
21898 assert_eq!(
21899 engine
21900 .portfolio_in(&hub, &HardwareInfo::detect())
21901 .await
21902 .revoked_copies,
21903 want
21904 );
21905 let opts = doctor::DoctorOptions {
21906 deep: false,
21907 repair: false,
21908 };
21909 let report = doctor::diagnose_at_isolated(&state, &models, &hub, &opts);
21910 assert_eq!(report.revoked_copies, want);
21911 let without = doctor::DoctorReport {
21912 revoked_copies: Vec::new(),
21913 ..report.clone()
21914 };
21915 assert_eq!(
21916 report.is_healthy(),
21917 without.is_healthy(),
21918 "informational only"
21919 );
21920
21921 let outside = root_path.join("outside");
21924 std::fs::create_dir_all(&outside).unwrap();
21925 std::fs::write(outside.join("private"), b"user data").unwrap();
21926 let names_path = catalog::revoked_names_path(&state);
21927 let mut names = catalog::load_revoked_names(&names_path);
21928 names.insert("../outside".into(), copied.id.clone());
21929 catalog::save_revoked_names(&names_path, &names).unwrap();
21930 assert_eq!(retire::revoked_copies(&state, &models), want);
21931
21932 std::fs::remove_dir_all(©).unwrap();
21934 std::fs::create_dir_all(©).unwrap();
21935 std::os::unix::fs::symlink(shared.join("model.gguf"), copy.join("model.gguf")).unwrap();
21936 assert!(retire::revoked_copies(&state, &models).is_empty());
21937 }
21938
21939 #[tokio::test]
21944 async fn agents_pinning_a_revoked_model_are_reported_by_id_or_name() {
21945 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
21946 let root = tempfile::tempdir().unwrap();
21947 let root_path = root.path().canonicalize().unwrap();
21948 let (engine, _hub, _models) = isolated_engine(&root_path);
21949 let mut row = crate::registry::builtin_catalog()
21950 .into_iter()
21951 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
21952 .unwrap();
21953 row.id = "signed/revoked-pinned".into();
21954 row.name = "revoked-pinned".into();
21955 let registry = engine.current_registry();
21956 let builtin = crate::registry::builtin_catalog()
21957 .into_iter()
21958 .find(|m| {
21959 !m.is_local()
21960 && registry.find_by_name(&m.name).map(|f| f.id.as_str()) == Some(m.id.as_str())
21961 })
21962 .unwrap();
21963 let fine = crate::registry::builtin_catalog()
21964 .into_iter()
21965 .find(|m| m.id != builtin.id && !m.is_local())
21966 .unwrap();
21967 let path = catalog::cache_path(&root_path.join("state"));
21968 let doc = |models, revoked, version| catalog::CatalogDoc {
21969 version,
21970 models,
21971 revoked,
21972 };
21973 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 65);
21974 engine
21975 .adopt_verified_catalog(&path, &v2, &key)
21976 .await
21977 .unwrap();
21978 let pins = std::collections::BTreeMap::from([
21979 (row.id.clone(), vec!["by-id".to_string()]),
21980 (
21981 row.name.clone(),
21982 vec!["monthly".to_string(), "digest".to_string()],
21983 ),
21984 (builtin.name.clone(), vec!["weekly".to_string()]),
21985 (fine.id.clone(), vec!["untouched".to_string()]),
21986 ]);
21987 engine.set_external_pins(pins.clone());
21988 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
21989
21990 let (v3, _) = catalog::signed_test_catalog(
21991 doc(vec![], vec![row.id.clone(), builtin.id.clone()], 3),
21992 65,
21993 );
21994 engine
21995 .adopt_verified_catalog(&path, &v3, &key)
21996 .await
21997 .unwrap();
21998 let expect = |agents: &[&str], pin: &str, model_id: &str| portfolio::RevokedPin {
21999 pin: pin.to_string(),
22000 model_id: model_id.to_string(),
22001 agents: agents.iter().map(|a| a.to_string()).collect(),
22002 };
22003 let mut want = vec![
22004 expect(&["by-id"], &row.id, &row.id),
22005 expect(&["digest", "monthly"], &row.name, &row.id),
22006 expect(&["weekly"], &builtin.name, &builtin.id),
22007 ];
22008 want.sort_by(|a, b| a.pin.cmp(&b.pin));
22009 assert_eq!(engine.revoked_pins(&engine.current_registry()), want);
22010 let state = root_path.join("state");
22013 assert!(
22014 catalog::load_revoked_names(&catalog::revoked_models_path(&state))
22015 .contains_key(&row.name)
22016 );
22017 assert!(
22018 !catalog::load_revoked_names(&catalog::revoked_names_path(&state))
22019 .contains_key(&row.name)
22020 );
22021 let hub = root_path.join("hub");
22023 let hw = HardwareInfo::detect();
22024 assert_eq!(engine.portfolio_in(&hub, &hw).await.revoked_pins, want);
22025 assert_eq!(engine.concierge_status(false).await.revoked_pins, want);
22026
22027 drop(engine);
22030 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
22031 let (restarted, _hub, _models) = isolated_engine(&root_path);
22032 std::env::remove_var("CAR_CATALOG_PUBKEY");
22033 restarted.set_external_pins(pins);
22034 assert_eq!(restarted.revoked_pins(&restarted.current_registry()), want);
22035 }
22036
22037 #[tokio::test]
22041 async fn a_pin_by_a_name_a_live_row_holds_is_not_reported() {
22042 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22043 let root = tempfile::tempdir().unwrap();
22044 let root_path = root.path().canonicalize().unwrap();
22045 let (engine, _hub, _models) = isolated_engine(&root_path);
22046 let base = crate::registry::builtin_catalog()
22047 .into_iter()
22048 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22049 .unwrap();
22050 let named = |id: &str| {
22051 let mut row = base.clone();
22052 row.id = id.into();
22053 row.name = "shared-name".into();
22054 row
22055 };
22056 let (old, fresh) = (named("signed/old"), named("signed/fresh"));
22057 let path = catalog::cache_path(&root_path.join("state"));
22058 let doc = |models, revoked, version| catalog::CatalogDoc {
22059 version,
22060 models,
22061 revoked,
22062 };
22063 let (v2, key) = catalog::signed_test_catalog(doc(vec![old.clone()], vec![], 2), 66);
22064 engine
22065 .adopt_verified_catalog(&path, &v2, &key)
22066 .await
22067 .unwrap();
22068 engine.set_external_pins(std::collections::BTreeMap::from([(
22069 "shared-name".to_string(),
22070 vec!["monthly".to_string()],
22071 )]));
22072
22073 let (v3, _) =
22074 catalog::signed_test_catalog(doc(vec![fresh.clone()], vec![old.id.clone()], 3), 66);
22075 engine
22076 .adopt_verified_catalog(&path, &v3, &key)
22077 .await
22078 .unwrap();
22079 assert!(
22080 engine.revoked_pins(&engine.current_registry()).is_empty(),
22081 "the name belongs to a live row"
22082 );
22083
22084 let (v4, _) = catalog::signed_test_catalog(doc(vec![], vec![old.id.clone()], 4), 66);
22087 engine
22088 .adopt_verified_catalog(&path, &v4, &key)
22089 .await
22090 .unwrap();
22091 assert!(engine.revoked_pins(&engine.current_registry()).is_empty());
22092 }
22093
22094 #[tokio::test]
22097 async fn a_revoked_lane_default_is_cleared_and_recorded() {
22098 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22099 let root = tempfile::tempdir().unwrap();
22100 let root_path = root.path().canonicalize().unwrap();
22101 let (engine, _hub, _models) = isolated_engine(&root_path);
22102 let mut row = crate::registry::builtin_catalog()
22103 .into_iter()
22104 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22105 .unwrap();
22106 row.id = "signed/revoked-default".into();
22107 row.name = "revoked-default".into();
22108 let other = crate::registry::builtin_catalog()
22109 .into_iter()
22110 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Code))
22111 .unwrap();
22112 let path = catalog::cache_path(&root_path.join("state"));
22113 let doc = |models, revoked, version| catalog::CatalogDoc {
22114 version,
22115 models,
22116 revoked,
22117 };
22118 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 63);
22119 engine
22120 .adopt_verified_catalog(&path, &v2, &key)
22121 .await
22122 .unwrap();
22123 engine
22124 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22125 .unwrap();
22126 engine
22127 .set_lane_default(None, crate::intent::UseCase::Coding, &other.id)
22128 .unwrap();
22129
22130 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 63);
22131 engine
22132 .adopt_verified_catalog(&path, &v3, &key)
22133 .await
22134 .unwrap();
22135 assert_eq!(
22136 engine.lane_default(None, crate::intent::UseCase::Assistant),
22137 None
22138 );
22139 assert_eq!(
22140 engine.lane_default(None, crate::intent::UseCase::Coding),
22141 Some(other.id.clone()),
22142 "a lane on another model is untouched"
22143 );
22144 let cleared = engine
22145 .concierge_actions(0)
22146 .into_iter()
22147 .find(|a| a.kind == action_ledger::ConciergeActionKind::RevokedLaneCleared)
22148 .expect("the clearing is recorded");
22149 assert_eq!(cleared.model_id, row.id);
22150 assert!(cleared.detail.contains("revoked"), "{}", cleared.detail);
22151 let now = std::time::SystemTime::now()
22153 .duration_since(std::time::UNIX_EPOCH)
22154 .unwrap()
22155 .as_secs();
22156 let shown = engine
22157 .recent_concierge_actions(now)
22158 .into_iter()
22159 .find(|a| a.kind == "lane_cleared")
22160 .expect("the clearing is in recent actions");
22161 assert_eq!(shown.subject, row.id);
22162 assert_eq!(shown.undo, None);
22163 let refused = engine
22165 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22166 .unwrap_err();
22167 assert!(refused.contains("revoked"), "{refused}");
22168 }
22169
22170 #[tokio::test]
22173 async fn a_revocation_cached_before_a_restart_still_clears_its_lane() {
22174 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22175 let root = tempfile::tempdir().unwrap();
22176 let root_path = root.path().canonicalize().unwrap();
22177 let mut row = crate::registry::builtin_catalog()
22178 .into_iter()
22179 .find(|m| !m.is_local() && m.has_capability(ModelCapability::Generate))
22180 .unwrap();
22181 row.id = "signed/revoked-while-down".into();
22182 row.name = "revoked-while-down".into();
22183 let state = root_path.join("state");
22184 let path = catalog::cache_path(&state);
22185 let doc = |models, revoked, version| catalog::CatalogDoc {
22186 version,
22187 models,
22188 revoked,
22189 };
22190 let (v2, key) = catalog::signed_test_catalog(doc(vec![row.clone()], vec![], 2), 64);
22191 {
22192 let (engine, _hub, _models) = isolated_engine(&root_path);
22193 engine
22194 .adopt_verified_catalog(&path, &v2, &key)
22195 .await
22196 .unwrap();
22197 engine
22198 .set_lane_default(None, crate::intent::UseCase::Assistant, &row.id)
22199 .unwrap();
22200 }
22201 let (v3, _) = catalog::signed_test_catalog(doc(vec![], vec![row.id.clone()], 3), 64);
22203 catalog::save_verified(&path, &v3).unwrap();
22204 std::env::set_var("CAR_CATALOG_PUBKEY", &key);
22205 let (engine, _hub, _models) = isolated_engine(&root_path);
22206 std::env::remove_var("CAR_CATALOG_PUBKEY");
22207 assert_eq!(
22208 engine.lane_default(None, crate::intent::UseCase::Assistant),
22209 Some(row.id.clone())
22210 );
22211 engine.enforce_catalog_revocations().await;
22212 assert_eq!(
22213 engine.lane_default(None, crate::intent::UseCase::Assistant),
22214 None
22215 );
22216 }
22217
22218 #[tokio::test]
22221 async fn a_request_reads_one_registry_generation_across_a_swap() {
22222 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22223 let root = tempfile::tempdir().unwrap();
22224 let (engine, _hub, _models) = isolated_engine(root.path());
22225 let mut added = crate::registry::builtin_catalog()
22226 .into_iter()
22227 .find(|m| !m.is_local())
22228 .unwrap();
22229 added.id = "signed/mid-request".into();
22230 added.name = "mid-request".into();
22231 let (catalog, _key) = crate::catalog::signed_test_catalog(
22232 crate::catalog::CatalogDoc {
22233 revoked: Vec::new(),
22234 version: 2,
22235 models: vec![added],
22236 },
22237 9,
22238 );
22239 engine
22240 .with_pinned_registry(async {
22241 assert!(engine.model_schema("mid-request").is_none());
22242 engine.swap_signed_catalog(catalog);
22243 assert!(
22244 engine.model_schema("mid-request").is_none(),
22245 "the request's own generation is unchanged"
22246 );
22247 assert!(engine
22248 .current_registry()
22249 .get("signed/mid-request")
22250 .is_some());
22251 })
22252 .await;
22253 assert!(engine.model_schema("mid-request").is_some());
22254 }
22255
22256 #[tokio::test]
22259 async fn a_refreshed_catalog_loads_live_and_a_stale_one_changes_nothing() {
22260 let _scopes = crate::openrouter::test_credential_and_environment_scope_async().await;
22261 let root = tempfile::tempdir().unwrap();
22262 let (engine, _hub, _models) = isolated_engine(root.path());
22263 let template = crate::registry::builtin_catalog()
22264 .into_iter()
22265 .find(|m| !m.is_local())
22266 .unwrap();
22267 let row = |id: &str| {
22268 let mut m = template.clone();
22269 m.id = id.into();
22270 m.name = id.replace('/', "-");
22271 m
22272 };
22273 let path = crate::catalog::cache_path(&root.path().join("state"));
22274 let (v3, key) = crate::catalog::signed_test_catalog(
22275 crate::catalog::CatalogDoc {
22276 revoked: Vec::new(),
22277 version: 3,
22278 models: vec![row("signed/v3")],
22279 },
22280 11,
22281 );
22282 let (v2, _) = crate::catalog::signed_test_catalog(
22283 crate::catalog::CatalogDoc {
22284 revoked: Vec::new(),
22285 version: 2,
22286 models: vec![row("signed/v2")],
22287 },
22288 11,
22289 );
22290 assert_eq!(engine.adopt_verified_catalog(&path, &v3, &key).await, Ok(1));
22291 assert!(engine.unified_registry().get("signed/v3").is_some());
22292 assert!(engine
22293 .adopt_verified_catalog(&path, &v2, &key)
22294 .await
22295 .is_err());
22296 assert!(engine.unified_registry().get("signed/v3").is_some());
22297 assert!(engine.unified_registry().get("signed/v2").is_none());
22298 }
22299
22300 #[cfg(unix)]
22304 #[tokio::test]
22305 async fn retiring_a_whole_repo_model_frees_it_and_leaves_nothing_behind() {
22306 let _serial = RETIRE_LOCKS.lock().await;
22307 let root = tempfile::tempdir().unwrap();
22308 let root_path = root.path().canonicalize().unwrap();
22309 let (engine, hub, models) = isolated_engine(&root_path);
22310 let id = plain_mlx_id(&engine);
22311 let id = id.as_str();
22312 let schema = engine.unified_registry().get(id).unwrap().clone();
22313 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22314 unreachable!()
22315 };
22316 let repo = hub_repo(
22317 &hub,
22318 hf_repo,
22319 &[("config.json", 10), ("model.safetensors", 500)],
22320 );
22321 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22322
22323 let dry = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22324 assert!(!dry.executed);
22325 assert!(repo.exists(), "a dry run deletes nothing");
22326
22327 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22328 assert!(outcome.executed, "{outcome:?}");
22329 assert!(outcome.errors.is_empty(), "{outcome:?}");
22330 assert!(!repo.exists());
22331 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
22332 let leftovers: Vec<_> = std::fs::read_dir(&hub)
22333 .unwrap()
22334 .filter_map(Result::ok)
22335 .collect();
22336 assert!(leftovers.is_empty(), "no quarantine left: {leftovers:?}");
22337 let journals = engine
22338 .model_management_store()
22339 .management_state_dir()
22340 .join("hub-removals");
22341 assert!(std::fs::read_dir(&journals).map_or(true, |mut d| d.next().is_none()));
22342 assert!(engine
22343 .concierge_actions(10)
22344 .iter()
22345 .any(|a| a.kind == action_ledger::ConciergeActionKind::Retire && a.model_id == id));
22346
22347 let again = engine.retire_plan_in(id, &hub).unwrap();
22348 assert!(
22349 again.deletions.is_empty() && again.projections.is_empty(),
22350 "{again:?}"
22351 );
22352 }
22353
22354 #[cfg(unix)]
22357 #[tokio::test]
22358 async fn retiring_a_gguf_model_leaves_the_users_checkout_alone() {
22359 let _serial = RETIRE_LOCKS.lock().await;
22360 let root = tempfile::tempdir().unwrap();
22361 let root_path = root.path().canonicalize().unwrap();
22362 let (engine, hub, _models) = isolated_engine(&root_path);
22363 let schema = engine
22364 .unified_registry()
22365 .all()
22366 .find(|m| m.id == "qwen/qwen3-8b:q4_k_m")
22367 .unwrap()
22368 .clone();
22369 let ModelSource::Local {
22370 hf_repo,
22371 hf_filename,
22372 tokenizer_repo,
22373 } = &schema.source
22374 else {
22375 unreachable!()
22376 };
22377 let quant = hub_repo(&hub, hf_repo, &[(hf_filename.as_str(), 300)]);
22378 let base = hub_repo(
22379 &hub,
22380 tokenizer_repo,
22381 &[("tokenizer.json", 7), ("model.safetensors", 900)],
22382 );
22383
22384 let outcome = engine
22385 .retire_model_in(&schema.id, &hub, false, None)
22386 .await
22387 .unwrap();
22388 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22389 assert!(std::fs::symlink_metadata(quant.join("snapshots/s").join(hf_filename)).is_err());
22390 assert!(!quant.join("blobs/b0").exists());
22391 assert!(base.join("snapshots/s/tokenizer.json").exists());
22392 assert!(base.join("snapshots/s/model.safetensors").exists());
22393 assert_eq!(std::fs::metadata(base.join("blobs/b1")).unwrap().len(), 900);
22394 }
22395
22396 #[cfg(unix)]
22400 #[tokio::test]
22401 async fn retirement_refuses_a_changed_plan_and_a_running_download() {
22402 let _serial = RETIRE_LOCKS.lock().await;
22403 let root = tempfile::tempdir().unwrap();
22404 let root_path = root.path().canonicalize().unwrap();
22405 let (engine, hub, _models) = isolated_engine(&root_path);
22406 let id = plain_mlx_id(&engine);
22407 let id = id.as_str();
22408 let ModelSource::Mlx { hf_repo, .. } =
22409 engine.unified_registry().get(id).unwrap().source.clone()
22410 else {
22411 unreachable!()
22412 };
22413 let repo = hub_repo(&hub, &hf_repo, &[("w", 10)]);
22414 let reviewed = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22415 assert!(!reviewed.plan.digest.is_empty());
22416
22417 std::fs::write(repo.join("blobs/late"), vec![0u8; 99]).unwrap();
22418 let changed = engine
22419 .retire_model_in(id, &hub, false, Some(&reviewed.plan.digest))
22420 .await
22421 .unwrap();
22422 assert!(!changed.executed);
22423 assert!(changed
22424 .plan
22425 .refusals
22426 .contains(&retire::RetireRefusal::StateChanged));
22427 assert!(repo.join("blobs/late").exists() && repo.join("blobs/b0").exists());
22428
22429 let busy = crate::download::try_acquire_model_lock(id).unwrap();
22430 let blocked = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22431 assert!(!blocked.executed);
22432 assert!(blocked
22433 .plan
22434 .refusals
22435 .contains(&retire::RetireRefusal::Downloading));
22436 assert!(repo.exists());
22437 drop(busy);
22438
22439 let current = engine.retire_model_in(id, &hub, true, None).await.unwrap();
22440 let done = engine
22441 .retire_model_in(id, &hub, false, Some(¤t.plan.digest))
22442 .await
22443 .unwrap();
22444 assert!(done.executed && done.errors.is_empty(), "{done:?}");
22445 assert!(!repo.exists());
22446 }
22447
22448 #[cfg(unix)]
22451 #[tokio::test]
22452 async fn a_shared_auxiliary_does_not_make_twins() {
22453 let _serial = RETIRE_LOCKS.lock().await;
22454 let root = tempfile::tempdir().unwrap();
22455 let root_path = root.path().canonicalize().unwrap();
22456 let (mut engine, hub, _models) = isolated_engine(&root_path);
22457 let ltx = engine
22458 .unified_registry()
22459 .get("mlx/ltx-2.3:q4")
22460 .unwrap()
22461 .clone();
22462 let ModelSource::Mlx {
22463 hf_repo: ltx_repo, ..
22464 } = ltx.source.clone()
22465 else {
22466 unreachable!()
22467 };
22468 let mut second = ltx.clone();
22469 second.id = "user/ltx-q8".into();
22470 second.name = "ltx-q8".into();
22471 second.source = ModelSource::Mlx {
22472 hf_repo: "someone/ltx-q8".into(),
22473 hf_weight_file: None,
22474 };
22475 engine.unified_registry_mut().register_project_model(second);
22476 let ltx_dir = hub_repo(&hub, <x_repo, &[("config.json", 5), ("w", 50)]);
22477 hub_repo(&hub, "someone/ltx-q8", &[("config.json", 5), ("w", 70)]);
22478 hub_repo(
22479 &hub,
22480 crate::registry::LTX_TEXT_ENCODER,
22481 &[("config.json", 5), ("w", 90)],
22482 );
22483 let plan = engine.retire_plan_in("mlx/ltx-2.3:q4", &hub).unwrap();
22484 assert!(plan.also_retires.is_empty(), "{plan:?}");
22485 assert!(
22486 plan.deletions.iter().all(|d| !matches!(
22487 d,
22488 retire::HubDeletion::Repo { repo, .. } if repo == "someone/ltx-q8"
22489 )),
22490 "{plan:?}"
22491 );
22492 assert!(ltx_dir.exists());
22493 }
22494
22495 #[cfg(unix)]
22497 #[tokio::test]
22498 async fn twins_sharing_a_repo_retire_together() {
22499 let _serial = RETIRE_LOCKS.lock().await;
22500 let root = tempfile::tempdir().unwrap();
22501 let root_path = root.path().canonicalize().unwrap();
22502 let (engine, hub, _models) = isolated_engine(&root_path);
22503 let id = "mlx/gemma-4-12b-it:4bit";
22504 let ModelSource::Mlx { hf_repo, .. } =
22505 engine.unified_registry().get(id).unwrap().source.clone()
22506 else {
22507 unreachable!()
22508 };
22509 let repo = hub_repo(&hub, &hf_repo, &[("config.json", 5), ("w", 50)]);
22510 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22511 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22512 assert_eq!(
22513 outcome.plan.also_retires,
22514 vec!["vllm-mlx/gemma-4-12b-it".to_string()]
22515 );
22516 assert!(!repo.exists());
22517 let retired: Vec<String> = engine
22518 .concierge_actions(10)
22519 .into_iter()
22520 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
22521 .map(|a| a.model_id)
22522 .collect();
22523 assert!(retired.contains(&id.to_string()), "{retired:?}");
22524 assert!(
22525 retired.contains(&"vllm-mlx/gemma-4-12b-it".to_string()),
22526 "{retired:?}"
22527 );
22528
22529 let again = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22531 assert!(!again.executed);
22532 assert!(again
22533 .plan
22534 .refusals
22535 .contains(&retire::RetireRefusal::NothingToRetire));
22536 assert_eq!(
22537 engine
22538 .concierge_actions(10)
22539 .iter()
22540 .filter(|a| a.kind == action_ledger::ConciergeActionKind::Retire)
22541 .count(),
22542 2
22543 );
22544 }
22545
22546 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22551 #[tokio::test]
22552 async fn a_receipted_projection_retires_and_the_row_stays_fetchable() {
22553 let _serial = RETIRE_LOCKS.lock().await;
22554 let root = tempfile::tempdir().unwrap();
22555 let root_path = root.path().canonicalize().unwrap();
22556 let (engine, hub, models) = isolated_engine(&root_path);
22557 let id = plain_mlx_id(&engine);
22558 let id = id.as_str();
22559 let schema = engine.unified_registry().get(id).unwrap().clone();
22560 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22561 unreachable!()
22562 };
22563 let repo = hub_repo(
22564 &hub,
22565 hf_repo,
22566 &[("config.json", 10), ("model.safetensors", 500)],
22567 );
22568 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name)).unwrap();
22569 engine.adopt_model_into_car(id).await.unwrap();
22570 let store = engine.model_management_store();
22571 assert!(store.receipt_path(id).exists());
22572
22573 let plan = engine.retire_plan_in(id, &hub).unwrap();
22574 assert!(
22575 plan.projections
22576 .iter()
22577 .any(|p| p.receipt_for.as_deref() == Some(id)),
22578 "{plan:?}"
22579 );
22580 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
22581 assert!(outcome.executed && outcome.errors.is_empty(), "{outcome:?}");
22582 assert!(!repo.exists());
22583 assert!(std::fs::symlink_metadata(models.join(&schema.name)).is_err());
22584 assert!(!store.receipt_path(id).exists());
22585 assert!(!store.tombstone_path(id).exists(), "retired, not disabled");
22586 assert!(store.car_enabled(id).unwrap());
22587 let intents = store.management_state_dir().join("retiring");
22588 assert!(std::fs::read_dir(&intents).map_or(true, |mut d| d.next().is_none()));
22589 }
22590
22591 #[cfg(unix)]
22593 #[test]
22594 fn a_projection_holding_a_real_file_is_not_removed() {
22595 let root = tempfile::tempdir().unwrap();
22596 let models = root.path().canonicalize().unwrap().join("weights");
22597 let projection = models.join("Some-MLX");
22598 std::fs::create_dir_all(&projection).unwrap();
22599 std::os::unix::fs::symlink("/nowhere", projection.join("link")).unwrap();
22600 std::fs::write(projection.join("notes.txt"), b"mine").unwrap();
22601 assert!(retire::remove_links_only_tree(&projection, &models).is_err());
22602 assert!(std::fs::symlink_metadata(projection.join("link")).is_ok());
22603 assert!(projection.join("notes.txt").exists());
22604 let elsewhere = root.path().canonicalize().unwrap().join("elsewhere");
22606 std::fs::create_dir_all(&elsewhere).unwrap();
22607 assert!(retire::remove_links_only_tree(&elsewhere, &models).is_err());
22608 assert!(elsewhere.exists());
22609 }
22610
22611 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22616 #[tokio::test]
22617 async fn portfolio_keeps_defaults_and_waits_for_evidence() {
22618 let _serial = RETIRE_LOCKS.lock().await;
22619 let root = tempfile::tempdir().unwrap();
22620 let root_path = root.path().canonicalize().unwrap();
22621 let (engine, hub, models) = isolated_engine(&root_path);
22622 let hw = HardwareInfo {
22623 total_ram_mb: 64 * 1024,
22624 ..HardwareInfo::detect()
22625 };
22626 let protected = engine.protected_models(&engine.unified_registry(), true);
22629 let idle_id = [
22630 "mlx/qwen3-30b-a3b:4bit",
22631 "mlx/qwen3-1.7b:3bit",
22632 "mlx/qwen3-0.6b:6bit",
22633 ]
22634 .into_iter()
22635 .find(|id| !protected.contains_key(*id))
22636 .unwrap();
22637 let mut installed = Vec::new();
22638 for id in [idle_id, "mlx/qwen3-8b:4bit"] {
22639 let schema = engine.unified_registry().get(id).unwrap().clone();
22640 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
22641 unreachable!()
22642 };
22643 let repo = hub_repo(
22644 &hub,
22645 hf_repo,
22646 &[("config.json", 10), ("model.safetensors", 500)],
22647 );
22648 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
22649 .unwrap();
22650 installed.push(id);
22651 }
22652 engine.lane_defaults_cache.write().unwrap().set(
22653 None,
22654 crate::intent::UseCase::Assistant,
22655 "mlx/qwen3-8b:4bit".to_string(),
22656 1,
22657 );
22658
22659 let fresh = engine.portfolio_in(&hub, &hw).await;
22660 let ids: Vec<&str> = fresh.models.iter().map(|m| m.model_id.as_str()).collect();
22661 for id in &installed {
22662 assert!(ids.contains(id), "{ids:?}");
22663 }
22664 let verdict = |p: &portfolio::Portfolio, id: &str| {
22665 p.models.iter().find(|m| m.model_id == id).unwrap().clone()
22666 };
22667 assert_eq!(
22668 verdict(&fresh, "mlx/qwen3-8b:4bit").verdict,
22669 portfolio::Verdict::Protected
22670 );
22671 let small = verdict(&fresh, idle_id);
22672 assert_eq!(small.verdict, portfolio::Verdict::Unknown, "{small:?}");
22673 assert_eq!(small.action, portfolio::PortfolioAction::Keep);
22674
22675 engine
22677 .model_management_store()
22678 .set_usage_tracking_since_for_test(1);
22679 let aged = engine.portfolio_in(&hub, &hw).await;
22680 let small = verdict(&aged, idle_id);
22681 assert_eq!(small.verdict, portfolio::Verdict::Idle, "{small:?}");
22682 assert_eq!(small.action, portfolio::PortfolioAction::Retire);
22683 assert!(aged.reclaimable_bytes >= 510, "{aged:?}");
22684 let digest = small.retire.unwrap().digest;
22685 let done = engine
22686 .retire_model_in(idle_id, &hub, false, Some(&digest))
22687 .await
22688 .unwrap();
22689 assert!(done.executed && done.errors.is_empty(), "{done:?}");
22690 assert_eq!(
22691 verdict(&aged, "mlx/qwen3-8b:4bit").verdict,
22692 portfolio::Verdict::Protected
22693 );
22694
22695 let pinned = plain_mlx_id(&engine);
22698 engine.set_external_pins(std::collections::BTreeMap::from([(
22699 pinned.clone(),
22700 vec!["monthly".to_string()],
22701 )]));
22702 let plan = engine.retire_plan_in(&pinned, &hub).unwrap();
22703 assert!(
22704 plan.refusals.iter().any(|r| matches!(
22705 r,
22706 retire::RetireRefusal::Protected { why } if why.contains("monthly")
22707 )),
22708 "{plan:?}"
22709 );
22710 engine.set_external_pins(Default::default());
22711
22712 let tool = engine
22714 .retire_plan_in(LOCAL_TOOL_MODEL_PULL_ID, &hub)
22715 .unwrap();
22716 assert!(
22717 tool.refusals
22718 .iter()
22719 .any(|r| matches!(r, retire::RetireRefusal::Protected { .. })),
22720 "{tool:?}"
22721 );
22722 }
22723
22724 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
22731 #[tokio::test]
22732 async fn maintenance_retires_under_pressure_and_respects_keep_and_churn() {
22733 use crate::update_prefs::UpdatePolicy;
22734 let _serial = RETIRE_LOCKS.lock().await;
22735 let root = tempfile::tempdir().unwrap();
22736 let root_path = root.path().canonicalize().unwrap();
22737 let (engine, hub, _models) = isolated_engine(&root_path);
22738 let hw = HardwareInfo {
22739 total_ram_mb: 64 * 1024,
22740 ..HardwareInfo::detect()
22741 };
22742 let id = plain_mlx_id(&engine);
22743 let ModelSource::Mlx { hf_repo, .. } =
22744 engine.unified_registry().get(&id).unwrap().source.clone()
22745 else {
22746 unreachable!()
22747 };
22748 let layout = || hub_repo(&hub, &hf_repo, &[("config.json", 10), ("w", 500)]);
22749 let repo = layout();
22750 engine
22751 .model_management_store()
22752 .set_usage_tracking_since_for_test(1);
22753 let gb = 1_000_000_000;
22754 let roomy = Some(maintenance::DiskSpace {
22755 free_bytes: 500 * gb,
22756 total_bytes: 1_000 * gb,
22757 });
22758 let tight = Some(maintenance::DiskSpace {
22759 free_bytes: 50 * gb,
22760 total_bytes: 1_000 * gb,
22761 });
22762 let none = crate::nudge::NudgeState::default();
22763
22764 let calm = engine
22765 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22766 .await;
22767 assert!(calm.retired.is_none(), "{calm:?}");
22768 assert!(calm.plan.held.iter().any(|h| h.model_id == id), "{calm:?}");
22769 assert!(repo.exists());
22770
22771 let notify = engine
22772 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, tight)
22773 .await;
22774 assert!(notify.retired.is_none());
22775 assert!(
22776 notify.plan.suggest.iter().any(|d| d.model_id == id),
22777 "{notify:?}"
22778 );
22779 assert!(repo.exists(), "Notify never acts");
22780
22781 let off = engine
22782 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Off, &none, tight)
22783 .await;
22784 assert_eq!(off.plan, maintenance::MaintenancePlan::default());
22785
22786 let mut kept = crate::nudge::NudgeState::default();
22787 kept.dismiss_labeled(
22788 maintenance::keep_key(&id),
22789 crate::concierge::DismissReason::Wrong,
22790 1,
22791 );
22792 let held = engine
22793 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &kept, tight)
22794 .await;
22795 assert!(held.retired.is_none());
22796 assert!(repo.exists(), "kept");
22797
22798 let pressed = engine
22799 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22800 .await;
22801 let retired = pressed.retired.as_ref().expect("retired under pressure");
22802 assert_eq!(retired.model_id, id);
22803 assert!(!repo.exists());
22804 let record = engine.portfolio_in(&hub, &hw).await.recent_actions;
22806 let entry = record
22807 .iter()
22808 .find(|a| a.kind == "retire" && a.subject == id)
22809 .expect("the retirement is on record");
22810 assert_eq!(
22811 entry.undo.as_deref(),
22812 Some(format!("car models pull {id}").as_str())
22813 );
22814 assert!(!entry.interrupted);
22815 let kinds: Vec<_> = engine
22816 .concierge_actions(10)
22817 .into_iter()
22818 .filter(|a| a.model_id == id)
22819 .map(|a| a.kind)
22820 .collect();
22821 assert!(kinds.contains(&action_ledger::ConciergeActionKind::RetireStarted));
22822 assert!(kinds.contains(&action_ledger::ConciergeActionKind::Retire));
22823
22824 let repo = layout();
22826 let again = engine
22827 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22828 .await;
22829 assert!(again.retired.is_none(), "{again:?}");
22830 assert!(again
22831 .plan
22832 .held
22833 .iter()
22834 .any(|h| h.model_id == id && h.why.contains("fetched back")));
22835 assert!(repo.exists());
22836
22837 let old = repo.join("blobs/aaaa.incomplete");
22839 let fresh = repo.join("blobs/bbbb.incomplete");
22840 std::fs::write(&old, b"half").unwrap();
22841 std::fs::write(&fresh, b"half").unwrap();
22842 let two_days_ago =
22843 std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 24 * 60 * 60);
22844 std::fs::File::options()
22845 .write(true)
22846 .open(&old)
22847 .unwrap()
22848 .set_modified(two_days_ago)
22849 .unwrap();
22850 let swept = engine
22851 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22852 .await;
22853 assert_eq!(swept.partials_discarded, vec![old.clone()], "{swept:?}");
22854 assert!(!old.exists() && fresh.exists());
22855 }
22856
22857 #[cfg(unix)]
22860 #[tokio::test]
22861 async fn maintenance_retires_an_orphan_car_fetched_only_under_pressure() {
22862 use crate::update_prefs::UpdatePolicy;
22863 let _serial = RETIRE_LOCKS.lock().await;
22864 let root = tempfile::tempdir().unwrap();
22865 let root_path = root.path().canonicalize().unwrap();
22866 let (engine, hub, models) = isolated_engine(&root_path);
22867 let hw = HardwareInfo::detect();
22868 let orphan = hub_repo(&hub, "dropped/model", &[("config.json", 10), ("w", 900)]);
22869 let theirs = hub_repo(
22870 &hub,
22871 "someone/checkout",
22872 &[("tokenizer.json", 5), ("w", 50)],
22873 );
22874 let fetched = |repo: &str, files: &[&str]| -> Vec<retire::Fetched> {
22875 files
22876 .iter()
22877 .map(|f| (repo.to_string(), f.to_string(), Some("s".to_string())))
22878 .collect()
22879 };
22880 retire::record_download(
22881 &root_path.join("state"),
22882 &hub,
22883 "old/row",
22884 &fetched("dropped/model", &["config.json", "w"]),
22885 );
22886 retire::record_download(
22887 &root_path.join("state"),
22888 &hub,
22889 "old/row",
22890 &fetched("someone/checkout", &["tokenizer.json"]),
22891 );
22892 let gb = 1_000_000_000;
22893 let none = crate::nudge::NudgeState::default();
22894 let roomy = Some(maintenance::DiskSpace {
22895 free_bytes: 500 * gb,
22896 total_bytes: 1_000 * gb,
22897 });
22898 let calm = engine
22899 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, roomy)
22900 .await;
22901 assert!(calm.orphan_retired.is_none());
22902 assert!(orphan.exists());
22903
22904 let tight = Some(maintenance::DiskSpace {
22905 free_bytes: 10 * gb,
22906 total_bytes: 1_000 * gb,
22907 });
22908 let link = models.join("Dropped-Model");
22911 std::os::unix::fs::symlink(orphan.join("snapshots/s"), &link).unwrap();
22912 let blocked = engine
22913 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22914 .await;
22915 assert!(blocked.orphan_retired.is_none(), "{blocked:?}");
22916 assert!(
22917 blocked
22918 .errors
22919 .iter()
22920 .any(|e| e.contains("resolves into it")),
22921 "{blocked:?}"
22922 );
22923 assert!(orphan.exists());
22924 std::fs::remove_file(&link).unwrap();
22925
22926 let pressed = engine
22927 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22928 .await;
22929 assert_eq!(
22930 pressed.orphan_retired.as_ref().map(|o| o.repo.as_str()),
22931 Some("dropped/model"),
22932 "{pressed:?}"
22933 );
22934 assert!(!orphan.exists());
22935 assert!(theirs.exists(), "a user's checkout is never touched");
22936 let again = engine
22937 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, tight)
22938 .await;
22939 assert!(again.orphan_retired.is_none(), "{again:?}");
22940 assert!(theirs.exists());
22941 }
22942
22943 #[tokio::test]
22947 async fn maintenance_raises_an_unchosen_memory_policy_on_evidence() {
22948 use crate::resource_policy::{
22949 FileResourcePolicyRepository, ResourcePolicy, ResourcePolicyRepository, ResourceProfile,
22950 };
22951 use crate::update_prefs::UpdatePolicy;
22952 let _serial = RETIRE_LOCKS.lock().await;
22953 let root = tempfile::tempdir().unwrap();
22954 let root_path = root.path().canonicalize().unwrap();
22955 let hub = root_path.join("hub");
22956 std::fs::create_dir_all(&hub).unwrap();
22957 let engine = InferenceEngine::new(InferenceConfig {
22958 state_root: root_path.join("state"),
22959 models_dir: root_path.join("weights"),
22960 ..InferenceConfig::default()
22961 });
22962 let hw = HardwareInfo {
22963 total_ram_mb: 64 * 1024,
22964 ..HardwareInfo::detect()
22965 };
22966 let big = engine
22968 .unified_registry()
22969 .all()
22970 .find(|m| {
22971 m.is_local()
22972 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::everyday())).fit
22973 == recommend::ModelFitStatus::TooBig
22974 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::local_focused())).fit
22975 == recommend::ModelFitStatus::Fits
22976 })
22977 .expect("a row only Local-focused admits")
22978 .id
22979 .clone();
22980 let ledger = engine
22981 .config
22982 .state_models_dir()
22983 .join("outcome_ledger.jsonl");
22984 std::fs::create_dir_all(ledger.parent().unwrap()).unwrap();
22985 let now = std::time::SystemTime::now()
22986 .duration_since(std::time::UNIX_EPOCH)
22987 .unwrap()
22988 .as_secs();
22989 let lines: String = (0..3)
22990 .map(|i| {
22991 format!(
22992 "{}\n",
22993 serde_json::json!({
22994 "trace_id": format!("t{i}"), "model_id": big, "task": "generate",
22995 "routing_reason": "explicit model", "latency_ms": 0,
22996 "input_tokens": 0, "output_tokens": 0, "error": "refused for memory",
22997 "attribution": "resource_blocked",
22998 "timestamp": now - 60 - i * 24 * 60 * 60
22999 })
23000 )
23001 })
23002 .collect();
23003 std::fs::write(&ledger, lines).unwrap();
23004 let none = crate::nudge::NudgeState::default();
23005 let repository = FileResourcePolicyRepository::new(root_path.join("state"));
23006
23007 let unobserved = engine
23009 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23010 .await;
23011 assert!(unobserved.resource_policy.is_none(), "{unobserved:?}");
23012
23013 engine
23014 .model_management_store()
23015 .set_usage_tracking_since_for_test(now - 4 * 24 * 60 * 60);
23016 let notify = engine
23017 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Notify, &none, None)
23018 .await;
23019 let suggested = notify.resource_policy.expect("suggested under Notify");
23020 assert!(!suggested.applied);
23021 assert!(!repository.path().exists(), "Notify saves nothing");
23022
23023 let auto = engine
23024 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23025 .await;
23026 assert!(
23027 auto.resource_policy.as_ref().is_some_and(|p| p.applied),
23028 "{auto:?}"
23029 );
23030 assert_eq!(repository.load().unwrap(), ResourcePolicy::local_focused());
23031 assert_eq!(
23032 repository.concierge_marker(),
23033 Some(ResourcePolicy::local_focused())
23034 );
23035 assert_eq!(
23036 engine.active_local_resource_policy().policy.profile,
23037 ResourceProfile::LocalFocused
23038 );
23039
23040 let small = engine
23043 .unified_registry()
23044 .all()
23045 .find(|m| {
23046 m.is_local()
23047 && recommend::model_fit(m, &hw, Some(&ResourcePolicy::everyday())).fit
23048 == recommend::ModelFitStatus::Fits
23049 })
23050 .unwrap()
23051 .id
23052 .clone();
23053 let later = std::time::SystemTime::now()
23054 .duration_since(std::time::UNIX_EPOCH)
23055 .unwrap()
23056 .as_secs()
23057 + 1;
23058 let mut text = std::fs::read_to_string(&ledger).unwrap();
23059 for i in 0..3 {
23060 text.push_str(&format!(
23061 "{}\n",
23062 serde_json::json!({
23063 "trace_id": format!("live{i}"), "model_id": small, "task": "generate",
23064 "routing_reason": "explicit model", "latency_ms": 0,
23065 "input_tokens": 0, "output_tokens": 0, "error": "not enough memory free",
23066 "attribution": "resource_blocked", "timestamp": later
23067 })
23068 ));
23069 }
23070 std::fs::write(&ledger, text).unwrap();
23071 let reverted = engine
23072 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23073 .await;
23074 let undo = reverted.resource_policy.expect("reverted");
23075 assert!(
23076 undo.applied && undo.to == ResourceProfile::Everyday,
23077 "{undo:?}"
23078 );
23079 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
23080 assert!(repository.concierge_choice().unwrap().reverted);
23081 let settled = engine
23082 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23083 .await;
23084 assert!(settled.resource_policy.is_none(), "{settled:?}");
23085
23086 repository.save(&ResourcePolicy::everyday()).unwrap();
23088 repository.clear_concierge_marker().unwrap();
23089 engine.apply_local_resource_policy(ResourcePolicy::everyday());
23090 let chosen = engine
23091 .maintenance_pass_with(&hub, &hw, false, UpdatePolicy::Auto, &none, None)
23092 .await;
23093 assert!(chosen.resource_policy.is_none(), "{chosen:?}");
23094 assert_eq!(repository.load().unwrap(), ResourcePolicy::everyday());
23095 }
23096
23097 #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
23103 #[tokio::test]
23104 async fn a_curated_upgrade_switches_the_lane_that_used_the_old_model() {
23105 let _serial = RETIRE_LOCKS.lock().await;
23106 let root = tempfile::tempdir().unwrap();
23107 let root_path = root.path().canonicalize().unwrap();
23108 let (engine, hub, models) = isolated_engine(&root_path);
23109 let from = plain_mlx_id(&engine);
23110 let to = "mlx/qwen3-8b:4bit".to_string();
23111 for id in [&from, &to] {
23112 let schema = engine.unified_registry().get(id).unwrap().clone();
23113 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23114 unreachable!()
23115 };
23116 let repo = hub_repo(
23117 &hub,
23118 hf_repo,
23119 &[("config.json", 10), ("model.safetensors", 500)],
23120 );
23121 std::os::unix::fs::symlink(repo.join("snapshots/s"), models.join(&schema.name))
23122 .unwrap();
23123 }
23124 let finding = |source| crate::upgrade::UpgradeFinding {
23125 from_id: from.clone(),
23126 from_name: from.clone(),
23127 to_id: to.clone(),
23128 to_name: to.clone(),
23129 reason: "newer".into(),
23130 trust_tier: crate::schema::TrustTier::Curated,
23131 source,
23132 target_pullable: true,
23133 };
23134 let now = 1;
23135 let none = crate::nudge::NudgeState::default();
23136
23137 let idle = engine
23139 .apply_one_curated_upgrade_from(
23140 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23141 &hub,
23142 &none,
23143 now,
23144 )
23145 .await
23146 .unwrap();
23147 assert!(idle.is_none());
23148
23149 engine
23150 .set_lane_default(None, crate::intent::UseCase::Assistant, &from)
23151 .unwrap();
23152 let community = engine
23154 .apply_one_curated_upgrade_from(
23155 vec![finding(crate::upgrade::UpgradeSource::Upstream)],
23156 &hub,
23157 &none,
23158 now,
23159 )
23160 .await
23161 .unwrap();
23162 assert!(community.is_none());
23163 let mut dismissed = crate::nudge::NudgeState::default();
23164 dismissed.dismiss(&format!("{from}=>{to}"));
23165 assert!(engine
23166 .apply_one_curated_upgrade_from(
23167 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23168 &hub,
23169 &dismissed,
23170 now,
23171 )
23172 .await
23173 .unwrap()
23174 .is_none());
23175
23176 let upgraded = engine
23177 .apply_one_curated_upgrade_from(
23178 vec![finding(crate::upgrade::UpgradeSource::Curated)],
23179 &hub,
23180 &none,
23181 now,
23182 )
23183 .await
23184 .unwrap()
23185 .expect("upgraded");
23186 assert_eq!(upgraded.lanes, vec!["assistant".to_string()]);
23187 assert_eq!(
23188 engine
23189 .lane_default(None, crate::intent::UseCase::Assistant)
23190 .as_deref(),
23191 Some(to.as_str())
23192 );
23193 let record = engine.recent_concierge_actions(now);
23194 let entry = record
23195 .iter()
23196 .find(|a| a.kind == "upgrade")
23197 .expect("on record");
23198 assert_eq!(
23199 entry.undo.as_deref(),
23200 Some("car concierge rollback assistant")
23201 );
23202 assert_eq!(entry.use_case.as_deref(), Some("assistant"));
23203 assert_eq!(entry.prior.as_deref(), Some(from.as_str()));
23204 }
23205
23206 #[tokio::test]
23209 async fn concierge_ask_without_a_question_states_the_facts_without_a_model() {
23210 let _serial = RETIRE_LOCKS.lock().await;
23211 let root = tempfile::tempdir().unwrap();
23212 let engine = InferenceEngine::new(InferenceConfig {
23213 state_root: root.path().join("state"),
23214 models_dir: root.path().join("weights"),
23215 ..InferenceConfig::default()
23216 });
23217 let answer = engine.concierge_ask(None).await.unwrap();
23218 assert!(answer.model_used.is_none());
23219 assert_eq!(answer.answer, answer.summary);
23220 assert!(
23221 answer.summary.contains("Installed local models:"),
23222 "{}",
23223 answer.summary
23224 );
23225 assert!(answer.summary.contains("Memory policy for local models:"));
23226 }
23227
23228 #[cfg(unix)]
23230 #[tokio::test]
23231 async fn a_refused_retirement_deletes_nothing() {
23232 let _serial = RETIRE_LOCKS.lock().await;
23233 let root = tempfile::tempdir().unwrap();
23234 let root_path = root.path().canonicalize().unwrap();
23235 let (engine, hub, _models) = isolated_engine(&root_path);
23236 let id = "mlx/qwen3-4b:4bit";
23237 let schema = engine.unified_registry().get(id).unwrap().clone();
23238 let ModelSource::Mlx { hf_repo, .. } = &schema.source else {
23239 unreachable!()
23240 };
23241 let repo = hub_repo(&hub, hf_repo, &[("w", 10)]);
23242 engine.lane_defaults_cache.write().unwrap().set(
23243 None,
23244 crate::intent::UseCase::Assistant,
23245 id.to_string(),
23246 1,
23247 );
23248 let outcome = engine.retire_model_in(id, &hub, false, None).await.unwrap();
23249 assert!(!outcome.executed);
23250 assert!(repo.exists());
23251 }
23252
23253 #[test]
23256 fn retire_plan_refuses_a_model_stored_outside_the_hub() {
23257 let root = tempfile::tempdir().unwrap();
23258 let engine = InferenceEngine::new(InferenceConfig {
23259 state_root: root.path().join("state"),
23260 models_dir: root.path().join("weights"),
23261 ..InferenceConfig::default()
23262 });
23263 let whisper = engine
23264 .unified_registry()
23265 .all()
23266 .find(|m| matches!(m.source, ModelSource::WhisperCpp { .. }))
23267 .expect("a whisper.cpp row")
23268 .id
23269 .clone();
23270 let plan = engine.retire_plan_in(&whisper, root.path()).unwrap();
23271 assert!(
23272 matches!(
23273 &plan.refusals[..],
23274 [retire::RetireRefusal::Unsupported { .. }]
23275 ),
23276 "{plan:?}"
23277 );
23278 }
23279
23280 #[test]
23282 fn downloads_are_recorded_per_hub() {
23283 let root = tempfile::tempdir().unwrap();
23284 let fetched = [(
23285 "renamed/repo".to_string(),
23286 "model.safetensors".to_string(),
23287 Some("r1".to_string()),
23288 )];
23289 let hub = root.path().join("hub");
23290 retire::record_download(root.path(), &hub, "old/row", &fetched);
23291 let recorded = retire::recorded_files(root.path(), &hub);
23292 assert_eq!(
23293 recorded.get("renamed/repo"),
23294 Some(&retire::Owned::Files(std::collections::BTreeSet::from([
23295 retire::OwnedFile {
23296 path: "model.safetensors".into(),
23297 revision: Some("r1".into()),
23298 }
23299 ])))
23300 );
23301 assert!(retire::recorded_files(root.path(), Path::new("/elsewhere")).is_empty());
23302 }
23303
23304 #[cfg(unix)]
23309 #[tokio::test]
23310 async fn a_failed_ledger_write_leaves_the_profiles_untouched_and_retries_once() {
23311 use std::os::unix::fs::PermissionsExt;
23312 let root = tempfile::tempdir().unwrap();
23313 let state = root.path().join("state");
23314 let models = state.join("models");
23315 std::fs::create_dir_all(&models).unwrap();
23316 let ledger = models.join("outcome_ledger.jsonl");
23317 let real = "mlx/qwen3-4b:4bit";
23318 let profiles = write_legacy_ledger(&ledger, real, "ghost/never-ran");
23319 let mut seed = OutcomeTracker::new();
23320 seed.import_profiles(profiles);
23321 seed.save_to_file(&models.join("outcome_profiles.json"))
23322 .unwrap();
23323 let engine = InferenceEngine::new(InferenceConfig {
23324 state_root: state.clone(),
23325 models_dir: root.path().join("weights"),
23326 ..InferenceConfig::default()
23327 });
23328 let before = profile_json(&*engine.outcome_tracker().read().await, real);
23329 let dirty_before = engine.outcome_tracker().read().await.is_dirty();
23330 assert!(
23331 engine
23332 .outcome_tracker()
23333 .read()
23334 .await
23335 .profile(real)
23336 .unwrap()
23337 .fail_count
23338 > 1
23339 );
23340
23341 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o555)).unwrap();
23343 let failed = engine.reconcile_outcome_ledger().await;
23344 std::fs::set_permissions(&models, std::fs::Permissions::from_mode(0o755)).unwrap();
23345 assert!(failed.is_err(), "the write was refused");
23346 {
23347 let tracker = engine.outcome_tracker();
23348 let tracker = tracker.read().await;
23349 assert_eq!(profile_json(&tracker, real), before, "profiles untouched");
23350 assert_eq!(
23351 tracker.is_dirty(),
23352 dirty_before,
23353 "the failed reconcile left nothing new for the periodic save to persist"
23354 );
23355 }
23356
23357 engine.reconcile_outcome_ledger().await.unwrap();
23359 engine.reconcile_outcome_ledger().await.unwrap();
23360 let p = engine
23361 .outcome_tracker()
23362 .read()
23363 .await
23364 .profile(real)
23365 .unwrap()
23366 .clone();
23367 assert_eq!(
23368 p.fail_count, 1,
23369 "only the genuine failure remains, taken back once"
23370 );
23371 }
23372
23373 #[test]
23376 fn engine_construction_never_rewrites_the_ledger() {
23377 let root = tempfile::tempdir().unwrap();
23378 let state = root.path().join("state");
23379 let models = state.join("models");
23380 std::fs::create_dir_all(&models).unwrap();
23381 let ledger = models.join("outcome_ledger.jsonl");
23382 write_legacy_ledger(&ledger, "mlx/real:4bit", "ghost/never-ran");
23383 let before = std::fs::read(&ledger).unwrap();
23384 let _engine = InferenceEngine::new(InferenceConfig {
23385 state_root: state,
23386 models_dir: root.path().join("weights"),
23387 ..InferenceConfig::default()
23388 });
23389 assert_eq!(std::fs::read(&ledger).unwrap(), before);
23390 }
23391
23392 #[test]
23393 fn dispatch_outcome_guards_keep_real_failures_and_existing_rejections_distinct() {
23394 let model_id = "test/model";
23395
23396 let real_failure = InferenceError::InferenceFailed("provider returned HTTP 500".into());
23397 let mut tracker = OutcomeTracker::new();
23398 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
23399 record_dispatch_failure(&mut tracker, &trace, &real_failure);
23400 let profile = tracker.profile(model_id).expect("real failure profile");
23401 assert_eq!(profile.success_count, 0);
23402 assert_eq!(profile.fail_count, 1);
23403 let receipt = tracker.drain_ledger().pop().unwrap();
23404 assert_eq!(receipt.success, Some(false));
23405 assert!(error_counts_against_circuit_breaker(&real_failure));
23406 let mut breakers = crate::routing_ext::CircuitBreakerRegistry::new(1, 60);
23407 breakers.record_failure(model_id);
23408 assert_eq!(
23409 breakers.state(model_id),
23410 Some(crate::routing_ext::CircuitState::Open)
23411 );
23412
23413 for rejection in [
23414 InferenceError::GatewayUnconfigured {
23415 provider: "parslee".into(),
23416 namespace: "parslee/openrouter/".into(),
23417 status: 503,
23418 message: "not configured".into(),
23419 },
23420 InferenceError::ContentRefused {
23421 provider: "parslee".into(),
23422 kind: Some("invalid_request_error".into()),
23423 code: Some("content_policy_violation".into()),
23424 message: "content refused".into(),
23425 },
23426 ] {
23427 let mut tracker = OutcomeTracker::new();
23428 let trace = tracker.record_start(model_id, InferenceTask::Generate, "test");
23429 record_dispatch_failure(&mut tracker, &trace, &rejection);
23430 assert!(
23431 tracker.profile(model_id).is_none(),
23432 "existing rejection changed model profile: {rejection}"
23433 );
23434 let receipt = tracker.drain_ledger().pop().unwrap();
23435 assert_eq!(receipt.success, None, "{rejection}");
23436 assert_eq!(receipt.quality, None, "{rejection}");
23437 assert!(!error_counts_against_circuit_breaker(&rejection));
23438 }
23439 }
23440
23441 #[tokio::test(flavor = "current_thread")]
23454 async fn openrouter_out_of_credits_does_not_degrade_the_model() {
23455 use wiremock::matchers::{method, path};
23456 use wiremock::{Mock, MockServer, ResponseTemplate};
23457
23458 let _credential_scope = crate::openrouter::test_credential_scope();
23459 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
23460 let server = MockServer::start().await;
23461 Mock::given(method("POST"))
23462 .and(path("/v1/chat/completions"))
23463 .respond_with(
23464 ResponseTemplate::new(402)
23465 .set_body_string(r#"{"error":{"code":402,"message":"Insufficient credits"}}"#),
23466 )
23467 .mount(&server)
23468 .await;
23469
23470 let tmp = TempDir::new().unwrap();
23471 let models_dir = tmp.path().join("models");
23472 let mut engine = InferenceEngine::new(test_config(models_dir.clone()));
23473 let mut schema = crate::openrouter::curated_schemas()
23474 .into_iter()
23475 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
23476 .unwrap();
23477 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
23478 *endpoint = server.uri();
23479 }
23480 let model_id = schema.id.clone();
23481 engine.unified_registry_mut().register_project_model(schema);
23482
23483 let err = engine
23484 .generate_tracked(GenerateRequest {
23485 prompt: "bill me".into(),
23486 model: Some(model_id.clone()),
23487 params: GenerateParams {
23488 strict_model: true,
23489 ..Default::default()
23490 },
23491 ..Default::default()
23492 })
23493 .await
23494 .expect_err("402 must not succeed");
23495
23496 match &err {
23497 InferenceError::ProviderAccount {
23498 provider, status, ..
23499 } => {
23500 assert_eq!(status, &402);
23501 assert_eq!(provider, "openrouter");
23502 }
23503 other => panic!("expected ProviderAccount, got {other:?}"),
23504 }
23505 assert!(
23506 !error_counts_against_circuit_breaker(&err),
23507 "an account rejection must not feed the per-model breaker"
23508 );
23509
23510 let tracker_handle = engine.outcome_tracker();
23512 let tracker = tracker_handle.read().await;
23513 let profile = tracker.profile(&model_id).cloned();
23514 assert!(
23515 profile.as_ref().is_none_or(|p| p.fail_count == 0),
23516 "account rejection degraded the model profile: {profile:?}"
23517 );
23518 drop(tracker);
23519
23520 let ledger = crate::outcome::read_ledger(&models_dir.join("outcome_ledger.jsonl"), 0);
23524 let entry = ledger
23525 .iter()
23526 .find(|e| e.model_id == model_id)
23527 .expect("the attempt must still produce a receipt");
23528 assert!(
23529 entry.success.is_none() && entry.quality.is_none(),
23530 "receipt must record the attempt without a verdict: {entry:?}"
23531 );
23532
23533 assert!(
23535 engine
23536 .adaptive_router
23537 .circuit_breakers
23538 .lock()
23539 .unwrap()
23540 .state(&model_id)
23541 .is_none(),
23542 "account rejection must not create breaker state for the model"
23543 );
23544 }
23545
23546 #[tokio::test(flavor = "current_thread")]
23547 async fn openrouter_stream_account_error_is_unattributed_and_never_completes_successfully() {
23548 use wiremock::matchers::{method, path};
23549 use wiremock::{Mock, MockServer, ResponseTemplate};
23550
23551 let _credential_scope = crate::openrouter::test_credential_scope();
23552 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
23553 let server = MockServer::start().await;
23554 Mock::given(method("POST"))
23555 .and(path("/v1/chat/completions"))
23556 .respond_with(ResponseTemplate::new(200).set_body_raw(
23557 "data: {\"error\":{\"code\":402,\"message\":\"private balance details\"}}\n\n",
23558 "text/event-stream",
23559 ))
23560 .mount(&server)
23561 .await;
23562
23563 let tmp = TempDir::new().unwrap();
23564 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23565 let mut schema = crate::openrouter::curated_schemas()
23566 .into_iter()
23567 .find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
23568 .unwrap();
23569 if let ModelSource::RemoteApi { endpoint, .. } = &mut schema.source {
23570 *endpoint = server.uri();
23571 }
23572 let model_id = schema.id.clone();
23573 engine.unified_registry_mut().register_project_model(schema);
23574
23575 let mut handle = engine
23576 .generate_tracked_stream(GenerateRequest {
23577 prompt: "fail after headers".into(),
23578 model: Some(model_id.clone()),
23579 ..Default::default()
23580 })
23581 .await
23582 .unwrap();
23583 let mut events = Vec::new();
23584 while let Some(event) = handle.events.recv().await {
23585 events.push(event);
23586 }
23587 assert!(matches!(
23588 events.as_slice(),
23589 [StreamEvent::Error(message)] if message == "OpenRouter account is out of credits"
23590 ));
23591 assert!(!events
23592 .iter()
23593 .any(|event| matches!(event, StreamEvent::Done { .. })));
23594
23595 let mut receipts = Vec::new();
23598 for _ in 0..50 {
23599 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
23600 if !receipts.is_empty() {
23601 break;
23602 }
23603 tokio::task::yield_now().await;
23604 }
23605 let [receipt] = receipts.as_slice() else {
23606 panic!("expected one receipt, got {receipts:?}");
23607 };
23608 assert_eq!(receipt.success, None);
23609 assert_eq!(
23610 receipt.attribution,
23611 Some(crate::outcome::UnattributedReason::Account)
23612 );
23613 assert!(
23614 engine
23615 .outcome_tracker()
23616 .read()
23617 .await
23618 .profile(&model_id)
23619 .is_none(),
23620 "an account rejection must not touch the model's profile"
23621 );
23622 }
23623
23624 #[tokio::test(flavor = "current_thread")]
23627 async fn tracked_stream_dropped_by_the_caller_is_cancelled_not_a_failure() {
23628 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23629 let tmp = TempDir::new().unwrap();
23630 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23631 pin_test_local_admission_memory(&mut engine);
23632 let model_id = install_small_local_fixture(&engine);
23633 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
23634
23635 let stream = engine
23636 .generate_tracked_stream(GenerateRequest {
23637 prompt: "interrupted".into(),
23638 model: Some(model_id.clone()),
23639 params: GenerateParams {
23640 strict_model: true,
23641 ..Default::default()
23642 },
23643 ..Default::default()
23644 })
23645 .await
23646 .unwrap();
23647 drop(stream);
23648
23649 let mut receipts = Vec::new();
23650 for _ in 0..200 {
23651 receipts.extend(engine.outcome_tracker().write().await.drain_ledger());
23652 if !receipts.is_empty() {
23653 break;
23654 }
23655 tokio::task::yield_now().await;
23656 }
23657 crate::offload::set_local_offload(None);
23658 let [receipt] = receipts.as_slice() else {
23659 panic!("expected one receipt, got {receipts:?}");
23660 };
23661 assert_eq!(receipt.success, None);
23662 assert_eq!(
23663 receipt.attribution,
23664 Some(crate::outcome::UnattributedReason::Cancelled)
23665 );
23666 assert!(engine
23667 .outcome_tracker()
23668 .read()
23669 .await
23670 .profile(&model_id)
23671 .is_none_or(|p| p.fail_count == 0));
23672 }
23673
23674 #[test]
23678 fn an_apple_guardrail_decline_does_not_end_the_fallback_chain() {
23679 let refusal = |provider: &str| InferenceError::ContentRefused {
23680 provider: provider.into(),
23681 kind: Some("guardrailViolation".into()),
23682 code: None,
23683 message: "declined".into(),
23684 };
23685 assert!(!error_ends_fallback_chain(&refusal("apple")));
23686 assert!(error_ends_fallback_chain(&refusal("parslee")));
23687 assert!(!error_counts_against_circuit_breaker(&refusal("apple")));
23688 }
23689
23690 #[tokio::test(flavor = "current_thread")]
23691 async fn tracked_stream_without_done_records_failure_not_success() {
23692 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23693 let tmp = TempDir::new().unwrap();
23694 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23695 pin_test_local_admission_memory(&mut engine);
23696 let model_id = install_small_local_fixture(&engine);
23697 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: false })));
23698
23699 let mut stream = engine
23700 .generate_tracked_stream(GenerateRequest {
23701 prompt: "must not count as success".into(),
23702 model: Some(model_id.clone()),
23703 params: GenerateParams {
23704 strict_model: true,
23705 ..Default::default()
23706 },
23707 ..Default::default()
23708 })
23709 .await
23710 .unwrap();
23711 while stream.events.recv().await.is_some() {}
23712 crate::offload::set_local_offload(None);
23713
23714 for _ in 0..50 {
23715 if engine
23716 .outcome_tracker()
23717 .read()
23718 .await
23719 .profile(&model_id)
23720 .is_some_and(|profile| profile.fail_count == 1)
23721 {
23722 break;
23723 }
23724 tokio::task::yield_now().await;
23725 }
23726 let profile = engine
23727 .outcome_tracker()
23728 .read()
23729 .await
23730 .profile(&model_id)
23731 .cloned()
23732 .unwrap();
23733 assert_eq!(profile.fail_count, 1);
23734 assert_eq!(profile.success_count, 0);
23735 }
23736
23737 #[tokio::test(flavor = "current_thread")]
23738 async fn google_vertex_terminal_matrix_records_only_deliberate_finishes_as_success() {
23739 use wiremock::matchers::{method, path};
23740 use wiremock::{Mock, MockServer, ResponseTemplate};
23741
23742 let _provider_env = crate::openrouter::test_environment_scope_async().await;
23743 let _env = ENV_MUTEX.lock().await;
23744 unsafe { std::env::set_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY", "matrix-key") };
23745
23746 for (protocol, reason, should_succeed) in [
23747 (schema::ApiProtocol::Google, "STOP", true),
23748 (schema::ApiProtocol::Google, "SAFETY", false),
23749 (schema::ApiProtocol::VertexAi, "MAX_TOKENS", true),
23750 (
23751 schema::ApiProtocol::VertexAi,
23752 "MALFORMED_FUNCTION_CALL",
23753 false,
23754 ),
23755 ] {
23756 let server = MockServer::start().await;
23757 let expected_path = match protocol {
23758 schema::ApiProtocol::Google => "/v1beta/models/gemini-test:streamGenerateContent",
23759 schema::ApiProtocol::VertexAi => {
23760 "/publishers/google/models/gemini-test:streamGenerateContent"
23761 }
23762 _ => unreachable!(),
23763 };
23764 Mock::given(method("POST"))
23765 .and(path(expected_path))
23766 .respond_with(ResponseTemplate::new(200).set_body_raw(
23767 format!(
23768 "data: {{\"candidates\":[{{\"content\":{{\"parts\":[{{\"text\":\"matrix\"}}]}},\"finishReason\":\"{reason}\"}}]}}\n\n"
23769 ),
23770 "text/event-stream",
23771 ))
23772 .mount(&server)
23773 .await;
23774
23775 let id = format!("test/{protocol:?}-{reason}");
23776 let tmp = TempDir::new().unwrap();
23777 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23778 engine.register_model(remote_stream_fixture_schema(
23779 &id,
23780 server.uri(),
23781 protocol,
23782 "CAR_GOOGLE_OUTCOME_MATRIX_KEY",
23783 ));
23784 let mut stream = engine
23785 .generate_tracked_stream(GenerateRequest {
23786 prompt: "matrix".into(),
23787 model: Some(id.clone()),
23788 params: GenerateParams {
23789 strict_model: true,
23790 ..Default::default()
23791 },
23792 ..Default::default()
23793 })
23794 .await
23795 .unwrap();
23796 let mut events = Vec::new();
23797 while let Some(event) = stream.events.recv().await {
23798 events.push(event);
23799 }
23800 for _ in 0..50 {
23801 if engine
23802 .outcome_tracker()
23803 .read()
23804 .await
23805 .profile(&id)
23806 .is_some_and(|profile| profile.total_calls == 1)
23807 {
23808 break;
23809 }
23810 tokio::task::yield_now().await;
23811 }
23812 let profile = engine
23813 .outcome_tracker()
23814 .read()
23815 .await
23816 .profile(&id)
23817 .cloned()
23818 .unwrap();
23819 assert_eq!(
23820 (profile.success_count, profile.fail_count),
23821 if should_succeed { (1, 0) } else { (0, 1) },
23822 "{protocol:?}/{reason}: {events:?}"
23823 );
23824 }
23825
23826 unsafe { std::env::remove_var("CAR_GOOGLE_OUTCOME_MATRIX_KEY") };
23827 }
23828
23829 #[tokio::test(flavor = "current_thread")]
23830 async fn remote_primary_stream_setup_failure_falls_back_to_installed_local_dispatch() {
23831 use wiremock::matchers::{method, path};
23832 use wiremock::{Mock, MockServer, ResponseTemplate};
23833
23834 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23835 let _env = ENV_MUTEX.lock().await;
23836 let server = MockServer::start().await;
23837 Mock::given(method("POST"))
23838 .and(path("/v1/chat/completions"))
23839 .respond_with(ResponseTemplate::new(503))
23840 .mount(&server)
23841 .await;
23842 unsafe { std::env::set_var("CAR_STREAM_FALLBACK_TEST_KEY", "fixture") };
23843
23844 let tmp = TempDir::new().unwrap();
23845 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23846 pin_test_local_admission_memory(&mut engine);
23847 let _local_id = install_small_local_fixture(&engine);
23848 let remote_id = "test/remote-primary";
23849 engine.register_model(ModelSchema {
23850 id: remote_id.into(),
23851 name: "remote-primary".into(),
23852 provider: "test".into(),
23853 family: "test".into(),
23854 version: "1".into(),
23855 capabilities: vec![ModelCapability::Generate],
23856 context_length: 8_192,
23857 max_output_tokens: Some(1_024),
23858 param_count: String::new(),
23859 quantization: None,
23860 performance: Default::default(),
23861 cost: Default::default(),
23862 source: ModelSource::RemoteApi {
23863 endpoint: server.uri(),
23864 api_key_env: "CAR_STREAM_FALLBACK_TEST_KEY".into(),
23865 api_key_envs: vec![],
23866 api_version: None,
23867 protocol: schema::ApiProtocol::OpenAiCompat,
23868 },
23869 tags: vec![],
23870 supported_params: vec![],
23871 public_benchmarks: vec![],
23872 trust_tier: TrustTier::Community,
23873 deprecated: false,
23874 available: true,
23875 weights_ready: true,
23876 });
23877 crate::offload::set_local_offload(Some(Arc::new(FixtureLocalOffload { emit_done: true })));
23878
23879 let mut stream = engine
23880 .generate_tracked_stream(GenerateRequest {
23881 prompt: "fall back".into(),
23882 model: Some(remote_id.into()),
23883 ..Default::default()
23884 })
23885 .await
23886 .expect("compatible installed local model should be dispatched");
23887 assert_ne!(stream.model_used, remote_id);
23888 assert!(
23889 engine
23890 .unified_registry()
23891 .get(&stream.model_used)
23892 .is_some_and(ModelSchema::is_local),
23893 "fallback must stay on a compatible local model: {}",
23894 stream.model_used
23895 );
23896 let mut saw_done = false;
23897 while let Some(event) = stream.events.recv().await {
23898 saw_done |= matches!(event, StreamEvent::Done { .. });
23899 }
23900 assert!(saw_done);
23901
23902 crate::offload::set_local_offload(None);
23903 unsafe { std::env::remove_var("CAR_STREAM_FALLBACK_TEST_KEY") };
23904 }
23905
23906 #[tokio::test(flavor = "current_thread")]
23907 async fn exact_model_id_rejects_display_name_without_dispatch() {
23908 let tmp = TempDir::new().unwrap();
23909 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23910 let mut first = remote_stream_fixture_schema(
23911 "test/ambiguous-a:1",
23912 "http://127.0.0.1:9".into(),
23913 schema::ApiProtocol::OpenAiCompat,
23914 "CAR_AMBIGUOUS_EXACT_KEY",
23915 );
23916 first.name = "Shared Alias".into();
23917 let mut second = first.clone();
23918 second.id = "test/ambiguous-b:1".into();
23919 engine.register_model(first);
23920 engine.register_model(second);
23921 let mut request = GenerateRequest {
23922 prompt: "must not dispatch".into(),
23923 ..Default::default()
23924 };
23925 pin_exact_model_id(&mut request, "Shared Alias".into()).unwrap();
23926
23927 let error = engine
23928 .generate_tracked(request)
23929 .await
23930 .expect_err("an exact-id pin must never resolve a display name");
23931 assert!(matches!(error, InferenceError::ModelNotFound(_)));
23932 }
23933
23934 #[tokio::test(flavor = "current_thread")]
23935 async fn exact_openai_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
23936 let _env = ENV_MUTEX.lock().await;
23937 assert_remote_model_identity_contract(
23938 schema::ApiProtocol::OpenAiCompat,
23939 "openai/newsroom-gpt-5.5-2026-04-23:test",
23940 "newsroom-gpt-5.5-2026-04-23-test",
23941 "CAR_OPENAI_IDENTITY_TEST_KEY",
23942 )
23943 .await;
23944 }
23945
23946 #[tokio::test(flavor = "current_thread")]
23947 async fn exact_anthropic_pin_reports_catalog_id_and_loose_request_keeps_provider_name() {
23948 let _env = ENV_MUTEX.lock().await;
23949 assert_remote_model_identity_contract(
23950 schema::ApiProtocol::Anthropic,
23951 "anthropic/newsroom-claude-opus-4-8:test",
23952 "newsroom-claude-opus-4-8-test",
23953 "CAR_ANTHROPIC_IDENTITY_TEST_KEY",
23954 )
23955 .await;
23956 }
23957
23958 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
23959 #[tokio::test(flavor = "current_thread")]
23960 async fn exact_model_id_nonstream_bypasses_mlx_equivalent_substitution() {
23961 let _offload_guard = crate::offload::test_offload_lock().lock().await;
23962 let tmp = TempDir::new().unwrap();
23963 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
23964 let exact_id = install_exact_pin_equivalent_fixture(&engine);
23965 let offload = Arc::new(ExactPinCaptureOffload::new());
23966 crate::offload::set_local_offload(Some(offload.clone()));
23967
23968 let mut request = GenerateRequest {
23969 prompt: "use only the immutable row I selected".into(),
23970 ..Default::default()
23971 };
23972 pin_exact_model_id(&mut request, exact_id.clone()).unwrap();
23973 let error = engine.generate_tracked(request).await.unwrap_err();
23974
23975 assert!(offload.dispatched_models().is_empty());
23976 assert!(error.to_string().contains(&exact_id), "{error}");
23977 assert!(
23978 error
23979 .to_string()
23980 .contains("MLX-equivalent substitution is disabled for exact pins"),
23981 "{error}"
23982 );
23983 crate::offload::set_local_offload(None);
23984 }
23985
23986 #[tokio::test(flavor = "current_thread")]
23987 async fn catalog_identity_mismatch_rejects_before_provider_dispatch() {
23988 use wiremock::matchers::{method, path};
23989 use wiremock::{Mock, MockServer, ResponseTemplate};
23990
23991 let _env = ENV_MUTEX.lock().await;
23992 let server = MockServer::start().await;
23993 Mock::given(method("POST"))
23994 .and(path("/v1/chat/completions"))
23995 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
23996 "choices": [{"message": {"content": "must not run"}, "finish_reason": "stop"}],
23997 })))
23998 .mount(&server)
23999 .await;
24000 unsafe { std::env::set_var("CAR_PRECONDITION_TEST_KEY", "fixture") };
24001
24002 let tmp = TempDir::new().unwrap();
24003 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24004 let model_id = "test/catalog-precondition:1";
24005 engine.register_model(remote_stream_fixture_schema(
24006 model_id,
24007 server.uri(),
24008 schema::ApiProtocol::OpenAiCompat,
24009 "CAR_PRECONDITION_TEST_KEY",
24010 ));
24011 let snapshot = engine.catalog_snapshot().unwrap();
24012 let row_digest = snapshot
24013 .model_by_exact_id(model_id)
24014 .unwrap()
24015 .row_digest
24016 .clone();
24017
24018 for (expected_row_digest, expected_catalog_revision) in [
24019 (
24020 Some("0".repeat(64)),
24021 Some(snapshot.catalog_revision.clone()),
24022 ),
24023 (Some(row_digest.clone()), Some("f".repeat(64))),
24024 ] {
24025 let mut request = GenerateRequest {
24026 prompt: "must not dispatch".into(),
24027 expected_row_digest,
24028 expected_catalog_revision,
24029 ..Default::default()
24030 };
24031 pin_exact_model_id(&mut request, model_id.into()).unwrap();
24032 let error = engine.generate_tracked(request).await.unwrap_err();
24033 assert!(matches!(
24034 error,
24035 InferenceError::CatalogPreconditionMismatch { .. }
24036 ));
24037 }
24038 assert!(
24039 server.received_requests().await.unwrap().is_empty(),
24040 "catalog identity mismatches must fail before provider dispatch"
24041 );
24042
24043 unsafe { std::env::remove_var("CAR_PRECONDITION_TEST_KEY") };
24044 }
24045
24046 #[tokio::test(flavor = "current_thread")]
24047 async fn exact_nonstream_identity_survives_thinking_retry() {
24048 use std::sync::atomic::{AtomicUsize, Ordering};
24049 use wiremock::matchers::{method, path};
24050 use wiremock::{Mock, MockServer, ResponseTemplate};
24051
24052 let _env = ENV_MUTEX.lock().await;
24053 let server = MockServer::start().await;
24054 let calls = Arc::new(AtomicUsize::new(0));
24055 let response_calls = calls.clone();
24056 Mock::given(method("POST"))
24057 .and(path("/v1/chat/completions"))
24058 .respond_with(move |_request: &wiremock::Request| {
24059 let attempt = response_calls.fetch_add(1, Ordering::SeqCst);
24060 let content = if attempt == 0 { "" } else { "recovered" };
24061 ResponseTemplate::new(200).set_body_json(serde_json::json!({
24062 "choices": [{
24063 "message": {"content": content},
24064 "finish_reason": if attempt == 0 { "length" } else { "stop" },
24065 }],
24066 "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
24067 }))
24068 })
24069 .mount(&server)
24070 .await;
24071 unsafe { std::env::set_var("CAR_IDENTITY_RETRY_KEY", "fixture") };
24072
24073 let tmp = TempDir::new().unwrap();
24074 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24075 let model_id = "test/identity-retry:1";
24076 engine.register_model(remote_stream_fixture_schema(
24077 model_id,
24078 server.uri(),
24079 schema::ApiProtocol::OpenAiCompat,
24080 "CAR_IDENTITY_RETRY_KEY",
24081 ));
24082 let snapshot = engine.catalog_snapshot().unwrap();
24083 let expected_row = snapshot.model_by_exact_id(model_id).unwrap();
24084 let mut request = GenerateRequest {
24085 prompt: "retry once".into(),
24086 expected_row_digest: Some(expected_row.row_digest.clone()),
24087 expected_catalog_revision: Some(snapshot.catalog_revision.clone()),
24088 ..Default::default()
24089 };
24090 pin_exact_model_id(&mut request, model_id.into()).unwrap();
24091
24092 let result = engine.generate_tracked(request).await.unwrap();
24093 assert_eq!(calls.load(Ordering::SeqCst), 2, "must exercise retry path");
24094 assert_eq!(result.stop_reason.as_deref(), Some("thinking_recovered"));
24095 assert_eq!(
24096 result.model_identity.requested_model_id.as_deref(),
24097 Some(model_id)
24098 );
24099 assert_eq!(result.model_identity.resolved_model_id, model_id);
24100 assert_eq!(result.model_identity.row_digest, expected_row.row_digest);
24101 assert_eq!(
24102 result.model_identity.catalog_revision,
24103 snapshot.catalog_revision
24104 );
24105
24106 unsafe { std::env::remove_var("CAR_IDENTITY_RETRY_KEY") };
24107 }
24108
24109 #[tokio::test(flavor = "current_thread")]
24110 async fn managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns() {
24111 if !crate::run_in_isolated_test_process(
24112 "tests::managed_openrouter_reasoning_items_roundtrip_verbatim_across_two_turns",
24113 "CAR_MANAGED_REASONING_ROUNDTRIP_CHILD",
24114 ) {
24115 return;
24116 }
24117 let _home = crate::openrouter::StateRootScope::new();
24118 use wiremock::matchers::{header, method, path};
24119 use wiremock::{Mock, MockServer, ResponseTemplate};
24120
24121 let _provider_env = crate::openrouter::test_environment_scope_async().await;
24122 let _env = ENV_MUTEX.lock().await;
24123 let bearer = "managed-reasoning-roundtrip-bearer";
24124
24125 let server = MockServer::start().await;
24126 unsafe {
24127 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
24128 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
24129 }
24130 Mock::given(method("GET"))
24131 .and(path("/api/v1/organizations/me"))
24132 .and(header("authorization", format!("Bearer {bearer}")))
24133 .respond_with(
24134 ResponseTemplate::new(200)
24135 .set_body_json(serde_json::json!({"organizationId": "org-roundtrip"})),
24136 )
24137 .mount(&server)
24138 .await;
24139 Mock::given(method("GET"))
24140 .and(path("/connect/session"))
24141 .respond_with(
24142 ResponseTemplate::new(200)
24143 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
24144 )
24145 .mount(&server)
24146 .await;
24147 Mock::given(method("POST"))
24148 .and(path("/api/v1/orgs/org-roundtrip/inference/responses"))
24149 .respond_with(ResponseTemplate::new(200).set_body_raw(
24150 include_str!("../tests/fixtures/parslee-openrouter-reasoning-roundtrip.sse"),
24151 "text/event-stream",
24152 ))
24153 .expect(2)
24154 .mount(&server)
24155 .await;
24156
24157 let tmp = TempDir::new().unwrap();
24158 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24159 let schema = crate::openrouter::curated_schemas()
24160 .into_iter()
24161 .find(|schema| schema.id == "parslee/openrouter/frontier-general")
24162 .unwrap();
24163 engine.unified_registry_mut().register_project_model(schema);
24164
24165 let first = engine
24166 .generate_tracked(GenerateRequest {
24167 prompt: "first".into(),
24168 model: Some("parslee/openrouter/frontier-general".into()),
24169 ..Default::default()
24170 })
24171 .await
24172 .expect("first managed turn");
24173 let expected_reasoning = serde_json::json!({
24174 "type": "reasoning",
24175 "id": "rs_car_roundtrip",
24176 "status": "completed",
24177 "summary": [{"type": "summary_text", "text": "safe summary"}],
24178 "encrypted_content": "opaque-encrypted-reasoning",
24179 });
24180 assert_eq!(
24181 first.provider_output_items,
24182 vec![expected_reasoning.clone()]
24183 );
24184
24185 let mut history = vec![Message::User {
24186 content: "first".into(),
24187 }];
24188 first.append_assistant_history(&mut history, first.tool_calls.clone());
24189 history.push(Message::User {
24190 content: "continue".into(),
24191 });
24192
24193 let second = engine
24194 .generate_tracked(GenerateRequest {
24195 prompt: String::new(),
24196 model: Some("parslee/openrouter/frontier-general".into()),
24197 messages: Some(history),
24198 ..Default::default()
24199 })
24200 .await
24201 .expect("second managed turn");
24202 assert_eq!(second.text, "first answer");
24203
24204 let requests = server.received_requests().await.unwrap();
24205 let posts: Vec<serde_json::Value> = requests
24206 .iter()
24207 .filter(|request| {
24208 request.method.as_str() == "POST"
24209 && request.url.path() == "/api/v1/orgs/org-roundtrip/inference/responses"
24210 })
24211 .map(|request| serde_json::from_slice(&request.body).unwrap())
24212 .collect();
24213 assert_eq!(posts.len(), 2);
24214 for body in &posts {
24215 assert_eq!(body["store"], false);
24216 assert_eq!(
24217 body["include"],
24218 serde_json::json!(["reasoning.encrypted_content"])
24219 );
24220 }
24221 let second_input = posts[1]["input"].as_array().unwrap();
24222 let reasoning_index = second_input
24223 .iter()
24224 .position(|item| item == &expected_reasoning)
24225 .expect("second request must replay the exact reasoning item");
24226 let assistant_index = second_input
24227 .iter()
24228 .position(|item| item["role"] == "assistant")
24229 .expect("second request assistant turn");
24230 let user_index = second_input
24231 .iter()
24232 .position(|item| item["content"] == "continue")
24233 .expect("second request user turn");
24234 assert!(
24235 reasoning_index < assistant_index && assistant_index < user_index,
24236 "provider output order must be reasoning, assistant text, then the next user turn"
24237 );
24238
24239 unsafe {
24240 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
24241 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
24242 }
24243 }
24244
24245 #[tokio::test(flavor = "current_thread")]
24246 async fn managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures() {
24247 if !crate::run_in_isolated_test_process(
24248 "tests::managed_partial_eof_fails_buffered_and_streamed_turns_and_records_only_failures",
24249 "CAR_MANAGED_PARTIAL_EOF_CHILD",
24250 ) {
24251 return;
24252 }
24253 let _home = crate::openrouter::StateRootScope::new();
24254 use wiremock::matchers::{header, method, path};
24255 use wiremock::{Mock, MockServer, ResponseTemplate};
24256
24257 let _provider_env = crate::openrouter::test_environment_scope_async().await;
24258 let _env = ENV_MUTEX.lock().await;
24259 let bearer = "managed-partial-outcome-bearer";
24260
24261 let server = MockServer::start().await;
24262 unsafe {
24263 std::env::set_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV, bearer);
24264 std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
24265 }
24266 Mock::given(method("GET"))
24267 .and(path("/api/v1/organizations/me"))
24268 .and(header("authorization", format!("Bearer {bearer}")))
24269 .respond_with(
24270 ResponseTemplate::new(200)
24271 .set_body_json(serde_json::json!({"organizationId": "org-partial-outcome"})),
24272 )
24273 .mount(&server)
24274 .await;
24275 Mock::given(method("GET"))
24276 .and(path("/connect/session"))
24277 .respond_with(
24278 ResponseTemplate::new(200)
24279 .set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
24280 )
24281 .mount(&server)
24282 .await;
24283 Mock::given(method("POST"))
24284 .and(path("/api/v1/orgs/org-partial-outcome/inference/responses"))
24285 .respond_with(ResponseTemplate::new(200).set_body_raw(
24286 "event: response.output_text.delta\ndata: {\"delta\":\"partial must fail\"}\n\n",
24287 "text/event-stream",
24288 ))
24289 .expect(2)
24290 .mount(&server)
24291 .await;
24292
24293 let tmp = TempDir::new().unwrap();
24294 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24295 let model_id = "parslee/openrouter/frontier-general";
24296 let schema = crate::openrouter::curated_schemas()
24297 .into_iter()
24298 .find(|schema| schema.id == model_id)
24299 .unwrap();
24300 engine.unified_registry_mut().register_project_model(schema);
24301
24302 let buffered_error = engine
24303 .generate_tracked(GenerateRequest {
24304 prompt: "buffered".into(),
24305 model: Some(model_id.into()),
24306 params: GenerateParams {
24307 strict_model: true,
24308 ..Default::default()
24309 },
24310 ..Default::default()
24311 })
24312 .await
24313 .expect_err("buffered partial EOF must fail");
24314 assert!(
24315 buffered_error.to_string().contains("response.completed"),
24316 "unexpected buffered error: {buffered_error}"
24317 );
24318
24319 let mut stream = engine
24320 .generate_tracked_stream(GenerateRequest {
24321 prompt: "streamed".into(),
24322 model: Some(model_id.into()),
24323 params: GenerateParams {
24324 strict_model: true,
24325 ..Default::default()
24326 },
24327 ..Default::default()
24328 })
24329 .await
24330 .expect("HTTP streaming request starts");
24331 let mut events = Vec::new();
24332 while let Some(event) = stream.events.recv().await {
24333 events.push(event);
24334 }
24335 assert!(
24336 matches!(events.last(), Some(StreamEvent::Error(message)) if message.contains("response.completed"))
24337 );
24338 assert!(!events
24339 .iter()
24340 .any(|event| matches!(event, StreamEvent::Done { .. })));
24341
24342 for _ in 0..50 {
24343 if engine
24344 .outcome_tracker()
24345 .read()
24346 .await
24347 .profile(model_id)
24348 .is_some_and(|profile| profile.fail_count == 2)
24349 {
24350 break;
24351 }
24352 tokio::task::yield_now().await;
24353 }
24354 let profile = engine
24355 .outcome_tracker()
24356 .read()
24357 .await
24358 .profile(model_id)
24359 .cloned()
24360 .unwrap();
24361 assert_eq!(profile.fail_count, 2);
24362 assert_eq!(profile.success_count, 0);
24363
24364 unsafe {
24365 std::env::remove_var(crate::remote::PARSLEE_ACCESS_TOKEN_ENV);
24366 std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
24367 }
24368 }
24369
24370 #[tokio::test]
24371 async fn tokenize_rejects_known_remote_model_with_unsupported_mode() {
24372 let tmp = TempDir::new().unwrap();
24377 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24378 let remote_id = engine
24379 .list_schemas()
24380 .into_iter()
24381 .find(|s| !s.is_local())
24382 .map(|s| s.id)
24383 .expect("built-in catalog should include at least one remote model schema");
24384
24385 let err = engine
24386 .tokenize(&remote_id, "hello")
24387 .await
24388 .expect_err("remote tokenize must error");
24389 match err {
24390 InferenceError::UnsupportedMode { mode, backend, .. } => {
24391 assert_eq!(mode, "tokenize/detokenize");
24392 assert_eq!(backend, "remote");
24393 }
24394 other => panic!("expected UnsupportedMode, got {other:?}"),
24395 }
24396
24397 let err = engine
24398 .detokenize(&remote_id, &[1, 2, 3])
24399 .await
24400 .expect_err("remote detokenize must error");
24401 assert!(
24402 matches!(err, InferenceError::UnsupportedMode { .. }),
24403 "expected UnsupportedMode, got {err:?}"
24404 );
24405 }
24406
24407 #[test]
24408 fn unsupported_mode_does_not_trip_circuit_breaker() {
24409 let unsupported = InferenceError::UnsupportedMode {
24415 mode: "structured-output-json-schema",
24416 backend: "anthropic",
24417 reason: "not wired under the pinned API version",
24418 };
24419 assert!(!error_counts_against_circuit_breaker(&unsupported));
24420 assert!(error_counts_against_circuit_breaker(
24421 &InferenceError::InferenceFailed("API returned 500".into())
24422 ));
24423 assert!(error_counts_against_circuit_breaker(
24424 &InferenceError::InferenceFailed("API returned 429".into())
24425 ));
24426 }
24427
24428 #[test]
24436 fn a_content_refusal_does_not_trip_the_circuit_breaker() {
24437 let refused = InferenceError::ContentRefused {
24438 provider: "parslee".into(),
24439 kind: Some("invalid_request_error".into()),
24440 code: Some("content_policy_violation".into()),
24441 message: "content refused".into(),
24442 };
24443 assert!(!error_counts_against_circuit_breaker(&refused));
24444
24445 let rendered = refused.to_string();
24448 assert!(rendered.contains("content grounds"), "{rendered}");
24449 assert!(rendered.contains("content_policy_violation"), "{rendered}");
24450
24451 assert!(error_counts_against_circuit_breaker(
24453 &InferenceError::InferenceFailed("managed inference failed".into())
24454 ));
24455 }
24456
24457 #[test]
24467 fn a_content_refusal_ends_the_fallback_chain() {
24468 let refused = InferenceError::ContentRefused {
24469 provider: "parslee".into(),
24470 kind: Some("invalid_request_error".into()),
24471 code: Some("content_policy_violation".into()),
24472 message: "content refused".into(),
24473 };
24474 assert!(error_ends_fallback_chain(&refused));
24475
24476 for still_advances in [
24480 InferenceError::InferenceFailed("managed inference failed".into()),
24481 InferenceError::InferenceFailed("API returned 503".into()),
24482 InferenceError::ModelNotFound("parslee/reasoning".into()),
24483 InferenceError::UnsupportedMode {
24484 mode: "json_schema",
24485 backend: "anthropic",
24486 reason: "structured output is not supported by this protocol",
24487 },
24488 InferenceError::GatewayUnconfigured {
24489 provider: "parslee".into(),
24490 namespace: "parslee/openrouter/".into(),
24491 status: 503,
24492 message: "not configured".into(),
24493 },
24494 ] {
24495 assert!(
24496 !error_ends_fallback_chain(&still_advances),
24497 "must keep advancing the chain: {still_advances}"
24498 );
24499 }
24500 }
24501
24502 #[test]
24511 fn recovery_hints_do_not_rewrite_a_content_refusal() {
24512 let refused = InferenceError::ContentRefused {
24513 provider: "parslee".into(),
24514 kind: Some("invalid_request_error".into()),
24515 code: Some("content_policy_violation".into()),
24516 message: "blocked: 403 forbidden by the content filter".into(),
24518 };
24519 let out = apply_exhaustion_recovery_hint(refused);
24520 assert!(
24521 matches!(out, InferenceError::ContentRefused { .. }),
24522 "{out:?}"
24523 );
24524
24525 let signed_out = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
24527 "no credential for proprietary provider 'parslee'".into(),
24528 ));
24529 assert!(
24530 matches!(signed_out, InferenceError::InferenceFailed(ref m) if m.contains("car auth")),
24531 "{signed_out:?}"
24532 );
24533
24534 let transient = apply_exhaustion_recovery_hint(InferenceError::InferenceFailed(
24536 "API returned 500".into(),
24537 ));
24538 assert_eq!(transient.to_string(), "inference failed: API returned 500");
24539 }
24540
24541 #[test]
24549 fn unconfigured_gateway_does_not_trip_circuit_breaker() {
24550 let unconfigured = InferenceError::GatewayUnconfigured {
24551 provider: "parslee".into(),
24552 namespace: "parslee/openrouter/".into(),
24553 status: 503,
24554 message: "OpenRouter inference is not configured on this Parslee environment.".into(),
24555 };
24556 assert!(!error_counts_against_circuit_breaker(&unconfigured));
24557 let rendered = unconfigured.to_string();
24561 assert!(rendered.contains("parslee/openrouter/"), "{rendered}");
24562 assert!(rendered.contains("not configured"), "{rendered}");
24563 }
24564
24565 #[test]
24571 fn namespace_drop_spares_siblings_outside_the_prefix() {
24572 let namespace = "parslee/openrouter/";
24573 let mut queue: std::collections::VecDeque<String> = [
24574 "parslee/openrouter/open-fast",
24575 "parslee/reasoning",
24576 "parslee/openrouter/frontier-general",
24577 "parslee/advisor",
24578 "anthropic/claude-opus-4-8:latest",
24579 ]
24580 .into_iter()
24581 .map(String::from)
24582 .collect();
24583
24584 queue.retain(|id| !id.starts_with(namespace));
24585
24586 assert_eq!(
24587 queue.iter().collect::<Vec<_>>(),
24588 vec![
24589 "parslee/reasoning",
24590 "parslee/advisor",
24591 "anthropic/claude-opus-4-8:latest"
24592 ],
24593 "only the unconfigured namespace may be dropped"
24594 );
24595 }
24596
24597 #[test]
24598 fn engine_loads_benchmark_priors_on_startup() {
24599 let _env = ENV_MUTEX.blocking_lock();
24600 let tmp = TempDir::new().unwrap();
24601 let priors_path = tmp.path().join("benchmark_priors.json");
24602 std::fs::write(
24603 &priors_path,
24604 serde_json::json!({
24605 "model_id": "qwen/qwen3-8b:q4_k_m",
24606 "overall_score": 0.88
24607 })
24608 .to_string(),
24609 )
24610 .unwrap();
24611
24612 unsafe {
24613 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
24614 }
24615
24616 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24617 let tracker = engine.outcome_tracker.blocking_read();
24618 let profile = tracker
24619 .profile("qwen/qwen3-8b:q4_k_m")
24620 .expect("benchmark prior should create a profile");
24621 assert!((profile.ema_quality - 0.88).abs() < 0.01);
24622
24623 unsafe {
24624 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
24625 }
24626 }
24627
24628 #[test]
24629 fn benchmark_priors_do_not_override_observed_profiles() {
24630 let _env = ENV_MUTEX.blocking_lock();
24631 let tmp = TempDir::new().unwrap();
24632 let models_dir = tmp.path().join("models");
24633 std::fs::create_dir_all(&models_dir).unwrap();
24634
24635 let observed = vec![ModelProfile {
24636 model_id: "qwen/qwen3-8b:q4_k_m".into(),
24637 total_calls: 12,
24638 success_count: 3,
24639 fail_count: 9,
24640 total_latency_ms: 1200,
24641 total_input_tokens: 0,
24642 total_output_tokens: 0,
24643 total_cache_read_input_tokens: 0,
24644 total_cache_creation_input_tokens: 0,
24645 task_stats: std::collections::HashMap::new(),
24646 ema_quality: 0.21,
24647 prior_sample_size: 0,
24648 quality_observations: 0,
24649 quality_per_1k_tokens: 0.0,
24650 updated_at: 1,
24651 }];
24652 std::fs::write(
24653 models_dir.join("outcome_profiles.json"),
24654 serde_json::to_string(&observed).unwrap(),
24655 )
24656 .unwrap();
24657
24658 let priors_path = tmp.path().join("benchmark_priors.json");
24659 std::fs::write(
24660 &priors_path,
24661 serde_json::json!({
24662 "model_id": "qwen/qwen3-8b:q4_k_m",
24663 "overall_score": 0.95
24664 })
24665 .to_string(),
24666 )
24667 .unwrap();
24668
24669 unsafe {
24670 std::env::set_var("CAR_BENCHMARK_PRIORS_PATH", &priors_path);
24671 }
24672
24673 let engine = InferenceEngine::new(test_config(models_dir));
24674 let tracker = engine.outcome_tracker.blocking_read();
24675 let profile = tracker
24676 .profile("qwen/qwen3-8b:q4_k_m")
24677 .expect("observed profile should remain present");
24678 assert!((profile.ema_quality - 0.21).abs() < 0.01);
24679 assert_eq!(profile.total_calls, 12);
24680
24681 unsafe {
24682 std::env::remove_var("CAR_BENCHMARK_PRIORS_PATH");
24683 }
24684 }
24685
24686 #[test]
24687 fn speech_runtime_package_spec_defaults_and_overrides() {
24688 let _env = ENV_MUTEX.blocking_lock();
24689 unsafe {
24690 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
24691 }
24692 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.2");
24693
24694 unsafe {
24695 std::env::set_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC", "mlx-audio==0.4.1");
24696 }
24697 assert_eq!(speech_runtime_mlx_audio_spec(), "mlx-audio==0.4.1");
24698
24699 unsafe {
24700 std::env::remove_var("CAR_SPEECH_RUNTIME_MLX_AUDIO_SPEC");
24701 }
24702 }
24703
24704 #[test]
24705 fn speech_runtime_spacy_model_spec_defaults_and_overrides() {
24706 let _env = ENV_MUTEX.blocking_lock();
24707 unsafe {
24708 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
24709 }
24710 assert!(
24711 speech_runtime_spacy_model_spec().starts_with("en-core-web-sm @ https://github.com/")
24712 );
24713
24714 unsafe {
24715 std::env::set_var(
24716 "CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC",
24717 "en-core-web-sm==3.8.0",
24718 );
24719 }
24720 assert_eq!(speech_runtime_spacy_model_spec(), "en-core-web-sm==3.8.0");
24721
24722 unsafe {
24723 std::env::remove_var("CAR_SPEECH_RUNTIME_SPACY_MODEL_SPEC");
24724 }
24725 }
24726
24727 #[test]
24728 fn kokoro_runtime_fallback_defaults_on() {
24729 unsafe {
24730 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
24731 }
24732 assert!(kokoro_runtime_fallback_enabled());
24733
24734 unsafe {
24735 std::env::set_var("CAR_SPEECH_KOKORO_FALLBACK", "false");
24736 }
24737 assert!(!kokoro_runtime_fallback_enabled());
24738
24739 unsafe {
24740 std::env::remove_var("CAR_SPEECH_KOKORO_FALLBACK");
24741 }
24742 }
24743
24744 #[test]
24745 fn preferred_local_tts_wins_over_builtin_rank() {
24746 let tmp = TempDir::new().unwrap();
24747 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24748 engine.set_speech_policy(SpeechPolicy {
24749 prefer_local: true,
24750 allow_remote_fallback: false,
24751 preferred_local_stt: None,
24752 preferred_local_tts: Some("Kokoro-82M-6bit".into()),
24753 preferred_remote_stt: None,
24754 preferred_remote_tts: None,
24755 });
24756
24757 let schema = engine
24758 .preferred_speech_schema(ModelCapability::TextToSpeech, true, false)
24759 .expect("preferred local TTS should resolve");
24760 #[cfg(not(target_os = "windows"))]
24767 assert_eq!(schema.name, "Kokoro-82M-6bit");
24768 #[cfg(target_os = "windows")]
24769 assert_eq!(schema.name, "Windows-Speech");
24770 }
24771
24772 #[test]
24773 fn preferred_discovered_vllm_mlx_model_wins_generate_routing() {
24774 let tmp = TempDir::new().unwrap();
24775 let mut config = test_config(tmp.path().join("models"));
24776 config.preferred_generation_model =
24777 Some("vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit".into());
24778 let mut engine = InferenceEngine::new(config);
24779 let schema = crate::vllm_mlx::to_model_schema(
24780 &crate::vllm_mlx::DiscoveredModel {
24781 id: "mlx-community/gemma-3n-E2B-it-lm-4bit".into(),
24782 owned_by: Some("mlx-community".into()),
24783 },
24784 "http://127.0.0.1:8001",
24785 );
24786 engine.register_model(schema);
24787
24788 let rt = tokio::runtime::Runtime::new().unwrap();
24789 let decision = rt.block_on(engine.route_adaptive("say hello in one sentence"));
24790 assert_eq!(
24791 decision.model_id,
24792 "vllm-mlx/mlx-community_gemma-3n-E2B-it-lm-4bit"
24793 );
24794 assert_eq!(decision.strategy, RoutingStrategy::Explicit);
24795 assert_eq!(decision.reason, "preferred generation model override");
24796 }
24797
24798 #[test]
24807 fn tools_request_on_non_tool_model_terminates_not_spins() {
24808 let tmp = TempDir::new().unwrap();
24809 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24810 let schema = crate::vllm_mlx::to_model_schema(
24812 &crate::vllm_mlx::DiscoveredModel {
24813 id: "test-org/embed-only-model".into(),
24814 owned_by: None,
24815 },
24816 "http://127.0.0.1:1", );
24818 let model_id = schema.id.clone();
24819 engine.register_model(schema);
24820
24821 let mut req = GenerateRequest {
24822 prompt: "call a tool".to_string(),
24823 model: Some(model_id),
24824 tools: Some(vec![serde_json::json!({
24825 "name": "noop", "description": "n", "parameters": {"type": "object"}
24826 })]),
24827 ..Default::default()
24828 };
24829 req.params.strict_model = true;
24830
24831 let rt = tokio::runtime::Runtime::new().unwrap();
24832 let out = rt.block_on(async {
24833 tokio::time::timeout(
24834 std::time::Duration::from_secs(10),
24835 engine.generate_tracked(req),
24836 )
24837 .await
24838 });
24839 let completed = out.expect("fallback loop must terminate, not spin");
24842 assert!(completed.is_err());
24843 }
24844
24845 fn fake_ready_speech_runtime(root: &Path) {
24856 managed_venv::seed_ready_venv(root);
24857 let runtime = SpeechRuntime::new(root.to_path_buf());
24858 for program in [&runtime.stt_program, &runtime.tts_program] {
24859 std::fs::create_dir_all(program.parent().expect("program has a parent")).unwrap();
24860 std::fs::write(program, b"").unwrap();
24861 }
24862 }
24863
24864 #[tokio::test]
24891 async fn prepare_speech_runtime_returns_the_root_doctor_reports() {
24892 let _env = ENV_MUTEX.lock().await;
24893 let tmp = TempDir::new().unwrap();
24894 let runtime_root = tmp.path().join("speech-runtime");
24895 fake_ready_speech_runtime(&runtime_root);
24896 unsafe {
24897 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
24898 }
24899
24900 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24901 let root = engine
24902 .prepare_speech_runtime()
24903 .await
24904 .expect("prepare should succeed against a ready runtime");
24905 let health = engine.speech_health();
24906
24907 assert_eq!(
24908 root, health.runtime.root,
24909 "prepare returned a different root than doctor reports"
24910 );
24911 assert!(
24912 root.exists(),
24913 "prepare returned {} but it does not exist",
24914 root.display()
24915 );
24916 assert!(
24917 health.runtime.installed,
24918 "doctor should report a ready runtime as installed"
24919 );
24920 assert_eq!(
24922 engine
24923 .prepare_speech_runtime()
24924 .await
24925 .expect("second prepare should succeed"),
24926 root
24927 );
24928
24929 unsafe {
24930 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
24931 }
24932 }
24933
24934 #[tokio::test]
24948 async fn prepare_speech_runtime_fails_when_bootstrap_fails() {
24949 let _env = ENV_MUTEX.lock().await;
24950 let tmp = TempDir::new().unwrap();
24951 let runtime_root = tmp.path().join("speech-runtime");
24952 assert!(!runtime_root.exists(), "precondition: root absent");
24953 unsafe {
24954 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", &runtime_root);
24955 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
24956 }
24957
24958 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
24959 let err = engine.prepare_speech_runtime().await.err();
24960 let health = engine.speech_health();
24961
24962 assert!(
24963 err.is_some(),
24964 "a failed bootstrap must fail the install, not return a root"
24965 );
24966 assert!(
24967 !health.runtime.installed,
24968 "doctor must not claim a runtime the bootstrap never built"
24969 );
24970
24971 unsafe {
24972 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
24973 std::env::remove_var("CAR_SPEECH_PYTHON");
24974 }
24975 }
24976
24977 #[cfg(target_os = "linux")]
24990 #[tokio::test]
24991 async fn a_runtime_that_cannot_be_built_no_longer_blocks_the_whisper_install() {
24992 let _env = ENV_MUTEX.lock().await;
24993 let tmp = TempDir::new().unwrap();
24994
24995 let cached = tmp
24999 .path()
25000 .join(".tokhn")
25001 .join("whisper")
25002 .join("ggml-large-v3-turbo-q5_0.bin");
25003 std::fs::create_dir_all(cached.parent().unwrap()).unwrap();
25004 std::fs::write(&cached, b"stand-in for the ggml weights").unwrap();
25005
25006 let previous_home = std::env::var_os("HOME");
25007 unsafe {
25008 std::env::set_var("HOME", tmp.path());
25009 std::env::set_var("CAR_SPEECH_RUNTIME_DIR", tmp.path().join("speech-runtime"));
25010 std::env::set_var("CAR_SPEECH_PYTHON", tmp.path().join("no-such-python"));
25014 }
25015
25016 let mut engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25017 let installed = engine.install_curated_speech().await;
25018
25019 let restore = || unsafe {
25020 match &previous_home {
25021 Some(value) => std::env::set_var("HOME", value),
25022 None => std::env::remove_var("HOME"),
25023 }
25024 std::env::remove_var("CAR_SPEECH_RUNTIME_DIR");
25025 std::env::remove_var("CAR_SPEECH_PYTHON");
25026 };
25027 let installed = match installed {
25028 Ok(installed) => installed,
25029 Err(error) => {
25030 restore();
25031 panic!("a runtime that cannot be built must not abort the install: {error}");
25032 }
25033 };
25034 let runtime_installed = engine.speech_health().runtime.installed;
25035 restore();
25036
25037 assert!(
25038 !runtime_installed,
25039 "precondition: the bootstrap must actually have failed, or this \
25040 test would pass without exercising anything"
25041 );
25042 let whisper = installed
25043 .iter()
25044 .find(|report| report.hf_repo == "ggerganov/whisper.cpp")
25045 .unwrap_or_else(|| {
25046 panic!(
25047 "the cross-platform whisper model must still be installed, got: {installed:?}"
25048 )
25049 });
25050 assert_eq!(whisper.snapshot_path, cached);
25051 assert!(
25052 installed
25053 .iter()
25054 .all(|report| report.hf_repo == "ggerganov/whisper.cpp"),
25055 "MLX weights are Apple-only and must not be pulled here — over a \
25056 gigabyte of them, for models this host can never load; got: {installed:?}"
25057 );
25058 }
25059
25060 #[test]
25068 fn alias_resolves_to_canonical_id_for_outcome_keying() {
25069 let tmp = TempDir::new().unwrap();
25070 let engine = InferenceEngine::new(test_config(tmp.path().join("models")));
25071
25072 let cases = [
25074 ("claude-sonnet-4-6", "anthropic/claude-sonnet-4-6:latest"),
25075 ("gpt-5.4", "openai/gpt-5.4:latest"),
25076 ("gemini-2.5-flash", "google/gemini-2.5-flash:latest"),
25077 ];
25078 for (alias, canonical) in cases {
25079 let resolved = engine
25082 .resolve_schema(alias)
25083 .map(|s| s.id)
25084 .unwrap_or_else(|| alias.to_string());
25085 assert_eq!(
25086 resolved, canonical,
25087 "alias `{alias}` must resolve to canonical `{canonical}` for outcome keying, got `{resolved}`"
25088 );
25089 assert_ne!(
25090 resolved, alias,
25091 "alias `{alias}` must NOT be recorded raw — that is the fragmentation bug"
25092 );
25093 }
25094 }
25095
25096 #[test]
25101 fn inference_result_serializes_with_full_shape() {
25102 use crate::tasks::generate::ToolCall;
25103 use std::collections::HashMap;
25104
25105 let mut args = HashMap::new();
25106 args.insert("path".to_string(), serde_json::json!("README.md"));
25107
25108 let result = InferenceResult {
25109 text: String::new(),
25110 bounding_boxes: Vec::new(),
25111 tool_calls: vec![ToolCall {
25112 id: None,
25113 name: "read_file".into(),
25114 arguments: args,
25115 }],
25116 trace_id: "trace-abc".into(),
25117 model_used: "test-model".into(),
25118 model_identity: InferenceModelIdentity {
25119 requested_model_id: Some("test/model:1".into()),
25120 resolved_model_id: "test/model:1".into(),
25121 row_digest: "a".repeat(64),
25122 catalog_revision: "b".repeat(64),
25123 },
25124 latency_ms: 1234,
25125 time_to_first_token_ms: Some(180),
25126 usage: Some(TokenUsage {
25127 prompt_tokens: 100,
25128 completion_tokens: 50,
25129 total_tokens: 150,
25130 context_window: 8192,
25131 ..Default::default()
25132 }),
25133 provider_output_items: Vec::new(),
25134 thinking: Vec::new(),
25135 stop_reason: Some("tool_use".into()),
25136 auth_fallback_from: None,
25137 local_last_resort: false,
25138 fallback_from: Vec::new(),
25139 };
25140
25141 let json = serde_json::to_value(&result).expect("serialize");
25142
25143 assert_eq!(json["stop_reason"].as_str(), Some("tool_use"));
25145
25146 assert_eq!(json["text"].as_str(), Some(""));
25148 assert_eq!(json["trace_id"].as_str(), Some("trace-abc"));
25149 assert_eq!(json["model_used"].as_str(), Some("test-model"));
25150 assert_eq!(json["requested_model_id"].as_str(), Some("test/model:1"));
25151 assert_eq!(json["resolved_model_id"].as_str(), Some("test/model:1"));
25152 assert_eq!(json["row_digest"].as_str().unwrap(), "a".repeat(64));
25153 assert_eq!(json["catalog_revision"].as_str().unwrap(), "b".repeat(64));
25154 assert_eq!(json["latency_ms"].as_u64(), Some(1234));
25155
25156 let tool_calls = json["tool_calls"].as_array().expect("tool_calls array");
25158 assert_eq!(tool_calls.len(), 1);
25159 assert_eq!(tool_calls[0]["name"].as_str(), Some("read_file"));
25160 assert_eq!(
25161 tool_calls[0]["arguments"]["path"].as_str(),
25162 Some("README.md")
25163 );
25164
25165 let usage = &json["usage"];
25167 assert_eq!(usage["prompt_tokens"].as_u64(), Some(100));
25168 assert_eq!(usage["completion_tokens"].as_u64(), Some(50));
25169 assert_eq!(usage["total_tokens"].as_u64(), Some(150));
25170 assert_eq!(usage["context_window"].as_u64(), Some(8192));
25171
25172 assert_eq!(json["time_to_first_token_ms"].as_u64(), Some(180));
25174 }
25175
25176 #[test]
25182 fn inference_result_top_level_keys_are_locked() {
25183 use std::collections::BTreeSet;
25184
25185 let result = InferenceResult {
25186 text: "anything".into(),
25187 bounding_boxes: Vec::new(),
25188 tool_calls: vec![],
25189 trace_id: "t".into(),
25190 model_used: "m".into(),
25191 model_identity: InferenceModelIdentity::default(),
25192 latency_ms: 0,
25193 time_to_first_token_ms: None,
25194 usage: None,
25195 provider_output_items: Vec::new(),
25196 thinking: Vec::new(),
25197 stop_reason: None,
25198 auth_fallback_from: None,
25199 local_last_resort: false,
25200 fallback_from: Vec::new(),
25201 };
25202
25203 let json = serde_json::to_value(&result).expect("serialize");
25204 let keys: BTreeSet<&str> = json
25205 .as_object()
25206 .expect("top-level object")
25207 .keys()
25208 .map(String::as_str)
25209 .collect();
25210
25211 let expected: BTreeSet<&str> = [
25212 "text",
25213 "tool_calls",
25214 "trace_id",
25215 "model_used",
25216 "requested_model_id",
25217 "resolved_model_id",
25218 "row_digest",
25219 "catalog_revision",
25220 "latency_ms",
25221 "time_to_first_token_ms",
25222 "usage",
25223 "stop_reason",
25224 ]
25225 .into_iter()
25226 .collect();
25227
25228 assert_eq!(
25229 keys, expected,
25230 "infer response top-level keys drifted -- update both the test \
25231 and the WebSocket protocol documentation if this is intentional"
25232 );
25233
25234 for key in &keys {
25236 assert!(
25237 !key.chars().any(|c| c.is_uppercase()) && !key.contains('-'),
25238 "key '{}' is not snake_case",
25239 key
25240 );
25241 }
25242 }
25243
25244 #[test]
25248 fn inference_result_serializes_plain_text_response() {
25249 let result = InferenceResult {
25250 text: "hello world".into(),
25251 bounding_boxes: Vec::new(),
25252 tool_calls: vec![],
25253 trace_id: "trace-xyz".into(),
25254 model_used: "test-model".into(),
25255 model_identity: InferenceModelIdentity::default(),
25256 latency_ms: 42,
25257 time_to_first_token_ms: None,
25258 usage: None,
25259 provider_output_items: Vec::new(),
25260 thinking: Vec::new(),
25261 stop_reason: None,
25262 auth_fallback_from: None,
25263 local_last_resort: false,
25264 fallback_from: Vec::new(),
25265 };
25266
25267 let json = serde_json::to_value(&result).expect("serialize");
25268 assert_eq!(json["text"], "hello world");
25269 assert!(json["stop_reason"].is_null());
25271 assert!(json["tool_calls"].is_array());
25272 assert_eq!(json["tool_calls"].as_array().unwrap().len(), 0);
25273 assert_eq!(json["model_used"], "test-model");
25274 assert!(json["usage"].is_null());
25275 assert!(json["time_to_first_token_ms"].is_null());
25278 }
25279
25280 #[test]
25281 fn append_assistant_history_preserves_responses_items_in_provider_order() {
25282 let reasoning = serde_json::json!({
25283 "type": "reasoning",
25284 "id": "rs_history",
25285 "status": "completed",
25286 "summary": [{"type": "summary_text", "text": "safe"}],
25287 "encrypted_content": "opaque",
25288 });
25289 let result: InferenceResult = serde_json::from_value(serde_json::json!({
25290 "text": "calling",
25291 "tool_calls": [{
25292 "id": "call_1",
25293 "name": "read_file",
25294 "arguments": {"path": "README.md"}
25295 }],
25296 "trace_id": "trace",
25297 "model_used": "gateway-alias",
25298 "resolved_model_id": "openrouter/anthropic/claude-sonnet-4.5",
25299 "local_last_resort": true,
25300 "latency_ms": 1,
25301 "provider_output_items": [reasoning.clone()],
25302 }))
25303 .unwrap();
25304 let mut history = vec![crate::tasks::generate::Message::User {
25305 content: "inspect".into(),
25306 }];
25307
25308 result.append_assistant_history(&mut history, result.tool_calls.clone());
25309
25310 assert!(matches!(
25311 &history[1],
25312 crate::tasks::generate::Message::ProviderOutputItems { protocol, items }
25313 if protocol == crate::protocol::OPENAI_RESPONSES_PROTOCOL
25314 && items == &vec![reasoning]
25315 ));
25316 assert!(matches!(
25317 &history[2],
25318 crate::tasks::generate::Message::Assistant {
25319 content,
25320 tool_calls,
25321 model_id,
25322 local_last_resort,
25323 ..
25324 } if content == "calling"
25325 && tool_calls[0].id.as_deref() == Some("call_1")
25326 && model_id.as_deref() == Some("openrouter/anthropic/claude-sonnet-4.5")
25327 && *local_last_resort
25328 ));
25329 }
25330
25331 #[test]
25332 fn append_assistant_history_leaves_personal_chat_history_unchanged() {
25333 let result: InferenceResult = serde_json::from_value(serde_json::json!({
25334 "text": "plain",
25335 "tool_calls": [],
25336 "trace_id": "trace",
25337 "model_used": "openrouter/openai/gpt-4.1-mini",
25338 "latency_ms": 1,
25339 }))
25340 .unwrap();
25341 let mut history = Vec::new();
25342
25343 result.append_assistant_history(&mut history, Vec::new());
25344
25345 assert_eq!(history.len(), 1);
25346 assert!(matches!(
25347 &history[0],
25348 crate::tasks::generate::Message::Assistant { content, .. } if content == "plain"
25349 ));
25350 }
25351
25352 #[test]
25364 fn generate_request_deserializes_intent_field_from_json_rpc_params() {
25365 use crate::intent::TaskHint;
25366 use crate::schema::ModelCapability;
25367
25368 let params = serde_json::json!({
25371 "prompt": "summarize this email",
25372 "intent": {
25373 "task": "chat",
25374 "prefer_local": true,
25375 "require": ["tool_use"],
25376 },
25377 });
25378
25379 let req: GenerateRequest =
25380 serde_json::from_value(params).expect("GenerateRequest deserialize");
25381
25382 let intent = req.intent.as_ref().expect("intent field deserialized");
25383 assert_eq!(intent.task, Some(TaskHint::Chat));
25384 assert!(intent.prefer_local);
25385 assert_eq!(intent.require, vec![ModelCapability::ToolUse]);
25386
25387 let back: serde_json::Value =
25391 serde_json::to_value(&req).expect("re-serialize GenerateRequest");
25392 assert_eq!(back["intent"]["task"], "chat");
25393 assert_eq!(back["intent"]["prefer_local"], true);
25394 assert_eq!(back["intent"]["require"][0], "tool_use");
25395
25396 let default_req: GenerateRequest = serde_json::from_value(serde_json::json!({
25401 "prompt": "x",
25402 "intent": {},
25403 }))
25404 .unwrap();
25405 let default_intent = default_req.intent.expect("present but empty");
25406 assert_eq!(default_intent.task, None);
25407 assert!(!default_intent.prefer_local);
25408 assert!(default_intent.require.is_empty());
25409
25410 let no_intent: GenerateRequest =
25413 serde_json::from_value(serde_json::json!({"prompt": "x"})).unwrap();
25414 assert!(no_intent.intent.is_none());
25415 }
25416
25417 #[test]
25418 fn rerank_prompt_matches_upstream_template_shape() {
25419 let p = rerank_prompt(
25420 "retrieve relevant passages",
25421 "who runs the treasury?",
25422 "doc x",
25423 );
25424 assert!(p.contains("<|im_start|>system"));
25425 assert!(p.contains("Note that the answer can only be \"yes\" or \"no\"."));
25426 assert!(p.contains("<|im_start|>user\n<Instruct>: retrieve relevant passages"));
25427 assert!(p.contains("<Query>: who runs the treasury?"));
25428 assert!(p.contains("<Document>: doc x<|im_end|>"));
25429 assert!(p.contains("<|im_start|>assistant\n<think>\n\n</think>\n\n"));
25430 }
25431
25432 #[test]
25433 fn rerank_score_yes_and_no_exactly() {
25434 assert_eq!(score_from_rerank_output("yes", "m"), 1.0);
25435 assert_eq!(score_from_rerank_output("no", "m"), 0.0);
25436 }
25437
25438 #[test]
25439 fn rerank_score_handles_case_leading_space_and_chat_sentinels() {
25440 assert_eq!(score_from_rerank_output(" Yes", "m"), 1.0);
25443 assert_eq!(score_from_rerank_output("\nno.", "m"), 0.0);
25444 assert_eq!(score_from_rerank_output("<|im_end|>yes", "m"), 1.0);
25445 }
25446
25447 #[test]
25448 fn rerank_score_scans_up_to_three_tokens() {
25449 assert_eq!(score_from_rerank_output("_bos_ yes", "m"), 1.0);
25452 }
25453
25454 #[test]
25455 fn rerank_score_unexpected_is_neutral() {
25456 assert_eq!(score_from_rerank_output("maybe", "m"), 0.5);
25459 assert_eq!(score_from_rerank_output("", "m"), 0.5);
25460 }
25461
25462 #[tokio::test]
25463 async fn pull_reuses_a_valid_directory_receipt_and_reports_removability() {
25464 let root = tempfile::tempdir().unwrap();
25465 let models_dir = root.path().join("models");
25466 std::fs::create_dir_all(&models_dir).unwrap();
25467 let engine = InferenceEngine::new(InferenceConfig {
25468 state_root: root.path().join("state"),
25469 models_dir: models_dir.clone(),
25470 ..InferenceConfig::default()
25471 });
25472 let schema = engine
25473 .unified_registry()
25474 .all()
25475 .find(|schema| matches!(schema.source, ModelSource::Local { .. }))
25476 .unwrap()
25477 .clone();
25478 let managed = models_dir.join(&schema.name);
25479 std::fs::create_dir_all(&managed).unwrap();
25480 std::fs::write(managed.join("model.gguf"), b"owned").unwrap();
25481 std::fs::write(managed.join("tokenizer.json"), b"{}").unwrap();
25482 engine
25483 .model_management
25484 .record_managed_artifact(
25485 &schema.id,
25486 model_source_identity(&schema),
25487 None,
25488 1,
25489 false,
25490 managed.clone(),
25491 )
25492 .unwrap();
25493 let row = engine
25494 .list_models_unified()
25495 .into_iter()
25496 .find(|row| row.id == schema.id)
25497 .unwrap();
25498 if model_management::directory_removal_supported() {
25499 assert!(engine.model_management.can_remove(&schema.id).unwrap());
25500 assert!(row.can_remove);
25501 assert_eq!(row.management_evidence.as_deref(), Some("install_receipt"));
25502 } else {
25503 assert!(!engine.model_management.can_remove(&schema.id).unwrap());
25504 assert!(!row.can_remove);
25505 assert_eq!(
25506 row.management_evidence.as_deref(),
25507 Some("install_receipt_directory_cleanup_unsupported")
25508 );
25509 }
25510
25511 let reused = engine.pull_model(&schema.id).await.unwrap();
25512 assert_eq!(reused, managed);
25513 }
25514}
25515
25516#[cfg(test)]
25517mod response_format_support_tests {
25518 use super::*;
25519
25520 #[test]
25524 fn rejection_reason_tracks_the_protocol_handler() {
25525 let root = tempfile::tempdir().unwrap();
25528 let engine = InferenceEngine::new(InferenceConfig {
25529 state_root: root.path().join("state"),
25530 models_dir: root.path().join("models"),
25531 ..InferenceConfig::default()
25532 });
25533 let rf = crate::tasks::generate::ResponseFormat::JsonObject;
25534 let models = engine.list_models_unified();
25535 let anthropic = models
25536 .iter()
25537 .find(|m| m.provider.eq_ignore_ascii_case("anthropic"));
25538 if let Some(m) = anthropic {
25539 let reason = engine
25540 .response_format_rejection_reason(&m.id, &rf)
25541 .expect("the Anthropic protocol rejects response_format");
25542 assert!(reason.contains("protocol rejects"), "{reason}");
25543 }
25544 let openrouter = models
25545 .iter()
25546 .find(|m| m.provider.eq_ignore_ascii_case("openrouter"));
25547 if let Some(m) = openrouter {
25548 assert_eq!(
25549 engine.response_format_rejection_reason(&m.id, &rf),
25550 None,
25551 "OpenRouter forwards the format upstream"
25552 );
25553 }
25554 assert!(
25555 anthropic.is_some() || openrouter.is_some(),
25556 "the builtin catalog should list at least one of the two providers this pins"
25557 );
25558 assert_eq!(
25559 engine.response_format_rejection_reason("no/such-model", &rf),
25560 None,
25561 "an unknown model is not a rejection"
25562 );
25563 }
25564}