1use std::collections::HashMap;
9use std::io::Write;
10use std::panic::AssertUnwindSafe;
11use std::path::Path;
12use std::pin::Pin;
13use std::sync::atomic::{AtomicU64, Ordering};
14use std::sync::{Arc, Mutex, PoisonError, RwLock};
15use std::task::{Context, Poll, ready};
16use std::time::{Duration, Instant};
17
18use bytes::Bytes;
19use cpa_common::gostr::GoStr;
20use cpa_core::config::Config;
21use cpa_core::credential::{Credential, MetadataPatch, Source};
22use cpa_core::exec::{ExecError, ExecStream, FailureScope};
23use cpa_exec::{Executors, Readiness};
24use futures_util::{FutureExt, Stream, StreamExt};
25use serde_json::{Map, Value};
26
27use crate::refresh::RefreshState;
28use crate::registry::Registry;
29use crate::scheduler::{Policy, Scheduler};
30
31type RegistryCache = (Arc<Config>, (u64, u64, u64), Arc<Registry>);
33
34pub struct Runtime {
35 config: RwLock<Arc<Config>>,
36 store: Arc<CredentialStore>,
37 pub executors: Executors,
38 refresh_task: Mutex<Option<tokio::task::AbortHandle>>,
39 refresh_state: Mutex<RefreshState>,
40 registry: Mutex<Option<RegistryCache>>,
42 oauth_sink: RwLock<Option<OAuthCallbackSink>>,
43 pool_offsets: Mutex<HashMap<String, usize>>,
45 usage: Arc<crate::usage::UsageQueue>,
47 local_model: std::sync::atomic::AtomicBool,
49 plugins: crate::plugins::PluginRuntime,
51 remote: RwLock<Option<Arc<dyn crate::remote::RemoteDispatch>>>,
53 pub(crate) deny_external: std::sync::atomic::AtomicBool,
56 config_changed: tokio::sync::watch::Sender<()>,
58 refresh_done: tokio::sync::watch::Sender<()>,
61 codex_refreshes: CodexRefreshes,
63 catalogs: Mutex<Option<crate::model_updater::Session>>,
65 config_commit: Mutex<()>,
67}
68
69#[derive(Clone, Default)]
73struct CodexRefreshes {
74 shutdown: tokio_util::sync::CancellationToken,
75 tasks: tokio_util::task::TaskTracker,
76}
77
78#[derive(Debug, Clone)]
80pub struct OAuthCallback {
81 pub provider: &'static str,
83 pub state: String,
84 pub code: String,
85 pub error: String,
86}
87
88#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub enum OAuthDelivery {
91 Delivered,
92 NotPending,
94 RequiresV8,
97}
98
99pub type OAuthCallbackSink = Arc<dyn Fn(&OAuthCallback) -> OAuthDelivery + Send + Sync>;
102
103impl Runtime {
104 pub fn new(config: Config, credentials: Vec<Credential>, executors: Executors) -> Self {
107 let mut policy = crate::management::policy(&config);
108 policy.compat_disable_cooling = crate::scheduler::compat_cooling(&config);
109 let cooldown_dir = cooldown_dir(&config, &policy);
110 let (enabled, strict) = signature_cache_config(&config);
111 cpa_translate::set_antigravity_signature_cache_config(enabled, strict);
112 let rt = Self {
113 config: RwLock::new(Arc::new(config)),
114 store: CredentialStore::new(credentials),
115 executors,
116 refresh_task: Mutex::default(),
117 refresh_state: Mutex::default(),
118 registry: Mutex::default(),
119 oauth_sink: RwLock::default(),
120 pool_offsets: Mutex::default(),
121 usage: Arc::default(),
122 local_model: Default::default(),
123 plugins: Default::default(),
124 remote: RwLock::default(),
125 deny_external: Default::default(),
126 config_changed: tokio::sync::watch::Sender::new(()),
127 refresh_done: tokio::sync::watch::Sender::new(()),
128 codex_refreshes: CodexRefreshes::default(),
129 catalogs: Mutex::default(),
130 config_commit: Mutex::default(),
131 };
132 rt.publish_policy(policy);
133 rt.store.configure_cooldown_store(cooldown_dir);
134 let _ = rt.store.error_events.set(rt.usage.clone());
136 let _ = rt
137 .store
138 .session_closer
139 .set(Box::new(rt.executors.credential_session_closer()));
140 rt
141 }
142
143 pub fn usage_queue(&self) -> &crate::usage::UsageQueue {
146 &self.usage
147 }
148
149 pub fn set_local_model(&self, on: bool) {
152 self.local_model.store(on, std::sync::atomic::Ordering::Relaxed);
153 }
154
155 pub fn set_cooldown_backend(&self, backend: Arc<dyn crate::cooldown_store::Backend>) {
158 self.store.set_cooldown_backend(backend);
159 }
160
161 pub fn local_model(&self) -> bool {
163 self.local_model.load(std::sync::atomic::Ordering::Relaxed)
164 }
165
166 pub fn with_plugin_host(mut self, host: cpa_plugin::Host) -> Self {
170 self.plugins = crate::plugins::PluginRuntime::with_host(host);
171 self
172 }
173
174 pub fn plugins(&self) -> &cpa_plugin::Host {
177 self.plugins.host()
178 }
179
180 pub(crate) fn plugin_runtime(&self) -> &crate::plugins::PluginRuntime {
181 &self.plugins
182 }
183
184 pub fn config(&self) -> Arc<Config> {
186 self.config.read().unwrap_or_else(PoisonError::into_inner).clone()
187 }
188
189 pub(crate) fn start_catalogs(&self, session: crate::model_updater::Session) {
192 let commit = self.config_commit.lock().unwrap_or_else(PoisonError::into_inner);
193 session.configure(&self.config(), true);
194 let previous = self
195 .catalogs
196 .lock()
197 .unwrap_or_else(PoisonError::into_inner)
198 .replace(session);
199 drop(commit);
200 drop(previous);
201 }
202
203 pub(crate) fn accepts_config(config: &Config) -> bool {
207 match cpa_core::config::validate_catalog_sources(config.document.get("models")) {
208 Ok(()) => true,
209 Err(error) => {
210 tracing::warn!("rejected invalid model catalog sources: {error}");
211 false
212 }
213 }
214 }
215
216 pub fn subscribe_config(&self) -> tokio::sync::watch::Receiver<()> {
218 self.config_changed.subscribe()
219 }
220
221 pub fn publish_config(&self, config: Config) -> bool {
225 let policy = crate::management::policy(&config);
226 self.publish_config_and_policy(config, policy)
227 }
228
229 pub fn publish_config_and_policy(&self, config: Config, mut policy: Policy) -> bool {
232 if !Self::accepts_config(&config) {
233 return false;
234 }
235 policy.compat_disable_cooling = crate::scheduler::compat_cooling(&config);
236 let (enabled, strict) = signature_cache_config(&config);
237 cpa_translate::set_antigravity_signature_cache_config(enabled, strict);
238 let dir = cooldown_dir(&config, &policy);
239 let ws_auth = crate::relay::ws_auth(&config);
240 let config = Arc::new(config);
241 let commit = self.config_commit.lock().unwrap_or_else(PoisonError::into_inner);
245 let previous = {
246 let mut current = self.config.write().unwrap_or_else(PoisonError::into_inner);
249 let previous = std::mem::replace(&mut *current, config.clone());
250 self.plugins.config_published(config.clone());
251 previous
252 };
253 if let Some(catalogs) = self.catalogs.lock().unwrap_or_else(PoisonError::into_inner).as_ref() {
255 catalogs.configure(&config, false);
256 }
257 drop(commit);
258 if ws_auth && !crate::relay::ws_auth(&previous) {
261 self.executors.google.aistudio.relay.stop();
262 }
263 self.publish_policy(policy);
264 self.store.configure_cooldown_store(dir);
265 self.config_changed.send_replace(());
266 true
267 }
268
269 pub fn request_snapshot(&self) -> (Arc<Config>, Arc<Policy>) {
271 let config = self.config.read().unwrap_or_else(PoisonError::into_inner);
272 (config.clone(), self.policy())
273 }
274
275 pub fn store(&self) -> &Arc<CredentialStore> {
276 &self.store
277 }
278
279 pub fn next_pool_offset(&self, key: &str, size: usize) -> usize {
281 if size <= 1 || key.trim().is_empty() {
282 return 0;
283 }
284 let key = key.trim();
285 let mut offsets = self.pool_offsets.lock().unwrap_or_else(PoisonError::into_inner);
286 if offsets.len() >= 4096 && !offsets.contains_key(key) {
291 offsets.clear();
292 }
293 let slot = offsets.entry(key.to_owned()).or_default();
294 let offset = if *slot >= 2_147_483_640 { 0 } else { *slot };
295 *slot = offset + 1;
296 offset % size
297 }
298
299 pub fn suspension(&self, credential_id: &str, model: &str) -> crate::registry::Suspension {
302 use crate::registry::Suspension;
303 let Some(credential) = self.store.get(credential_id) else {
304 return Suspension::None;
305 };
306 let aliases = crate::registry::global_aliases(&self.config());
307 let key = crate::registry::selection_model(&aliases, &credential, model);
308 self.store.suspension(&credential, &key)
309 }
310
311 pub fn set_oauth_callback_sink(&self, sink: Option<OAuthCallbackSink>) {
313 *self.oauth_sink.write().unwrap_or_else(PoisonError::into_inner) = sink;
314 }
315
316 pub fn deliver_oauth_callback(&self, callback: &OAuthCallback) -> OAuthDelivery {
317 let sink = self.oauth_sink.read().unwrap_or_else(PoisonError::into_inner).clone();
318 sink.map_or(OAuthDelivery::NotPending, |sink| sink(callback))
319 }
320
321 pub(crate) fn for_executor(&self, credential: &Arc<Credential>) -> Arc<Credential> {
324 if self.deny_external.load(std::sync::atomic::Ordering::Relaxed) {
325 crate::testing::guarded(credential, &self.config())
326 } else {
327 credential.clone()
328 }
329 }
330
331 pub fn registry(&self) -> Arc<Registry> {
334 let config = self.config();
335 let generation = (
339 self.store.epoch(),
340 cpa_core::registry::catalog_generation(),
341 cpa_core::registry::dynamic::capabilities_generation(),
342 );
343 let mut cache = self.registry.lock().unwrap_or_else(PoisonError::into_inner);
344 if let Some((cfg, g, registry)) = cache.as_ref()
345 && Arc::ptr_eq(cfg, &config)
346 && *g == generation
347 {
348 return registry.clone();
349 }
350 let snapshot = self.store.snapshot();
351 let registry = Arc::new(Registry::build(&config, &snapshot));
352 let credentials: Vec<_> = snapshot.iter().map(|c| self.for_executor(c)).collect();
353 self.executors.models_registered(&credentials, &config);
354 *cache = Some((config, generation, registry.clone()));
355 registry
356 }
357
358 pub fn policy(&self) -> Arc<Policy> {
359 self.store.policy.read().unwrap_or_else(PoisonError::into_inner).clone()
360 }
361
362 pub fn publish_policy(&self, policy: Policy) {
363 {
364 let mut current = self.store.policy.write().unwrap_or_else(PoisonError::into_inner);
365 self.store
366 .scheduler
367 .lock()
368 .unwrap_or_else(PoisonError::into_inner)
369 .configure(¤t, &policy);
370 *current = Arc::new(policy);
371 }
372 self.store.clear_disabled_cooldowns();
373 }
374
375 pub async fn acquire(
380 &self,
381 selection: Selection,
382 cfg: &Config,
383 policy: Arc<Policy>,
384 registry: &Registry,
385 ) -> Result<Lease, AcquireError> {
386 let aliases = crate::registry::global_aliases(cfg);
387 let scope = selection.clone();
388 let admit = admission(registry, &aliases, &scope, &self.executors, self.plugin_runtime());
389 let now = std::time::SystemTime::now();
391 let ranks = |c: &Credential| {
392 let provider = c.provider.trim().to_ascii_lowercase();
393 let snapshot = match provider.as_str() {
394 "claude" => self.executors.claude.quota().snapshot(&c.id),
395 "codex" => self.executors.codex.quota().snapshot(&c.id),
396 _ => None,
397 };
398 snapshot.map_or_else(crate::scheduler::Windows::default, |s| {
399 crate::scheduler::Windows::observed(&provider, &s.signals, s.observed_at, now)
400 })
401 };
402 let request = crate::plugins::scheduler_request(self, &selection, &policy, &admit)?;
403 let pick = crate::plugins::scheduler_pick(self, request).await?;
404 let mut lease = self.store.select_picked(selection, policy, &admit, &ranks, pick)?;
405 if self.executors.readiness(&lease.credential, cfg) != Readiness::PrepareNow {
406 return Ok(lease);
407 }
408 let id = lease.credential.id.clone();
409 match self.prepare_credential(&id, cfg, None).await {
410 Ok(current) => {
411 lease.credential = current;
412 lease.redispatch();
414 }
415 Err(error) => {
416 lease.complete(Outcome::Failure(error.clone()));
417 return Err(AcquireError::Prepare { id, error });
418 }
419 }
420 Ok(lease)
421 }
422
423 pub(crate) async fn lease_credential(
428 &self,
429 id: &str,
430 selection: Selection,
431 cfg: &Config,
432 prepare: impl FnOnce(&Credential) -> Credential,
433 ) -> Option<Lease> {
434 let mut credential = self.store.get(id).filter(|c| !self.store.tokens_rejected(c))?;
435 if self.executors.readiness(&credential, cfg) == Readiness::PrepareNow {
436 credential = self.prepare_credential(id, cfg, None).await.ok()?;
437 }
438 let (picked, seq) = {
440 let mut scheduler = self.store.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
441 (scheduler.outside_selection_generation(), scheduler.dispatch())
442 };
443 Some(Lease {
444 store: self.store.clone(),
445 credential: Arc::new(prepare(&credential)),
446 execution_model: selection.model.clone(),
447 selection,
448 attempt: self.store.attempts.fetch_add(1, Ordering::Relaxed),
449 policy: self.policy(),
450 reported: false,
451 picked,
452 seq,
453 remote: None,
454 lcp: None,
455 })
456 }
457
458 pub fn remote_dispatch(&self) -> Option<Arc<dyn crate::remote::RemoteDispatch>> {
460 self.remote.read().unwrap_or_else(PoisonError::into_inner).clone()
461 }
462
463 pub fn set_remote_dispatch(&self, dispatch: Option<Arc<dyn crate::remote::RemoteDispatch>>) {
466 *self.remote.write().unwrap_or_else(PoisonError::into_inner) = dispatch;
467 }
468
469 pub(crate) async fn acquire_remote(
473 &self,
474 dispatch: &dyn crate::remote::RemoteDispatch,
475 selection: Selection,
476 request: crate::remote::RemoteRequest,
477 releases: &crate::remote::PendingReleases,
478 ) -> Result<(Lease, Option<i64>, String), crate::remote::RemoteError> {
479 let grant = dispatch.dispatch(request).await?;
480 let lease = Lease {
481 store: self.store.clone(),
482 credential: Arc::new(grant.credential),
483 execution_model: selection.model.clone(),
484 selection,
485 attempt: self.store.attempts.fetch_add(1, Ordering::Relaxed),
486 policy: self.policy(),
487 reported: false,
488 picked: 0,
489 seq: 0,
490 remote: Some(crate::remote::RemoteEnd {
491 end: Some(grant.end),
492 releases: releases.clone(),
493 cancel: grant.cancel,
494 private: grant.private,
495 }),
496 lcp: None,
499 };
500 Ok((lease, grant.request_retry, grant.user_api_key))
501 }
502
503 pub(crate) async fn prepare_remote(&self, lease: &mut Lease, cfg: &Config) -> Result<(), ExecError> {
508 if self.executors.readiness(&lease.credential, cfg) != Readiness::PrepareNow {
509 return Ok(());
510 }
511 let lock = self.store.prepare_lock(&lease.credential.id);
512 let _guard = lock.lock().await;
513 let patch = self
514 .executors
515 .prepare(&self.for_executor(&lease.credential), cfg)
516 .await?;
517 let mut prepared = (*lease.credential).clone();
518 patch.apply(&mut prepared.metadata);
519 lease.credential = Arc::new(prepared);
520 Ok(())
521 }
522
523 pub async fn shutdown_codex_refreshes(&self) {
527 self.codex_refreshes.shutdown.cancel();
528 self.codex_refreshes.tasks.close();
529 self.codex_refreshes.tasks.wait().await;
530 }
531
532 pub(crate) async fn lock_codex_file(&self, path: &Path) -> Result<std::fs::File, String> {
535 wait_codex_lock(path, &self.codex_refreshes.shutdown).await
536 }
537
538 async fn prepare_credential(
542 &self,
543 id: &str,
544 cfg: &Config,
545 failed: Option<u64>,
546 ) -> Result<Arc<Credential>, ExecError> {
547 self.prepare_credential_with(id, cfg, failed, false, None).await
548 }
549
550 async fn prepare_credential_with(
556 &self,
557 id: &str,
558 cfg: &Config,
559 failed: Option<u64>,
560 management: bool,
561 refused: Option<&Credential>,
562 ) -> Result<Arc<Credential>, ExecError> {
563 test_hook("prepare", id);
564 let guard = self.store.prepare_lock(id).lock_owned().await;
567 let (current, registered) = self
570 .store
571 .get_registered(id)
572 .filter(|(c, _)| !c.disabled)
573 .ok_or_else(|| {
574 ExecError::local(
575 409,
576 FailureScope::Request,
577 "credential removed or disabled during preparation",
578 )
579 })?;
580 if !management && self.store.tokens_rejected(¤t) {
583 return Err(ExecError::local(
584 401,
585 FailureScope::Request,
586 "credential tokens were refused; sign in again",
587 ));
588 }
589 match failed {
590 Some(revision) if current.revision != revision => return Ok(current),
591 Some(_) => {}
592 None if self.readiness(¤t, cfg) == Readiness::Ready => return Ok(current),
593 None => {}
594 }
595 let refused = refused.map(|c| Arc::new(c.clone()));
596 let view = self.for_executor(¤t);
597 let plugin = crate::plugins::refresh::route(self, ¤t, cfg);
598 if let Source::File(path) = ¤t.source
602 && plugin.is_none()
603 && current.provider == "codex"
604 && !marked(¤t, "plugin_virtual")
605 && path.exists()
606 {
607 let mode = match failed {
608 Some(_) if management => cpa_exec::codex_refresh::RefreshMode::Management,
609 Some(_) => cpa_exec::codex_refresh::RefreshMode::Rejected,
610 None => cpa_exec::codex_refresh::RefreshMode::Due,
611 };
612 let refresher = self.executors.codex.refresher(&view, cfg);
613 let file = path.clone();
614 return refresh_codex_file(
615 &self.codex_refreshes,
616 self.store.clone(),
617 refresher,
618 guard,
619 current,
620 registered,
621 view,
622 file,
623 mode,
624 refused,
625 )
626 .await;
627 }
628 let prepared = match (&plugin, failed) {
631 (Some(route), _) => crate::plugins::refresh::refresh(self, route, &view, ¤t, registered.registration)
632 .await
633 .map(|r| (r.patch, r.answered)),
634 (None, Some(_)) if management => self.executors.prepare_management(&view, cfg).await.map(|p| (p, false)),
635 (None, Some(_)) => self.executors.prepare_forced(&view, cfg).await.map(|p| (p, false)),
636 (None, None) => self.executors.prepare(&view, cfg).await.map(|p| (p, false)),
637 };
638 let (patch, publish_proxy) = prepared.inspect_err(|e| {
640 if let Some(refused) = &refused {
641 self.store.reject_refused(refused, registered.registration, e);
642 }
643 })?;
644 let publish_proxy = move |c: &mut Credential| {
645 if publish_proxy {
646 publish_metadata_proxy(c);
647 }
648 };
649 let store = self.store.clone();
650 let fence = RefreshFence {
651 registered,
652 plugin: plugin.is_some(),
653 };
654 let committed = tokio::task::spawn_blocking({
655 let id = id.to_owned();
656 move || {
657 let _guard = guard;
658 commit_refresh(&store, &id, ¤t, fence, &patch, publish_proxy)
659 }
660 })
661 .await
662 .map_err(|e| PatchError::Io(e.to_string()))
663 .and_then(|r| r)
664 .map_err(|e| {
665 if plugin.is_some() {
667 crate::plugins::refresh::failed(&self.store, id, registered.registration, chrono::Utc::now());
668 }
669 ExecError::local(
670 500,
671 FailureScope::Request,
672 format!("committing prepared credential: {e:?}"),
673 )
674 })?;
675 Ok(committed)
676 }
677
678 fn readiness(&self, c: &Credential, cfg: &Config) -> Readiness {
682 match self.executors.readiness(c, cfg) {
683 Readiness::Ready if self.plugin_refresh_due(c, cfg, chrono::Utc::now()) => Readiness::RefreshSoon,
684 readiness => readiness,
685 }
686 }
687
688 fn plugin_refresh_due(&self, c: &Credential, cfg: &Config, now: chrono::DateTime<chrono::Utc>) -> bool {
689 crate::plugins::refresh::due_at(self, c, cfg).is_some_and(|at| at <= now)
690 }
691
692 pub async fn refresh_after_unauthorized(&self, credential: &Credential, cfg: &Config) -> Option<Arc<Credential>> {
699 if !self.executors.has_refresh_credential(credential) {
700 return None;
701 }
702 let refreshed = match self
703 .prepare_credential_with(&credential.id, cfg, Some(credential.revision), false, Some(credential))
704 .await
705 {
706 Ok(refreshed) => refreshed,
707 Err(error) => {
708 if self.store.tokens_rejected(credential) {
714 if !crate::classify::is_invalid_grant(error.status, &String::from_utf8_lossy(&error.body)) {
715 return None;
716 }
717 tracing::warn!(
718 "credential refresh failed for {} ({}) after an upstream 401: status {}: {}; it is not used again until it is signed in again or its file changes",
719 credential.provider,
720 credential.id,
721 error.status,
722 refresh_error_text(&error)
723 );
724 return None;
725 }
726 let next = self
729 .refresh_state
730 .lock()
731 .unwrap_or_else(PoisonError::into_inner)
732 .retry_at(credential)
733 .map(|at| at.saturating_duration_since(Instant::now()));
734 warn_refresh_failed(credential, " after an upstream 401", &error, next);
735 return None;
736 }
737 };
738 (refreshed.revision != credential.revision).then(|| {
739 tracing::info!(
740 "credential refreshed for {} ({}) after an upstream 401",
741 credential.provider,
742 credential.id
743 );
744 refreshed
745 })
746 }
747
748 pub fn start_auto_refresh(self: &Arc<Self>) {
753 let mut task = self.refresh_task.lock().unwrap_or_else(PoisonError::into_inner);
754 if let Some(previous) = task.take() {
755 previous.abort();
756 }
757 let weak = Arc::downgrade(self);
758 let handle = tokio::spawn(async move {
761 loop {
762 match AssertUnwindSafe(refresh_loop(weak.clone())).catch_unwind().await {
763 Ok(()) => break,
764 Err(panic) => {
765 tracing::error!(
766 "credential auto-refresh loop panicked ({}); restarting it in {}s",
767 panic_text(&*panic),
768 REFRESH_LOOP_RESTART.as_secs()
769 );
770 tokio::time::sleep(REFRESH_LOOP_RESTART).await;
771 }
772 }
773 }
774 });
775 *task = Some(handle.abort_handle());
776 }
777
778 pub fn stop_auto_refresh(&self) {
779 if let Some(task) = self.refresh_task.lock().unwrap_or_else(PoisonError::into_inner).take() {
780 task.abort();
781 }
782 }
783}
784
785async fn refresh_loop(weak: std::sync::Weak<Runtime>) {
787 const HORIZON: Duration = Duration::from_secs(600);
792 const COALESCE: Duration = Duration::from_secs(1);
794 let Some(rt) = weak.upgrade() else {
795 return;
796 };
797 let mut store_changes = rt.store.subscribe();
798 let mut config_changes = rt.subscribe_config();
799 let mut unauthorized = rt.store.subscribe_unauthorized();
800 let mut jobs_ended = rt.refresh_done.subscribe();
801 let mut workers = (0, Arc::new(tokio::sync::Semaphore::new(0)));
803 drop(rt);
804 loop {
805 let Some(rt) = weak.upgrade() else {
806 break;
807 };
808 store_changes.borrow_and_update();
811 config_changes.borrow_and_update();
812 unauthorized.borrow_and_update();
813 jobs_ended.borrow_and_update();
814 let cfg = rt.config();
815 if workers.0 != refresh_workers(&cfg) {
818 workers = (
819 refresh_workers(&cfg),
820 Arc::new(tokio::sync::Semaphore::new(refresh_workers(&cfg))),
821 );
822 }
823 let spawn = |c: &Arc<Credential>, rejected: bool| {
826 let job = run_refresh_job(rt.clone(), cfg.clone(), workers.1.clone(), c.clone(), rejected);
827 tokio::spawn(job)
828 };
829 let rejected = rt.store.unauthorized();
831 let snapshot = rt.store.snapshot();
832 let (now, wall) = (Instant::now(), chrono::Utc::now());
833 let horizon = chrono::Duration::from_std(HORIZON).unwrap_or_default();
834 let mut wake: Option<Instant> = None;
835 let mut soonest = |at: Instant| wake = Some(wake.map_or(at, |w| w.min(at)));
836 let jobs: Vec<_> = {
837 let mut state = rt.refresh_state.lock().unwrap_or_else(PoisonError::into_inner);
838 state.reconcile(&snapshot);
839 let mut jobs = Vec::new();
840 rt.store.forget_stale_unauthorized(&rejected, &snapshot);
841 let eventually = wall + chrono::Duration::days(36_500);
843 let refused = rt.store.rejected_ids(&snapshot);
845 for c in snapshot.iter().filter(|c| !refused.contains(&c.id)) {
846 if !refresh_candidate(c) {
847 if c.disabled && refreshable(c) && !state.disabled_noted(c) {
850 let plugin_due = crate::plugins::refresh::due_at(&rt, c, &cfg);
851 if rt.executors.needs_prepare_at(c, &cfg, eventually) || plugin_due.is_some() {
852 soonest(now + HORIZON);
853 }
854 let due = rt
855 .executors
856 .prepare_due(c, &cfg, wall, horizon)
857 .or_else(|| plugin_due.filter(|at| *at <= wall + horizon).map(|at| at.max(wall)));
858 match due {
859 Some(at) if at > wall => soonest(now + (at - wall).to_std().unwrap_or_default()),
860 Some(_) => {
861 state.note_disabled(c);
862 tracing::info!(
863 "credential refresh for {} ({}) is due but the credential is disabled; it is refreshed once it is enabled",
864 c.provider,
865 c.id
866 );
867 }
868 None => {}
869 }
870 }
871 continue;
872 }
873 test_hook("scan", &c.id);
874 if let Some((revision, noted)) = rejected.pending.get(&c.id)
878 && *revision == c.revision
879 && rt.executors.has_refresh_credential(c)
880 {
881 let at = rejected
882 .refreshed
883 .get(&c.id)
884 .filter(|(r, t)| {
885 *r <= *revision && noted.saturating_duration_since(*t) < REFRESH_FAILURE_BACKOFF
886 })
887 .map_or(*noted, |_| *noted + REFRESH_FAILURE_BACKOFF);
888 if at > now {
889 soonest(at);
890 } else if state.reserve(c, now) {
891 jobs.push(spawn(c, true));
892 } else {
893 warn_refresh_held(&mut state, c);
894 if let Some(at) = state.retry_at(c).filter(|at| *at > now) {
895 soonest(at);
896 }
897 }
898 continue;
899 }
900 let plugin_due = crate::plugins::refresh::due_at(&rt, c, &cfg);
901 if rt.executors.needs_prepare_at(c, &cfg, eventually) || plugin_due.is_some() {
902 soonest(now + HORIZON);
903 }
904 let due = rt
905 .executors
906 .prepare_due(c, &cfg, wall, horizon)
907 .or_else(|| plugin_due.filter(|at| *at <= wall + horizon).map(|at| at.max(wall)));
908 match due {
909 Some(at) if at > wall => soonest(now + (at - wall).to_std().unwrap_or_default()),
910 Some(_) if state.reserve(c, now) => jobs.push(spawn(c, false)),
911 Some(_) => {
913 warn_refresh_held(&mut state, c);
914 if let Some(at) = state.retry_at(c).filter(|at| *at > now) {
915 soonest(at);
916 }
917 }
918 None => {}
919 }
920 }
921 jobs
922 };
923 let busy = !jobs.is_empty();
924 futures_util::future::join_all(jobs.into_iter().map(|job| async move {
928 let _ = tokio::time::timeout(REFRESH_JOB_BOUND, job).await;
929 }))
930 .await;
931 drop(rt);
932 if busy {
935 continue;
936 }
937 let deadline = async {
938 match wake {
939 Some(at) => tokio::time::sleep_until(at.into()).await,
940 None => std::future::pending().await,
941 }
942 };
943 tokio::select! {
944 () = deadline => {}
945 changed = store_changes.changed() => {
946 if changed.is_err() {
947 break;
948 }
949 tokio::time::sleep(COALESCE).await;
950 }
951 changed = config_changes.changed() => {
952 if changed.is_err() {
953 break;
954 }
955 tokio::time::sleep(COALESCE).await;
956 }
957 changed = unauthorized.changed() => {
958 if changed.is_err() {
959 break;
960 }
961 }
962 changed = jobs_ended.changed() => {
963 if changed.is_err() {
964 break;
965 }
966 tokio::time::sleep(COALESCE).await;
967 }
968 }
969 }
970}
971
972async fn run_refresh_job(
982 rt: Arc<Runtime>,
983 cfg: Arc<Config>,
984 workers: Arc<tokio::sync::Semaphore>,
985 credential: Arc<Credential>,
986 rejected: bool,
987) {
988 let mut reservation = Reservation {
989 rt: rt.clone(),
990 credential: credential.clone(),
991 ended: false,
992 };
993 let worker = workers.acquire_owned().await;
994 let failed = rejected.then_some(credential.revision);
995 let refused = rejected.then_some(&*credential);
997 let prepare =
998 AssertUnwindSafe(rt.prepare_credential_with(&credential.id, &cfg, failed, false, refused)).catch_unwind();
999 tokio::pin!(prepare);
1000 let prepared = match tokio::time::timeout(REFRESH_JOB_BOUND, &mut prepare).await {
1004 Ok(prepared) => prepared,
1005 Err(_) => {
1006 tracing::warn!(
1007 "credential refresh for {} ({}) still running after {}s; no other refresh of it starts until it ends",
1008 credential.provider,
1009 credential.id,
1010 REFRESH_JOB_BOUND.as_secs()
1011 );
1012 drop(worker);
1013 prepare.await
1014 }
1015 };
1016 let (result, panicked) = match prepared {
1018 Ok(result) => (result, false),
1019 Err(panic) => {
1020 let text = format!("refresh panicked: {}", panic_text(&*panic));
1021 (Err(ExecError::local(500, FailureScope::Request, text)), true)
1022 }
1023 };
1024 if rejected && result.is_ok() {
1025 rt.store.forget_unauthorized(&credential.id, credential.revision);
1026 }
1027 let trigger = if rejected { " after an upstream 401" } else { "" };
1028 let current = rt.store.get(&credential.id);
1029 let Some(current) = current.filter(|c| match &result {
1032 Ok(prepared) => prepared.revision == c.revision,
1033 Err(_) => c.revision == credential.revision,
1034 }) else {
1035 reservation.end(|state| state.release(&credential.id));
1036 if let Err(error) = &result {
1037 warn_refresh_failed(&credential, trigger, error, None);
1038 }
1039 return;
1040 };
1041 let plugin = !panicked && crate::plugins::refresh::route(&rt, ¤t, &cfg).is_some();
1044 let ineffective = !plugin && result.is_ok() && rt.readiness(¤t, &cfg) != Readiness::Ready;
1045 let now = Instant::now();
1046 let retry = reservation.end(|state| {
1047 state.finish(¤t, result.as_ref().err().filter(|_| !plugin), ineffective, now);
1048 state.retry_at(¤t)
1049 });
1050 match &result {
1051 Err(error) => {
1052 let next = if plugin {
1053 crate::plugins::refresh::due_at(&rt, ¤t, &cfg)
1054 .map(|at| (at - chrono::Utc::now()).to_std().unwrap_or_default())
1055 } else {
1056 retry.map(|at| at.saturating_duration_since(now))
1057 };
1058 warn_refresh_failed(¤t, trigger, error, next);
1059 }
1060 Ok(_) if ineffective => {}
1062 Ok(_) if current.revision != credential.revision => {
1063 tracing::info!(
1064 "credential refreshed for {} ({}){trigger}",
1065 current.provider,
1066 current.id
1067 );
1068 }
1069 Ok(_) => {}
1070 }
1071}
1072
1073struct Reservation {
1077 rt: Arc<Runtime>,
1078 credential: Arc<Credential>,
1079 ended: bool,
1080}
1081
1082impl Reservation {
1083 fn end<T>(&mut self, record: impl FnOnce(&mut RefreshState) -> T) -> T {
1084 let recorded = record(&mut self.rt.refresh_state.lock().unwrap_or_else(PoisonError::into_inner));
1085 self.ended = true;
1086 self.rt.refresh_done.send_replace(());
1087 recorded
1088 }
1089}
1090
1091impl Drop for Reservation {
1092 fn drop(&mut self) {
1093 if !self.ended {
1094 let error = ExecError::local(500, FailureScope::Request, "refresh job ended without a result");
1095 let credential = self.credential.clone();
1096 self.end(|state| state.finish(&credential, Some(&error), false, Instant::now()));
1097 warn_refresh_failed(&credential, "", &error, None);
1098 }
1099 }
1100}
1101
1102impl Drop for Runtime {
1103 fn drop(&mut self) {
1104 if let Some(task) = self
1105 .refresh_task
1106 .get_mut()
1107 .unwrap_or_else(PoisonError::into_inner)
1108 .take()
1109 {
1110 task.abort();
1111 }
1112 }
1113}
1114
1115fn cooldown_dir(cfg: &Config, policy: &Policy) -> Option<std::path::PathBuf> {
1117 policy.save_cooldown_status.then(|| cfg.auth_dir.clone())
1118}
1119
1120const REFRESH_JOB_BOUND: Duration = Duration::from_secs(if cfg!(test) { 2 } else { 300 });
1124
1125const REFRESH_LOOP_RESTART: Duration = Duration::from_secs(if cfg!(test) { 0 } else { 5 });
1127
1128fn warn_refresh_failed(c: &Credential, trigger: &str, error: &ExecError, next: Option<Duration>) {
1135 let text = refresh_error_text(error);
1136 let next = match next {
1137 Some(after) if !after.is_zero() => format!("next attempt in {}s", ceil_secs(after)),
1138 _ => "next attempt when it is next due".to_owned(),
1139 };
1140 tracing::warn!(
1141 "credential refresh failed for {} ({}){trigger}: status {}: {text}; {next}",
1142 c.provider,
1143 c.id,
1144 error.status
1145 );
1146}
1147
1148fn refresh_error_text(error: &ExecError) -> String {
1150 match error.headers.get(crate::plugins::refresh::PLUGIN_ERROR) {
1151 Some(code) => match code.to_str().unwrap_or_default() {
1152 "" => "plugin refresh failed".to_owned(),
1153 code => format!("plugin refresh failed: {code}"),
1154 },
1155 None => crate::sanitize::summary(&String::from_utf8_lossy(&error.body)),
1156 }
1157}
1158
1159fn warn_refresh_held(state: &mut RefreshState, c: &Credential) {
1163 if state.note_hold(c) {
1164 tracing::warn!(
1165 "credential refresh for {} ({}) is due but its previous refresh is still running; it is tried again once that ends",
1166 c.provider,
1167 c.id
1168 );
1169 }
1170}
1171
1172fn ceil_secs(d: Duration) -> u64 {
1173 d.as_secs() + u64::from(d.subsec_nanos() > 0)
1174}
1175
1176fn panic_text(panic: &(dyn std::any::Any + Send)) -> &str {
1178 panic
1179 .downcast_ref::<&str>()
1180 .copied()
1181 .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
1182 .unwrap_or("non-string payload")
1183}
1184
1185const REFRESH_FAILURE_BACKOFF: Duration = Duration::from_secs(300);
1188
1189fn refresh_workers(cfg: &Config) -> usize {
1191 cfg.document
1192 .get("oauth")
1193 .and_then(|o| o.get("auth-auto-refresh-workers"))
1194 .and_then(serde_yaml_ng::Value::as_i64)
1195 .filter(|n| *n > 0)
1196 .map_or(16, |n| n as usize)
1197}
1198
1199pub(crate) enum Pick {
1201 Configured,
1203 Builtin(crate::scheduler::Strategy),
1205 Auth(String),
1207}
1208
1209#[allow(clippy::type_complexity)]
1215fn ready<'a>(
1216 creds: &'a [Arc<Credential>],
1217 scheduler: &crate::scheduler::Scheduler,
1218 selection: &Selection,
1219 policy: &Policy,
1220 admit: &Admit<'_>,
1221 now: Instant,
1222 configured: bool,
1223) -> Result<(Vec<(&'a Arc<Credential>, String)>, Vec<(&'a Arc<Credential>, String)>), AcquireError> {
1224 let eligible: Vec<_> = creds
1225 .iter()
1226 .filter(|c| !c.disabled && !selection.exclude.contains(&c.id))
1227 .filter(|c| policy.retry_limit(c) >= selection.retry_round && (!configured || scheduler.admits(c, policy)))
1228 .filter_map(|c| admit(c).map(|m| (c, m)))
1229 .collect();
1230 let candidates: Vec<_> = eligible
1231 .iter()
1232 .filter(|(c, m)| scheduler.wait(c, m, now).is_none() && !scheduler.tokens_rejected(c))
1233 .map(|(c, _)| (*c, crate::registry::provider_key(c)))
1234 .collect();
1235 if candidates.is_empty() {
1236 let cause = eligible
1237 .iter()
1238 .filter_map(|(c, m)| scheduler.last_error(c, m))
1239 .max_by_key(|s| s.deadline)
1240 .map(|s| s.error.clone());
1241 let wait = eligible.iter().filter_map(|(c, m)| scheduler.wait(c, m, now)).min();
1242 if !eligible.is_empty() && eligible.iter().all(|(c, m)| scheduler.quota_cooling(c, m, now)) {
1243 return Err(AcquireError::Cooldown {
1244 wait: wait.unwrap_or_default(),
1245 cause,
1246 });
1247 }
1248 return Err(AcquireError::Unavailable {
1249 retry_after: if eligible.is_empty() { None } else { wait },
1250 cause,
1251 });
1252 }
1253 Ok((eligible, candidates))
1254}
1255
1256#[derive(Debug)]
1257pub enum AcquireError {
1258 Unavailable {
1261 retry_after: Option<Duration>,
1262 cause: Option<String>,
1263 },
1264 Cooldown {
1266 wait: Duration,
1267 cause: Option<String>,
1268 },
1269 Prepare {
1270 id: String,
1271 error: ExecError,
1272 },
1273 Rejected {
1275 code: String,
1276 message: String,
1277 },
1278 Scheduler(ExecError),
1280}
1281
1282#[derive(Debug, Clone, Default)]
1284pub struct Selection {
1285 pub providers: Vec<String>,
1287 pub provider: String,
1289 pub model: String,
1291 pub session: Option<String>,
1293 pub session_parent: Option<String>,
1296 pub session_fork: bool,
1298 pub lcp: Option<Arc<crate::lcp::Request>>,
1301 pub exclude: Vec<String>,
1303 pub retry_round: usize,
1304 pub scheduler: Option<Arc<crate::plugins::SchedulerContext>>,
1307}
1308
1309impl Selection {
1310 pub fn new(provider: &str, model: &str) -> Self {
1311 Self {
1312 providers: vec![provider.to_owned()],
1313 model: model.to_owned(),
1314 ..Self::default()
1315 }
1316 }
1317
1318 pub fn provider_keys(&self) -> Vec<String> {
1320 if self.providers.is_empty() && !self.provider.is_empty() {
1321 vec![self.provider.clone()]
1322 } else {
1323 self.providers.clone()
1324 }
1325 }
1326}
1327
1328fn unchanged(current: &Credential, next: &Credential) -> bool {
1331 current.id == next.id
1332 && current.source == next.source
1333 && current.metadata == next.metadata
1334 && current.attributes == next.attributes
1335 && current.provider == next.provider
1336 && current.disabled == next.disabled
1337 && current.label == next.label
1338}
1339
1340pub fn standalone_admission<'a>(
1343 selection: &'a Selection,
1344 policy: &'a Policy,
1345) -> impl Fn(&Credential) -> Option<String> + 'a {
1346 let providers = selection.provider_keys();
1347 move |c| {
1348 if !providers.contains(&crate::registry::provider_key(c)) {
1349 return None;
1350 }
1351 let route = selection.model.trim();
1352 let prefix = crate::registry::credential_prefix(c);
1353 let base = crate::scheduler::canonical_model(route);
1354 if !prefix.is_empty()
1355 && !base.starts_with(&format!("{prefix}/"))
1356 && (policy.force_model_prefix || base.contains('/'))
1357 {
1358 return None;
1359 }
1360 let (models, _) = crate::registry::execution_models(&HashMap::new(), c, route);
1361 let model = models.into_iter().next().unwrap_or_default();
1362 let key = crate::scheduler::canonical_model(&model).go_lower();
1364 let excluded: Vec<String> = match c.attributes.get("excluded_models").filter(|v| !v.trim().is_empty()) {
1365 Some(list) => list.split(',').map(|p| p.trim().go_lower()).collect(),
1366 None => c
1367 .metadata
1368 .get("excluded_models")
1369 .and_then(serde_json::Value::as_array)
1370 .map(|l| {
1371 l.iter()
1372 .filter_map(|v| v.as_str())
1373 .map(|p| p.trim().go_lower())
1374 .collect()
1375 })
1376 .unwrap_or_default(),
1377 };
1378 if excluded.iter().any(|p| crate::registry::wildcard(p, &key)) {
1379 return None;
1380 }
1381 Some(model)
1382 }
1383}
1384
1385#[derive(Debug, Clone)]
1386pub enum Outcome {
1387 Success,
1388 Failure(ExecError),
1389 Neutral(ExecError),
1392 Cancelled,
1394}
1395
1396pub type Admit<'a> = dyn Fn(&Credential) -> Option<String> + 'a;
1399
1400pub fn admission<'a>(
1403 registry: &'a Registry,
1404 aliases: &'a HashMap<String, Vec<crate::registry::OAuthAlias>>,
1405 selection: &'a Selection,
1406 executors: &'a Executors,
1407 plugins: &'a crate::plugins::PluginRuntime,
1408) -> impl Fn(&Credential) -> Option<String> + 'a {
1409 move |c| {
1410 let provider = crate::registry::provider_key(c);
1411 if !selection.provider_keys().contains(&provider)
1412 || !(executors.supports(&c.provider) || plugins.has_executor(&c.provider))
1413 {
1414 return None;
1415 }
1416 let key = crate::registry::selection_model(aliases, c, &selection.model);
1417 let route = crate::scheduler::canonical_model(&selection.model);
1418 if route.is_empty() {
1419 return Some(key);
1420 }
1421 let selection_key = crate::scheduler::canonical_model(&key);
1422 (registry.client_supports(&c.id, route)
1423 || (selection_key != route && registry.client_supports(&c.id, selection_key)))
1424 .then_some(key)
1425 }
1426}
1427
1428pub struct Lease {
1430 store: Arc<CredentialStore>,
1431 pub credential: Arc<Credential>,
1432 pub selection: Selection,
1433 pub execution_model: String,
1435 pub attempt: u64,
1436 policy: Arc<Policy>,
1437 reported: bool,
1438 picked: u32,
1441 seq: u64,
1443 remote: Option<crate::remote::RemoteEnd>,
1446 pub lcp: Option<crate::lcp::Match>,
1448}
1449
1450impl Lease {
1451 pub fn is_remote(&self) -> bool {
1453 self.remote.is_some()
1454 }
1455
1456 pub(crate) fn home_private(&self) -> Option<&crate::remote::Private> {
1458 self.remote.as_ref().map(|remote| &remote.private)
1459 }
1460
1461 pub(crate) fn remote_cancelled(&self) -> Option<futures_util::future::BoxFuture<'static, ()>> {
1463 let signal = self.remote.as_ref()?.cancel.clone()?;
1464 Some(crate::remote::cancelled(signal))
1465 }
1466
1467 pub(crate) fn remote_cancel_requested(&self) -> bool {
1470 self.remote
1471 .as_ref()
1472 .and_then(|remote| remote.cancel.as_ref())
1473 .is_some_and(|signal| *signal.borrow())
1474 }
1475
1476 pub(crate) fn redispatch(&mut self) {
1479 if self.remote.is_none() {
1480 self.seq = self
1481 .store
1482 .scheduler
1483 .lock()
1484 .unwrap_or_else(PoisonError::into_inner)
1485 .dispatch();
1486 }
1487 }
1488
1489 pub fn complete(mut self, outcome: Outcome) {
1490 self.report(outcome);
1491 }
1492
1493 pub(crate) fn release_into(&mut self, releases: &crate::remote::PendingReleases) {
1496 if let Some(remote) = &mut self.remote {
1497 remote.releases = releases.clone();
1498 }
1499 }
1500
1501 pub fn accepted(&self) {
1505 if self.remote.is_none() {
1506 self.store.accept_probe(self);
1507 }
1508 }
1509
1510 pub fn note(&self, model: &str, outcome: &Outcome) {
1513 if self.remote.is_none() {
1514 self.store.record_model(self, model, outcome);
1515 }
1516 }
1517
1518 fn report(&mut self, outcome: Outcome) {
1519 if std::mem::replace(&mut self.reported, true) {
1520 return;
1521 }
1522 match &mut self.remote {
1523 Some(remote) => remote.finish(),
1526 None => self.store.record(self, &outcome),
1527 }
1528 }
1529}
1530
1531impl Drop for Lease {
1532 fn drop(&mut self) {
1533 self.report(Outcome::Cancelled);
1534 }
1535}
1536
1537impl std::fmt::Debug for Lease {
1538 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1539 f.debug_struct("Lease")
1540 .field("credential", &self.credential.id)
1541 .field("attempt", &self.attempt)
1542 .finish()
1543 }
1544}
1545
1546#[derive(Debug, PartialEq, Eq)]
1547pub enum PatchError {
1548 NotFound,
1549 Stale {
1551 current: u64,
1552 },
1553 TypeIsImmutable,
1555 ConfigBacked,
1557 Io(String),
1558 LockBusy(std::path::PathBuf),
1561}
1562
1563#[derive(Debug, Clone, Copy)]
1565pub(crate) struct RefreshFence {
1566 pub registered: crate::plugins::refresh::Registered,
1569 pub plugin: bool,
1571}
1572
1573#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1574pub struct AttemptStats {
1575 pub success: u64,
1576 pub failure: u64,
1577 pub cancelled: u64,
1578}
1579
1580struct Inner {
1581 creds: Vec<Arc<Credential>>,
1582 generation: u64,
1584 epoch: u64,
1586}
1587
1588#[derive(Debug, Clone, Default)]
1590pub(crate) struct Rejections {
1591 pub pending: HashMap<String, (u64, Instant)>,
1593 pub refreshed: HashMap<String, (u64, Instant)>,
1595}
1596
1597pub struct CredentialStore {
1598 inner: RwLock<Inner>,
1599 policy: RwLock<Arc<Policy>>,
1600 scheduler: Mutex<Scheduler>,
1601 attempts: AtomicU64,
1602 stats: [AtomicU64; 3],
1603 prepare_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
1604 write_locks: Mutex<HashMap<String, Arc<Mutex<()>>>>,
1609 reload: RwLock<()>,
1610 cooldown_dir: RwLock<Option<std::path::PathBuf>>,
1612 cooldown_backend: RwLock<Option<Arc<dyn crate::cooldown_store::Backend>>>,
1614 cooldown_write: Mutex<()>,
1616 activity: Mutex<HashMap<String, CredentialActivity>>,
1617 persisted: Mutex<HashMap<String, u64>>,
1619 provenance: Mutex<HashMap<String, Provenance>>,
1621 error_events: std::sync::OnceLock<Arc<crate::usage::UsageQueue>>,
1623 changed: tokio::sync::watch::Sender<()>,
1625 unauthorized: Mutex<Rejections>,
1629 unauthorized_noted: tokio::sync::watch::Sender<()>,
1630 session_closer: std::sync::OnceLock<SessionCloser>,
1633 refresh_schedules: crate::plugins::refresh::Schedules,
1637}
1638
1639type SessionCloser = Box<dyn Fn(&str) + Send + Sync>;
1641
1642fn removed_ids(before: &[Arc<Credential>], after: &[Arc<Credential>]) -> Vec<String> {
1644 let kept: std::collections::HashSet<&str> = after.iter().map(|c| c.id.as_str()).collect();
1645 before
1646 .iter()
1647 .filter(|c| !kept.contains(c.id.as_str()))
1648 .map(|c| c.id.clone())
1649 .collect()
1650}
1651
1652fn marked(credential: &Credential, key: &str) -> bool {
1655 credential
1656 .attributes
1657 .get(key)
1658 .is_some_and(|v| v.trim().eq_ignore_ascii_case("true"))
1659}
1660
1661#[derive(Clone)]
1666pub struct Provenance {
1667 pub revision: u64,
1668 pub sha256: [u8; 32],
1670 pub doc: Option<Arc<cpa_common::json::GoValue>>,
1673 pub loaded: bool,
1676}
1677
1678pub struct Commit {
1680 pub bytes: Vec<u8>,
1682 pub write: bool,
1686 pub disk: Option<Option<[u8; 32]>>,
1689 pub doc: Option<Arc<cpa_common::json::GoValue>>,
1691}
1692
1693fn go_typed(v: &Value, literal: bool) -> cpa_common::json::GoValue {
1696 use cpa_common::json::GoValue;
1697 match v {
1698 Value::Null => GoValue::Null,
1699 Value::Bool(b) => GoValue::Bool(*b),
1700 Value::Number(n) if literal || n.is_i64() || n.is_u64() => GoValue::Number(n.to_string()),
1701 Value::Number(n) => {
1702 let text = n.to_string();
1703 GoValue::Number(
1704 cpa_common::json::json_float(cpa_common::json::parse_float(text.as_bytes())).unwrap_or(text),
1705 )
1706 }
1707 Value::String(s) => GoValue::String(s.clone()),
1708 Value::Array(items) => GoValue::Array(items.iter().map(|v| go_typed(v, literal)).collect()),
1709 Value::Object(map) => GoValue::Object(map.iter().map(|(k, v)| (k.clone(), go_typed(v, literal))).collect()),
1710 }
1711}
1712
1713fn sha256(bytes: &[u8]) -> [u8; 32] {
1714 use sha2::{Digest, Sha256};
1715 Sha256::digest(bytes).into()
1716}
1717
1718impl CredentialStore {
1719 pub fn new(credentials: Vec<Credential>) -> Arc<Self> {
1720 let mut inner = Inner {
1721 creds: Vec::new(),
1722 generation: 0,
1723 epoch: 0,
1724 };
1725 for mut cred in credentials {
1726 inner.generation += 1;
1727 cred.revision = inner.generation;
1728 inner.creds.push(Arc::new(cred));
1729 }
1730 let refresh_schedules = crate::plugins::refresh::Schedules::default();
1731 crate::plugins::refresh::register(&refresh_schedules, &[], &inner.creds, &[]);
1732 crate::plugins::refresh::begin_bootstrap(&refresh_schedules);
1733 Arc::new(Self {
1734 inner: RwLock::new(inner),
1735 policy: RwLock::new(Arc::new(Policy::default())),
1736 scheduler: Mutex::default(),
1737 attempts: AtomicU64::new(0),
1738 stats: Default::default(),
1739 prepare_locks: Mutex::default(),
1740 write_locks: Mutex::default(),
1741 reload: RwLock::default(),
1742 cooldown_dir: RwLock::default(),
1743 cooldown_backend: RwLock::default(),
1744 cooldown_write: Mutex::default(),
1745 activity: Mutex::default(),
1746 persisted: Mutex::default(),
1747 provenance: Mutex::default(),
1748 error_events: std::sync::OnceLock::new(),
1749 changed: tokio::sync::watch::Sender::new(()),
1750 unauthorized: Mutex::default(),
1751 unauthorized_noted: tokio::sync::watch::Sender::new(()),
1752 session_closer: std::sync::OnceLock::new(),
1753 refresh_schedules,
1754 })
1755 }
1756
1757 pub(crate) fn get_registered(&self, id: &str) -> Option<(Arc<Credential>, crate::plugins::refresh::Registered)> {
1760 let inner = self.read();
1761 let credential = inner.creds.iter().find(|c| c.id == id)?.clone();
1762 Some((credential, self.registered(id)))
1763 }
1764
1765 fn registered(&self, id: &str) -> crate::plugins::refresh::Registered {
1767 crate::plugins::refresh::registered(&self.refresh_schedules, id).unwrap_or_default()
1768 }
1769
1770 pub(crate) fn record_refresh(
1776 &self,
1777 id: &str,
1778 registration: u64,
1779 at: Option<(u64, u64)>,
1780 record: impl FnOnce(&Credential, &mut crate::plugins::refresh::Schedule),
1781 ) -> Result<Arc<Credential>, PatchError> {
1782 let inner = self.read();
1783 let current = inner.creds.iter().find(|c| c.id == id).ok_or(PatchError::NotFound)?;
1784 let registered = self.registered(id);
1785 if registered.registration != registration {
1786 return Err(PatchError::NotFound);
1787 }
1788 if at.is_some_and(|at| at != (current.revision, registered.credential_version)) {
1789 return Err(PatchError::Stale {
1790 current: current.revision,
1791 });
1792 }
1793 crate::plugins::refresh::record(&self.refresh_schedules, current, record);
1794 Ok(current.clone())
1795 }
1796
1797 pub(crate) fn refresh_schedules(&self) -> &crate::plugins::refresh::Schedules {
1799 &self.refresh_schedules
1800 }
1801
1802 pub fn end_bootstrap(&self) {
1805 if crate::plugins::refresh::end_bootstrap(&self.refresh_schedules) {
1806 self.changed.send_replace(());
1808 }
1809 }
1810
1811 fn register(
1814 &self,
1815 before: &[Arc<Credential>],
1816 after: &[Arc<Credential>],
1817 saved: &[cpa_plugin::hostauth::SavedRefresh],
1818 ) {
1819 crate::plugins::refresh::register(&self.refresh_schedules, before, after, saved);
1820 }
1821
1822 pub fn configure_cooldown_store(&self, dir: Option<std::path::PathBuf>) {
1826 let old = self.cooldown_dir.read().unwrap_or_else(PoisonError::into_inner).clone();
1827 if old == dir {
1828 return;
1829 }
1830 if old.is_some() {
1831 self.persist_cooldowns();
1832 }
1833 *self.cooldown_dir.write().unwrap_or_else(PoisonError::into_inner) = dir.clone();
1834 if let Some(dir) = dir {
1835 self.restore_cooldowns(&dir);
1836 }
1837 }
1838
1839 pub fn set_cooldown_backend(&self, backend: Arc<dyn crate::cooldown_store::Backend>) {
1842 *self.cooldown_backend.write().unwrap_or_else(PoisonError::into_inner) = Some(backend);
1843 let dir = self.cooldown_dir.read().unwrap_or_else(PoisonError::into_inner).clone();
1844 if let Some(dir) = dir {
1845 self.restore_cooldowns(&dir);
1846 }
1847 }
1848
1849 fn cooldown_backend(&self) -> Option<Arc<dyn crate::cooldown_store::Backend>> {
1850 self.cooldown_backend
1851 .read()
1852 .unwrap_or_else(PoisonError::into_inner)
1853 .clone()
1854 }
1855
1856 fn restore_cooldowns(&self, dir: &std::path::Path) {
1859 let loaded = match self.cooldown_backend() {
1860 Some(backend) => backend.load(),
1861 None => crate::cooldown_store::load(dir),
1862 };
1863 let records = match loaded {
1864 Ok(records) => records,
1865 Err(error) => {
1866 tracing::warn!(%error, "failed to restore cooldown state");
1867 return;
1868 }
1869 };
1870 if !records.is_empty() {
1871 let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
1872 let inner = self.read();
1873 let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
1874 let (now, wall) = (Instant::now(), std::time::SystemTime::now());
1875 for record in &records {
1876 let Some(c) = inner.creds.iter().find(|c| c.id == record.auth_id.trim()) else {
1877 continue;
1878 };
1879 if c.disabled || policy.cooling_disabled(c) {
1880 continue;
1881 }
1882 let has_models = records
1883 .iter()
1884 .any(|r| r.auth_id.trim() == c.id && !r.model.trim().is_empty());
1885 scheduler.restore(c, record, has_models, policy.max_trusted_cooldown, now, wall);
1886 }
1887 }
1888 self.persist_cooldowns();
1889 }
1890
1891 pub fn persist_cooldowns(&self) {
1893 let Some(dir) = self.cooldown_dir.read().unwrap_or_else(PoisonError::into_inner).clone() else {
1894 return;
1895 };
1896 let _write = self.cooldown_write.lock().unwrap_or_else(PoisonError::into_inner);
1897 let wall = std::time::SystemTime::now();
1898 let mut records = {
1899 let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
1900 let inner = self.read();
1901 let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
1902 let now = Instant::now();
1903 inner
1904 .creds
1905 .iter()
1906 .filter(|c| !c.disabled && !policy.cooling_disabled(c))
1907 .flat_map(|c| scheduler.records(c, now, wall))
1908 .collect::<Vec<_>>()
1909 };
1910 records.sort_by(|a, b| (&a.provider, &a.auth_id, &a.model).cmp(&(&b.provider, &b.auth_id, &b.model)));
1911 let saved = match self.cooldown_backend() {
1912 Some(backend) => backend.save(records, wall),
1913 None => crate::cooldown_store::save(&dir, records, wall),
1914 };
1915 if let Err(error) = saved {
1916 tracing::warn!(%error, "failed to persist cooldown state");
1917 }
1918 }
1919
1920 fn persist_credential(&self, credential: &Arc<Credential>) {
1926 let Source::File(path) = &credential.source else {
1927 return;
1928 };
1929 if credential.disabled || marked(credential, "runtime_only") || marked(credential, "plugin_virtual") {
1931 return;
1932 }
1933 {
1934 let persisted = self.persisted.lock().unwrap_or_else(PoisonError::into_inner);
1935 if persisted.get(&credential.id) == Some(&credential.revision) {
1936 return;
1937 }
1938 }
1939 let written = self.write_unlocked(
1942 &credential.id,
1943 credential.revision,
1944 None,
1945 false,
1946 |current| {
1947 let mut metadata = current.metadata.clone();
1948 metadata.insert("disabled".into(), Value::Bool(current.disabled));
1949 let go = |bytes: &[u8]| cpa_common::json::GoValue::parse_f64(bytes);
1950 let Some(target) = serde_json::to_vec(&metadata).ok().and_then(|b| go(&b)) else {
1952 return Err(PatchError::NotFound);
1953 };
1954 let existing = std::fs::read(path).map_err(|_| PatchError::NotFound)?;
1955 if go(&existing).as_ref() == Some(&target) {
1956 return Ok(None);
1957 }
1958 Ok(Some((Some(path.clone()), target.marshal(), ())))
1959 },
1960 |inner, index, bytes, ()| {
1961 if let Ok(written) = serde_json::from_slice::<Map<String, Value>>(bytes) {
1965 let mut next = Credential::clone(&inner.creds[index]);
1966 next.metadata = written;
1967 inner.creds[index] = Arc::new(next);
1968 }
1969 let id = &inner.creds[index].id;
1973 let mut provenance = self.provenance.lock().unwrap_or_else(PoisonError::into_inner);
1974 match provenance.get_mut(id) {
1975 Some(p) if p.doc.is_some() => p.sha256 = sha256(bytes),
1976 Some(_) => {
1977 provenance.remove(id);
1978 }
1979 None => {}
1980 }
1981 },
1982 );
1983 match written {
1984 Ok(_) => {}
1985 Err(PatchError::Io(error)) => {
1986 tracing::warn!(id = %credential.id, %error, "failed to persist credential");
1987 return;
1988 }
1989 Err(_) => return,
1990 }
1991 self.persisted
1992 .lock()
1993 .unwrap_or_else(PoisonError::into_inner)
1994 .insert(credential.id.clone(), credential.revision);
1995 }
1996
1997 fn clear_replaced_unauthorized(&self, before: &[Arc<Credential>], after: &[Arc<Credential>]) -> bool {
2004 let replaced: Vec<&str> = after
2005 .iter()
2006 .filter(|next| {
2007 before
2008 .iter()
2009 .find(|c| c.id == next.id)
2010 .is_some_and(|previous| !previous.disabled && !next.disabled && credentials_changed(previous, next))
2011 })
2012 .map(|c| c.id.as_str())
2013 .collect();
2014 if replaced.is_empty() {
2015 return false;
2016 }
2017 let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2018 let now = Instant::now();
2019 let mut cleared = false;
2020 for id in replaced {
2021 cleared |= scheduler.clear_unauthorized(id, now);
2022 }
2023 cleared
2024 }
2025
2026 fn clear_disabled_cooldowns(&self) {
2028 let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
2029 let cleared = {
2030 let inner = self.read();
2031 self.scheduler
2032 .lock()
2033 .unwrap_or_else(PoisonError::into_inner)
2034 .clear_disabled(&inner.creds, &policy)
2035 };
2036 if cleared {
2037 self.persist_cooldowns();
2038 }
2039 }
2040
2041 pub fn snapshot(&self) -> Vec<Arc<Credential>> {
2042 self.read().creds.clone()
2043 }
2044
2045 pub fn get(&self, id: &str) -> Option<Arc<Credential>> {
2046 self.read().creds.iter().find(|c| c.id == id).cloned()
2047 }
2048
2049 pub fn select(self: &Arc<Self>, selection: Selection) -> Option<Lease> {
2052 let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
2053 let scope = selection.clone();
2054 let admit = standalone_admission(&scope, &policy);
2055 self.select_with(selection, policy.clone(), &admit).ok()
2056 }
2057
2058 pub fn select_with(
2059 self: &Arc<Self>,
2060 selection: Selection,
2061 policy: Arc<Policy>,
2062 admit: &Admit<'_>,
2063 ) -> Result<Lease, AcquireError> {
2064 self.select_ranked(selection, policy, admit, &|_| crate::scheduler::Windows::default())
2065 }
2066
2067 pub fn select_ranked(
2069 self: &Arc<Self>,
2070 selection: Selection,
2071 policy: Arc<Policy>,
2072 admit: &Admit<'_>,
2073 ranks: &crate::scheduler::Ranks<'_>,
2074 ) -> Result<Lease, AcquireError> {
2075 self.select_picked(selection, policy, admit, ranks, Pick::Configured)
2076 }
2077
2078 pub(crate) fn scheduler_candidates(
2082 &self,
2083 selection: &Selection,
2084 policy: &Policy,
2085 admit: &Admit<'_>,
2086 across: bool,
2087 ) -> Result<Vec<Arc<Credential>>, AcquireError> {
2088 let inner = self.read();
2089 let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2090 let (_, ready) = ready(
2091 &inner.creds,
2092 &scheduler,
2093 selection,
2094 policy,
2095 admit,
2096 Instant::now(),
2097 false,
2098 )?;
2099 let tier = ready.iter().map(|(c, _)| crate::scheduler::priority(c)).max();
2100 let mut out: Vec<_> = ready
2101 .into_iter()
2102 .filter(|(c, _)| across || Some(crate::scheduler::priority(c)) == tier)
2103 .map(|(c, _)| Arc::clone(c))
2104 .collect();
2105 out.sort_by(|a, b| a.id.cmp(&b.id));
2107 Ok(out)
2108 }
2109
2110 pub(crate) fn select_picked(
2112 self: &Arc<Self>,
2113 selection: Selection,
2114 policy: Arc<Policy>,
2115 admit: &Admit<'_>,
2116 ranks: &crate::scheduler::Ranks<'_>,
2117 pick: Pick,
2118 ) -> Result<Lease, AcquireError> {
2119 let now = Instant::now();
2120 let credential = {
2121 let inner = self.read();
2122 let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2123 let configured = matches!(pick, Pick::Configured);
2124 let (eligible, candidates) = ready(&inner.creds, &scheduler, &selection, &policy, admit, now, configured)?;
2125 let refs: Vec<(&Credential, &str)> = candidates.iter().map(|(c, p)| (c.as_ref(), p.as_str())).collect();
2126 let chosen = match &pick {
2127 Pick::Auth(id) => refs.iter().find(|(c, _)| c.id == *id).map(|(c, _)| (*c, None)),
2131 Pick::Builtin(strategy) => scheduler
2132 .pick_builtin(&refs, &selection, *strategy, ranks)
2133 .map(|c| (c, None)),
2134 Pick::Configured => None,
2135 };
2136 let (picked, lcp) = match chosen {
2137 Some(chosen) => chosen,
2138 None => {
2139 let refs: Vec<_> = refs.into_iter().filter(|(c, _)| scheduler.admits(c, &policy)).collect();
2140 if refs.is_empty() {
2141 return Err(ready(&inner.creds, &scheduler, &selection, &policy, admit, now, true)
2143 .expect_err("no admitted candidate is ready"));
2144 }
2145 scheduler.pick_session(&refs, &selection, &policy, ranks, now).unwrap()
2146 }
2147 };
2148 let (c, m) = eligible.iter().find(|(c, _)| c.id == picked.id).unwrap();
2149 let generation = scheduler.reserve_probe(c, m, now);
2150 (Arc::clone(c), lcp, generation, scheduler.dispatch())
2151 };
2152 let (credential, lcp, picked, seq) = credential;
2153 let execution_model = admit(&credential).unwrap_or_else(|| selection.model.clone());
2154 Ok(Lease {
2155 store: self.clone(),
2156 credential,
2157 selection,
2158 execution_model,
2159 attempt: self.attempts.fetch_add(1, Ordering::Relaxed),
2160 policy,
2161 reported: false,
2162 picked,
2163 seq,
2164 remote: None,
2165 lcp,
2166 })
2167 }
2168
2169 pub fn suspension(&self, credential: &Credential, model: &str) -> crate::registry::Suspension {
2171 use crate::registry::Suspension;
2172 let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2173 let now = Instant::now();
2174 let live = |m: &str| {
2175 scheduler
2176 .cooldowns
2177 .get(&(credential.id.clone(), crate::scheduler::canonical_model(m).to_owned()))
2178 .filter(|s| s.deadline > now)
2179 };
2180 if let Some(state) = live(model) {
2181 return match (state.quota, state.status) {
2182 (true, 429) => Suspension::Quota,
2183 (quota_exceeded, _) => Suspension::Other { quota_exceeded },
2184 };
2185 }
2186 if live("").is_some() {
2187 return Suspension::Other { quota_exceeded: false };
2189 }
2190 Suspension::None
2191 }
2192
2193 pub fn blocked(&self, credential: &Credential, model: &str) -> bool {
2197 let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2198 scheduler.cooling(credential, model, Instant::now()) || scheduler.tokens_rejected(credential)
2199 }
2200
2201 pub(crate) fn reject_refused(&self, refused: &Credential, registration: u64, error: &ExecError) -> bool {
2209 if !crate::classify::is_invalid_grant(error.status, &String::from_utf8_lossy(&error.body)) {
2210 return false;
2211 }
2212 let inner = self.read();
2213 let Some(current) = inner.creds.iter().find(|c| c.id == refused.id) else {
2214 return false;
2215 };
2216 let token = effective_access_token(refused);
2217 if current.disabled
2218 || self.registered(¤t.id).registration != registration
2219 || token.is_empty()
2220 || effective_access_token(current) != token
2221 || credentials_changed(refused, current)
2222 {
2223 return false;
2224 }
2225 self.scheduler
2226 .lock()
2227 .unwrap_or_else(PoisonError::into_inner)
2228 .reject_tokens(current.clone());
2229 true
2230 }
2231
2232 pub fn tokens_rejected(&self, c: &Credential) -> bool {
2234 self.scheduler
2235 .lock()
2236 .unwrap_or_else(PoisonError::into_inner)
2237 .tokens_rejected(c)
2238 }
2239
2240 fn rejected_ids(&self, snapshot: &[Arc<Credential>]) -> std::collections::HashSet<String> {
2242 let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2243 snapshot
2244 .iter()
2245 .filter(|c| scheduler.tokens_rejected(c))
2246 .map(|c| c.id.clone())
2247 .collect()
2248 }
2249
2250 pub fn reset_cooldown(&self, id: &str) -> Vec<String> {
2253 let models = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner).reset(id);
2254 if !models.is_empty() {
2255 self.persist_cooldowns();
2256 }
2257 models
2258 }
2259
2260 pub fn retry_wait(
2265 &self,
2266 selection: &Selection,
2267 policy: &Policy,
2268 status: u16,
2269 attempted: &[String],
2270 admit: &Admit<'_>,
2271 ) -> Option<Duration> {
2272 self.retry_wait_at(selection, policy, status, attempted, admit, Instant::now())
2273 }
2274
2275 fn retry_wait_at(
2276 &self,
2277 selection: &Selection,
2278 policy: &Policy,
2279 status: u16,
2280 attempted: &[String],
2281 admit: &Admit<'_>,
2282 now: Instant,
2283 ) -> Option<Duration> {
2284 let inner = self.read();
2285 let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2286 let wait = inner
2287 .creds
2288 .iter()
2289 .filter(|c| !c.disabled)
2290 .filter(|c| policy.retry_limit(c) > selection.retry_round && scheduler.admits(c, policy))
2291 .filter_map(|c| admit(c).map(|m| (c, m)))
2292 .filter(|(c, m)| scheduler.retry_eligible(c, m, now) && !scheduler.tokens_rejected(c))
2293 .map(|(c, m)| {
2294 let wait = scheduler.wait(c, &m, now).unwrap_or_default();
2295 if status == 429 && attempted.contains(&c.id) && !policy.cooling_disabled(c) {
2296 wait.max(Duration::from_secs(10))
2297 } else {
2298 wait
2299 }
2300 })
2301 .min()?;
2302 if wait > policy.max_retry_interval {
2303 None
2304 } else {
2305 Some(wait)
2306 }
2307 }
2308
2309 fn accept_probe(&self, lease: &Lease) {
2310 let inner = self.read();
2311 if !inner
2313 .creds
2314 .iter()
2315 .any(|c| c.id == lease.credential.id && c.revision == lease.credential.revision)
2316 {
2317 return;
2318 }
2319 self.scheduler
2320 .lock()
2321 .unwrap_or_else(PoisonError::into_inner)
2322 .accept_probe_picked(&lease.credential, &lease.execution_model, lease.picked, Instant::now());
2323 }
2324
2325 fn record(&self, lease: &Lease, outcome: &Outcome) {
2326 self.record_model(lease, &lease.execution_model, outcome);
2327 }
2328
2329 fn record_model(&self, lease: &Lease, model: &str, outcome: &Outcome) {
2330 let slot = match outcome {
2331 Outcome::Success => 0,
2332 Outcome::Failure(_) | Outcome::Neutral(_) => 1,
2333 Outcome::Cancelled => 2,
2334 };
2335 self.stats[slot].fetch_add(1, Ordering::Relaxed);
2336 if !matches!(outcome, Outcome::Cancelled) {
2337 self.note_activity(&lease.credential.id, matches!(outcome, Outcome::Success));
2338 }
2339 let inner = self.read();
2340 if inner
2343 .creds
2344 .iter()
2345 .any(|c| c.id == lease.credential.id && c.revision == lease.credential.revision)
2346 {
2347 let (now, wall) = (Instant::now(), std::time::SystemTime::now());
2348 let persist = self
2349 .cooldown_dir
2350 .read()
2351 .unwrap_or_else(PoisonError::into_inner)
2352 .is_some();
2353 let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2354 let before = persist.then(|| scheduler.records(&lease.credential, now, wall));
2355 scheduler.record_dispatched(
2356 &lease.credential,
2357 model,
2358 outcome,
2359 &lease.policy,
2360 lease.picked,
2361 lease.seq,
2362 now,
2363 );
2364 scheduler.session_result(
2365 &lease.credential,
2366 &lease.selection,
2367 outcome,
2368 &lease.policy,
2369 lease.lcp.as_ref(),
2370 now,
2371 );
2372 let rejected = matches!(outcome, Outcome::Failure(error)
2378 if lease.credential.provider == "codex"
2379 && crate::classify::names_unauthorized(error)
2380 && lease.policy.error_action(&lease.credential, error).cooldown);
2381 if let (Outcome::Failure(error) | Outcome::Neutral(error), Some(queue)) = (outcome, self.error_events.get())
2383 && queue.wants_errors()
2384 {
2385 let records = scheduler.records(&lease.credential, now, wall);
2386 queue.enqueue_error(&crate::error_events::payload(&lease.credential, model, error, &records));
2387 }
2388 let changed = before.is_some_and(|before| before != scheduler.records(&lease.credential, now, wall));
2390 drop(scheduler);
2391 drop(inner);
2392 if rejected {
2394 let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2395 let revision = lease.credential.revision;
2396 if state
2398 .pending
2399 .get(&lease.credential.id)
2400 .is_none_or(|(r, _)| *r != revision)
2401 {
2402 state.pending.insert(lease.credential.id.clone(), (revision, now));
2403 self.unauthorized_noted.send_replace(());
2404 }
2405 }
2406 if changed {
2407 self.persist_cooldowns();
2408 }
2409 self.persist_credential(&lease.credential);
2411 }
2412 }
2413
2414 pub fn subscribe(&self) -> tokio::sync::watch::Receiver<()> {
2416 self.changed.subscribe()
2417 }
2418
2419 pub(crate) fn unauthorized(&self) -> Rejections {
2422 self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner).clone()
2423 }
2424
2425 pub(crate) fn forget_unauthorized(&self, id: &str, revision: u64) {
2427 let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2428 if state.pending.get(id).is_some_and(|(r, _)| *r <= revision) {
2429 state.pending.remove(id);
2430 }
2431 }
2432
2433 pub(crate) fn forget_stale_unauthorized(&self, seen: &Rejections, snapshot: &[Arc<Credential>]) {
2436 for (id, (revision, _)) in &seen.pending {
2437 if snapshot
2438 .iter()
2439 .find(|c| c.id == *id)
2440 .is_none_or(|c| c.revision > *revision)
2441 {
2442 self.forget_unauthorized(id, *revision);
2443 }
2444 }
2445 let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2446 state
2447 .refreshed
2448 .retain(|id, _| !seen.refreshed.contains_key(id) || snapshot.iter().any(|c| c.id == *id));
2449 }
2450
2451 pub(crate) fn apply_refresh(
2459 &self,
2460 id: &str,
2461 expected_revision: u64,
2462 fence: RefreshFence,
2463 patch: &MetadataPatch,
2464 edit: impl FnOnce(&mut Credential),
2465 rotated: impl FnOnce(&Credential) -> bool,
2466 ) -> Result<(Arc<Credential>, bool), PatchError> {
2467 let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2468 let committed = self.apply_patch_with(id, expected_revision, Some(fence), patch, edit)?;
2469 let rotated = rotated(&committed);
2470 if rotated {
2471 state
2472 .refreshed
2473 .insert(id.to_owned(), (committed.revision, Instant::now()));
2474 }
2475 Ok((committed, rotated))
2476 }
2477
2478 fn admit_codex_refresh(
2488 &self,
2489 original: &Credential,
2490 registered: crate::plugins::refresh::Registered,
2491 patch: &cpa_exec::codex_refresh::RefreshPatch,
2492 ) -> Result<CodexAdmission, PatchError> {
2493 let id = original.id.as_str();
2494 let committed = {
2495 let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2496 let committed = loop {
2497 let revision = self.get(id).ok_or(PatchError::NotFound)?.revision;
2498 let mut discarded = None;
2499 let admitted = self.write_unlocked(
2503 id,
2504 revision,
2505 None,
2506 false,
2507 |current| {
2508 let now = self.registered(id);
2509 if now.registration != registered.registration {
2510 return Err(PatchError::NotFound);
2511 }
2512 let moved = now.credential_version != registered.credential_version;
2513 if (moved || credentials_changed(original, current)) && !patch.can_apply_to(¤t.metadata) {
2514 discarded = Some(current.clone());
2515 return Ok(None);
2516 }
2517 let Source::File(path) = ¤t.source else {
2518 return Err(PatchError::ConfigBacked);
2519 };
2520 let (out, written) = save_codex_refresh(path, current, patch);
2521 let Ok(Value::Object(metadata)) = serde_json::from_slice::<Value>(&out) else {
2522 unreachable!("Go's encoding of a JSON object");
2523 };
2524 let mut next = Credential::clone(current);
2525 next.metadata = metadata;
2526 next.refresh_derived();
2527 publish_metadata_proxy(&mut next);
2528 Ok(Some((None, out, (next, written))))
2529 },
2530 |inner, index, bytes, (mut next, written)| {
2531 inner.generation += 1;
2532 next.revision = inner.generation;
2533 if written {
2536 self.note_provenance(&next.id, next.revision, sha256(bytes), None);
2537 } else {
2538 self.provenance
2539 .lock()
2540 .unwrap_or_else(PoisonError::into_inner)
2541 .remove(&next.id);
2542 }
2543 let next = Arc::new(next);
2544 self.register(
2545 std::slice::from_ref(&inner.creds[index]),
2546 std::slice::from_ref(&next),
2547 &[],
2548 );
2549 inner.creds[index] = next;
2550 inner.epoch += 1;
2551 self.changed.send_replace(());
2552 inner.creds[index].clone()
2553 },
2554 );
2555 match admitted {
2556 Err(PatchError::Stale { .. }) => continue,
2558 Ok(None) => {
2559 let current = discarded.expect("only a discard writes nothing");
2560 return Ok(CodexAdmission::Discarded(current));
2561 }
2562 other => break other?.expect("checked above"),
2563 }
2564 };
2565 if credentials_changed(original, &committed) {
2566 state
2567 .refreshed
2568 .insert(id.to_owned(), (committed.revision, Instant::now()));
2569 }
2570 committed
2571 };
2572 if credentials_changed(original, &committed) {
2573 self.recover_unauthorized(id);
2574 }
2575 Ok(CodexAdmission::Committed(committed))
2576 }
2577
2578 pub(crate) fn subscribe_unauthorized(&self) -> tokio::sync::watch::Receiver<()> {
2580 self.unauthorized_noted.subscribe()
2581 }
2582
2583 pub fn epoch(&self) -> u64 {
2585 self.read().epoch
2586 }
2587
2588 pub fn stats(&self) -> AttemptStats {
2590 let get = |i: usize| self.stats[i].load(Ordering::Relaxed);
2591 AttemptStats {
2592 success: get(0),
2593 failure: get(1),
2594 cancelled: get(2),
2595 }
2596 }
2597
2598 pub fn prepare_lock(&self, id: &str) -> Arc<tokio::sync::Mutex<()>> {
2600 let mut locks = self.prepare_locks.lock().unwrap_or_else(PoisonError::into_inner);
2601 locks.entry(id.to_owned()).or_default().clone()
2602 }
2603
2604 pub fn apply_patch(
2608 &self,
2609 id: &str,
2610 expected_revision: u64,
2611 patch: &MetadataPatch,
2612 ) -> Result<Arc<Credential>, PatchError> {
2613 self.apply_patch_with(id, expected_revision, None, patch, |_| {})
2614 }
2615
2616 fn apply_patch_with(
2620 &self,
2621 id: &str,
2622 expected_revision: u64,
2623 fence: Option<RefreshFence>,
2624 patch: &MetadataPatch,
2625 edit: impl FnOnce(&mut Credential),
2626 ) -> Result<Arc<Credential>, PatchError> {
2627 let committed = self.write_unlocked(
2628 id,
2629 expected_revision,
2630 fence.map(|f| f.registered),
2631 false,
2632 |current| {
2633 if patch.remove.iter().any(|k| k == "type")
2634 || patch
2635 .set
2636 .get("type")
2637 .is_some_and(|t| t.as_str() != Some(current.provider.as_str()))
2638 {
2639 return Err(PatchError::TypeIsImmutable);
2640 }
2641 let Source::File(path) = ¤t.source else {
2642 return Err(PatchError::ConfigBacked);
2643 };
2644 let mut next = Credential::clone(current);
2645 patch.apply(&mut next.metadata);
2646 next.refresh_derived();
2647 edit(&mut next);
2648 next.metadata.insert("disabled".into(), Value::Bool(next.disabled));
2650 let mut bytes = serde_json::to_vec(&next.metadata).map_err(|e| PatchError::Io(e.to_string()))?;
2652 bytes.push(b'\n');
2653 let basis = self.patched_doc(current, path, patch, next.disabled);
2654 Ok(Some((Some(path.clone()), bytes, (next, basis))))
2655 },
2656 |inner, index, bytes, (mut next, basis)| {
2657 inner.generation += 1;
2658 next.revision = inner.generation;
2659 match basis {
2662 Some((doc, loaded)) => {
2663 let provenance = Provenance {
2664 revision: next.revision,
2665 sha256: sha256(bytes),
2666 doc: Some(Arc::new(doc)),
2667 loaded,
2668 };
2669 self.provenance
2670 .lock()
2671 .unwrap_or_else(PoisonError::into_inner)
2672 .insert(next.id.clone(), provenance);
2673 }
2674 None => {
2675 self.provenance
2676 .lock()
2677 .unwrap_or_else(PoisonError::into_inner)
2678 .remove(&next.id);
2679 }
2680 }
2681 let next = Arc::new(next);
2682 self.register(
2683 std::slice::from_ref(&inner.creds[index]),
2684 std::slice::from_ref(&next),
2685 &[],
2686 );
2687 if fence.is_some_and(|f| f.plugin) {
2692 crate::plugins::refresh::record(&self.refresh_schedules, &next, |c, s| {
2693 crate::plugins::refresh::committed(c, s, chrono::Utc::now())
2694 });
2695 }
2696 inner.creds[index] = next;
2697 inner.epoch += 1;
2698 self.changed.send_replace(());
2699 inner.creds[index].clone()
2700 },
2701 )?;
2702 Ok(committed.expect("apply_patch always writes"))
2703 }
2704
2705 fn write_unlocked<T, R>(
2719 &self,
2720 id: &str,
2721 expected_revision: u64,
2722 registered: Option<crate::plugins::refresh::Registered>,
2723 exclusive: bool,
2724 prepare: impl FnOnce(&Arc<Credential>) -> Result<Option<(Option<std::path::PathBuf>, Vec<u8>, T)>, PatchError>,
2725 publish: impl FnOnce(&mut Inner, usize, &[u8], T) -> R,
2726 ) -> Result<Option<R>, PatchError> {
2727 let lock = (!exclusive).then(|| {
2730 let mut locks = self.write_locks.lock().unwrap_or_else(PoisonError::into_inner);
2731 locks.entry(id.to_owned()).or_default().clone()
2732 });
2733 let _gate = if exclusive {
2734 (None, Some(self.reload.write().unwrap_or_else(PoisonError::into_inner)))
2735 } else {
2736 (Some(self.reload.read().unwrap_or_else(PoisonError::into_inner)), None)
2737 };
2738 let _own = lock.as_ref().map(|l| l.lock().unwrap_or_else(PoisonError::into_inner));
2739 let current = self
2740 .read()
2741 .creds
2742 .iter()
2743 .find(|c| c.id == id)
2744 .cloned()
2745 .ok_or(PatchError::NotFound)?;
2746 if let Some(registered) = registered {
2749 let now = self.registered(id);
2750 if now.registration != registered.registration {
2751 return Err(PatchError::NotFound);
2752 }
2753 if now.credential_version != registered.credential_version {
2754 return Err(PatchError::Stale {
2755 current: current.revision,
2756 });
2757 }
2758 }
2759 if current.revision != expected_revision {
2760 return Err(PatchError::Stale {
2761 current: current.revision,
2762 });
2763 }
2764 let Some((path, bytes, value)) = prepare(¤t)? else {
2765 return Ok(None);
2766 };
2767 if let Some(path) = path.filter(|_| !marked(¤t, "plugin_virtual")) {
2771 write_bytes_atomic(&path, &bytes).map_err(|e| PatchError::Io(e.to_string()))?;
2772 }
2773 let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
2774 let index = inner
2776 .creds
2777 .iter()
2778 .position(|c| c.id == id && c.revision == expected_revision)
2779 .ok_or(PatchError::NotFound)?;
2780 Ok(Some(publish(&mut inner, index, &bytes, value)))
2781 }
2782
2783 pub fn replace_config_backed(
2788 &self,
2789 next: Credential,
2790 expected_revision: u64,
2791 ) -> Result<Arc<Credential>, PatchError> {
2792 let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
2793 let generation = inner.generation + 1;
2794 let slot = inner
2795 .creds
2796 .iter_mut()
2797 .find(|c| c.id == next.id && matches!(c.source, Source::Config { .. } | Source::Runtime))
2798 .ok_or(PatchError::NotFound)?;
2799 if slot.revision != expected_revision {
2800 return Err(PatchError::Stale { current: slot.revision });
2801 }
2802 let mut next = next;
2803 next.source = slot.source.clone();
2804 next.revision = generation;
2805 let next = Arc::new(next);
2806 self.register(std::slice::from_ref(slot), std::slice::from_ref(&next), &[]);
2807 let cleared = self.clear_replaced_unauthorized(std::slice::from_ref(slot), std::slice::from_ref(&next));
2808 *slot = next;
2809 let committed = slot.clone();
2810 inner.generation = generation;
2811 inner.epoch += 1;
2814 self.changed.send_replace(());
2815 drop(inner);
2816 if cleared {
2817 self.persist_cooldowns();
2818 }
2819 self.clear_disabled_cooldowns();
2822 Ok(committed)
2823 }
2824
2825 pub fn replace_file_backed(
2832 &self,
2833 id: &str,
2834 expected_revision: u64,
2835 bytes: &[u8],
2836 next: Option<Credential>,
2837 ) -> Result<(), PatchError> {
2838 let commit = Commit {
2839 bytes: bytes.to_vec(),
2840 write: true,
2841 disk: None,
2842 doc: None,
2843 };
2844 self.commit_file_backed(id, expected_revision, commit, next)
2845 }
2846
2847 pub fn commit_file_backed(
2851 &self,
2852 id: &str,
2853 expected_revision: u64,
2854 commit: Commit,
2855 next: Option<Credential>,
2856 ) -> Result<(), PatchError> {
2857 let Commit {
2858 bytes,
2859 write,
2860 disk,
2861 doc,
2862 } = commit;
2863 let exclusive = next.as_ref().is_some_and(|n| n.id != id);
2865 let removed = self.write_unlocked(
2866 id,
2867 expected_revision,
2868 None,
2869 exclusive,
2870 |current| {
2871 let Source::File(path) = ¤t.source else {
2872 return Err(PatchError::ConfigBacked);
2873 };
2874 if let Some(want) = disk {
2875 let now = std::fs::read(path).ok().map(|b| sha256(&b));
2876 if now != want {
2877 return Err(PatchError::Stale {
2878 current: current.revision,
2879 });
2880 }
2881 }
2882 let held = if write { Some(sha256(&bytes)) } else { disk.flatten() };
2884 Ok(Some((write.then(|| path.clone()), bytes, (next, held))))
2885 },
2886 |inner, index, _, (next, held)| {
2887 let before = inner.creds.clone();
2888 let previous = inner.creds.remove(index);
2889 self.provenance
2890 .lock()
2891 .unwrap_or_else(PoisonError::into_inner)
2892 .remove(&previous.id);
2893 if let Some(mut next) = next {
2894 inner.generation += 1;
2895 next.revision = inner.generation;
2896 if let Some(held) = held {
2897 self.note_provenance(&next.id, next.revision, held, doc);
2898 }
2899 inner.creds.retain(|c| c.id != next.id);
2900 let at = index.min(inner.creds.len());
2901 inner.creds.insert(at, Arc::new(next));
2902 }
2903 self.register(&before, &inner.creds, &[]);
2904 self.clear_replaced_unauthorized(&before, &inner.creds);
2905 inner.epoch += 1;
2906 self.changed.send_replace(());
2907 self.scheduler
2908 .lock()
2909 .unwrap_or_else(PoisonError::into_inner)
2910 .reconcile(&inner.creds);
2911 removed_ids(&[previous], &inner.creds)
2912 },
2913 )?;
2914 self.close_sessions(removed.unwrap_or_default());
2915 self.clear_disabled_cooldowns();
2916 self.persist_cooldowns();
2917 Ok(())
2918 }
2919
2920 fn patched_doc(
2926 &self,
2927 current: &Credential,
2928 path: &std::path::Path,
2929 patch: &MetadataPatch,
2930 disabled: bool,
2931 ) -> Option<(cpa_common::json::GoValue, bool)> {
2932 use cpa_common::json::GoValue;
2933 let prior = self
2934 .provenance(¤t.id)
2935 .filter(|p| p.revision == current.revision)?;
2936 let (base, loaded) = match prior.doc {
2937 Some(doc) => (GoValue::clone(&doc), prior.loaded),
2938 None => {
2939 let bytes = std::fs::read(path).ok().filter(|b| sha256(b) == prior.sha256)?;
2940 (GoValue::parse(&bytes)?, true)
2941 }
2942 };
2943 let mut doc = base.into_object().ok()?;
2944 for key in &patch.remove {
2945 doc.remove(key);
2946 }
2947 for (key, value) in &patch.set {
2948 doc.insert(key.clone(), go_typed(value, loaded));
2949 }
2950 doc.insert("disabled".into(), GoValue::Bool(disabled));
2951 Some((GoValue::Object(doc), loaded))
2952 }
2953
2954 pub fn provenance(&self, id: &str) -> Option<Provenance> {
2956 self.provenance
2957 .lock()
2958 .unwrap_or_else(PoisonError::into_inner)
2959 .get(id)
2960 .cloned()
2961 }
2962
2963 fn note_provenance(&self, id: &str, revision: u64, sha256: [u8; 32], doc: Option<Arc<cpa_common::json::GoValue>>) {
2964 self.provenance.lock().unwrap_or_else(PoisonError::into_inner).insert(
2965 id.to_owned(),
2966 Provenance {
2967 revision,
2968 sha256,
2969 doc,
2970 loaded: false,
2971 },
2972 );
2973 }
2974
2975 pub fn note_loaded(&self, read: &std::collections::BTreeMap<std::path::PathBuf, [u8; 32]>) {
2981 let creds = self.snapshot();
2982 let mut provenance = self.provenance.lock().unwrap_or_else(PoisonError::into_inner);
2983 provenance.retain(|id, _| creds.iter().any(|c| c.id == *id));
2984 for c in creds.iter() {
2985 let Source::File(path) = &c.source else { continue };
2986 let Some(&hash) = read.get(path) else { continue };
2987 let same = provenance
2988 .get(&c.id)
2989 .is_some_and(|p| p.revision == c.revision && p.sha256 == hash);
2990 if !same {
2991 provenance.insert(
2992 c.id.clone(),
2993 Provenance {
2994 revision: c.revision,
2995 sha256: hash,
2996 doc: None,
2997 loaded: false,
2998 },
2999 );
3000 }
3001 }
3002 }
3003
3004 fn close_sessions(&self, removed: Vec<String>) {
3007 if let Some(close) = self.session_closer.get() {
3008 for id in removed {
3009 close(&id);
3010 }
3011 }
3012 }
3013
3014 pub fn reconcile(&self, credentials: Vec<Credential>) {
3017 self.reconcile_scanned(credentials, &std::collections::BTreeMap::new());
3018 }
3019
3020 pub fn reconcile_scanned(
3031 &self,
3032 credentials: Vec<Credential>,
3033 read: &std::collections::BTreeMap<std::path::PathBuf, [u8; 32]>,
3034 ) -> Vec<std::path::PathBuf> {
3035 self.reconcile_scanned_with(credentials, read, |_, _| {})
3036 }
3037
3038 pub fn reconcile_scanned_with(
3041 &self,
3042 credentials: Vec<Credential>,
3043 read: &std::collections::BTreeMap<std::path::PathBuf, [u8; 32]>,
3044 keep: impl FnOnce(&[Arc<Credential>], &mut Vec<Credential>),
3045 ) -> Vec<std::path::PathBuf> {
3046 use sha2::{Digest, Sha256};
3047 let reload = self.reload.write().unwrap_or_else(PoisonError::into_inner);
3048 let mut credentials = credentials;
3049 keep(&self.snapshot(), &mut credentials);
3050 let built: std::collections::HashSet<std::path::PathBuf> = credentials
3053 .iter()
3054 .filter(|c| c.revision == 0)
3055 .filter_map(|c| match &c.source {
3056 Source::File(path) => Some(path.clone()),
3057 _ => None,
3058 })
3059 .collect();
3060 let mut rejected = Vec::new();
3061 if !read.is_empty() {
3062 let current = self.snapshot();
3063 let by_id: HashMap<&str, &Arc<Credential>> = current.iter().map(|c| (c.id.as_str(), c)).collect();
3064 for cred in &mut credentials {
3065 let Source::File(path) = &cred.source else { continue };
3066 let Some(seen) = read.get(path) else { continue };
3067 let Some(&existing) = by_id.get(cred.id.as_str()) else {
3068 continue;
3069 };
3070 if unchanged(existing, cred) {
3071 continue;
3072 }
3073 let on_disk: Option<[u8; 32]> = cpa_core::config::credentials::read_regular(path)
3076 .filter(|data| !data.is_empty())
3077 .map(|data| Sha256::digest(&data).into());
3078 if on_disk.as_ref() != Some(seen) {
3079 rejected.push(path.clone());
3080 *cred = Credential::clone(existing);
3081 }
3082 }
3083 }
3084 let mut guard = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3085 let inner = &mut *guard;
3086 let mut next = Vec::with_capacity(credentials.len());
3087 let mut by_id = HashMap::with_capacity(inner.creds.len());
3088 for c in &inner.creds {
3089 by_id.entry(c.id.as_str()).or_insert(c);
3090 }
3091 let provisional = crate::plugins::refresh::provisional(&self.refresh_schedules);
3095 for mut cred in credentials {
3096 let same = |c: &Credential| {
3097 unchanged(c, &cred)
3098 && !(provisional.contains(c.id.as_str()) && c.next_refresh_after != cred.next_refresh_after)
3099 };
3100 match by_id.get(cred.id.as_str()).filter(|c| same(c)) {
3101 Some(&existing) => next.push(existing.clone()),
3102 None => {
3103 inner.generation += 1;
3104 cred.revision = inner.generation;
3105 next.push(Arc::new(cred));
3106 }
3107 }
3108 }
3109 let runtime: Vec<_> = inner
3112 .creds
3113 .iter()
3114 .filter(|c| c.source == Source::Runtime && !next.iter().any(|n| n.id == c.id))
3115 .cloned()
3116 .collect();
3117 next.extend(runtime);
3118 let removed = removed_ids(&inner.creds, &next);
3119 self.register(&inner.creds, &next, &[]);
3120 self.clear_replaced_unauthorized(&inner.creds, &next);
3121 inner.creds = next;
3122 inner.epoch += 1;
3123 self.changed.send_replace(());
3124 self.scheduler
3125 .lock()
3126 .unwrap_or_else(PoisonError::into_inner)
3127 .reconcile(&inner.creds);
3128 drop(guard);
3129 let mut loaded = read.clone();
3131 loaded.retain(|path, _| built.contains(path) && !rejected.contains(path));
3132 self.note_loaded(&loaded);
3133 drop(reload);
3134 self.close_sessions(removed);
3135 self.clear_disabled_cooldowns();
3136 self.persist_cooldowns();
3137 rejected
3138 }
3139
3140 pub fn replace_config(&self, keys: Vec<Credential>) {
3146 let reload = self.reload.write().unwrap_or_else(PoisonError::into_inner);
3147 let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3148 let mut next: Vec<_> = inner
3149 .creds
3150 .iter()
3151 .filter(|c| !matches!(c.source, Source::Config { .. }))
3152 .cloned()
3153 .collect();
3154 for mut key in keys {
3155 match inner
3156 .creds
3157 .iter()
3158 .find(|c| c.id == key.id && unchanged(c, &key))
3159 .cloned()
3160 {
3161 Some(existing) => next.push(existing),
3162 None => {
3163 inner.generation += 1;
3164 key.revision = inner.generation;
3165 next.push(Arc::new(key));
3166 }
3167 }
3168 }
3169 let removed = removed_ids(&inner.creds, &next);
3170 self.register(&inner.creds, &next, &[]);
3171 self.clear_replaced_unauthorized(&inner.creds, &next);
3172 inner.creds = next;
3173 inner.epoch += 1;
3174 self.changed.send_replace(());
3175 self.scheduler
3176 .lock()
3177 .unwrap_or_else(PoisonError::into_inner)
3178 .reconcile(&inner.creds);
3179 drop(inner);
3180 drop(reload);
3181 self.close_sessions(removed);
3182 self.clear_disabled_cooldowns();
3183 self.persist_cooldowns();
3184 }
3185
3186 pub fn upsert(&self, credential: Credential) {
3191 self.upsert_file(None, vec![credential], &[]);
3192 }
3193
3194 pub fn upsert_file(
3199 &self,
3200 path: Option<&std::path::Path>,
3201 credentials: Vec<Credential>,
3202 saved: &[cpa_plugin::hostauth::SavedRefresh],
3203 ) {
3204 let reload = self.reload.write().unwrap_or_else(PoisonError::into_inner);
3205 let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3206 let before = inner.creds.clone();
3207 let mut changed = false;
3208 let mut removed = Vec::new();
3209 if let Some(path) = path {
3210 let clean = cpa_core::config::credentials::absolute_clean_path;
3211 let target = clean(path);
3212 let before = std::mem::take(&mut inner.creds);
3213 for credential in before {
3214 let stale = !marked(&credential, "runtime_only")
3215 && !credentials.iter().any(|c| c.id == credential.id)
3216 && matches!(&credential.source, Source::File(p) if clean(p) == target);
3217 if stale {
3218 removed.push(credential.id.clone());
3219 } else {
3220 inner.creds.push(credential);
3221 }
3222 }
3223 changed = !removed.is_empty();
3224 }
3225 for mut credential in credentials {
3226 let slot = inner.creds.iter().position(|c| c.id == credential.id);
3227 if slot.is_some_and(|i| unchanged(&inner.creds[i], &credential)) {
3228 continue;
3229 }
3230 inner.generation += 1;
3231 credential.revision = inner.generation;
3232 let credential = Arc::new(credential);
3233 match slot {
3234 Some(i) => inner.creds[i] = credential,
3235 None => inner.creds.push(credential),
3236 }
3237 changed = true;
3238 }
3239 if !changed {
3240 return;
3241 }
3242 self.register(&before, &inner.creds, saved);
3243 self.clear_replaced_unauthorized(&before, &inner.creds);
3244 inner.epoch += 1;
3245 self.changed.send_replace(());
3246 self.scheduler
3247 .lock()
3248 .unwrap_or_else(PoisonError::into_inner)
3249 .reconcile(&inner.creds);
3250 drop(inner);
3251 drop(reload);
3252 self.close_sessions(removed);
3253 self.clear_disabled_cooldowns();
3254 self.persist_cooldowns();
3255 }
3256
3257 pub fn add_runtime(&self, mut credential: Credential) -> bool {
3260 let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3261 if inner.creds.iter().any(|c| c.id == credential.id && !c.disabled) {
3262 return false;
3263 }
3264 let before = inner.creds.clone();
3265 inner.creds.retain(|c| c.id != credential.id);
3266 inner.generation += 1;
3267 credential.revision = inner.generation;
3268 inner.creds.push(Arc::new(credential));
3269 self.register(&before, &inner.creds, &[]);
3270 inner.epoch += 1;
3271 self.changed.send_replace(());
3272 self.scheduler
3273 .lock()
3274 .unwrap_or_else(PoisonError::into_inner)
3275 .reconcile(&inner.creds);
3276 true
3277 }
3278
3279 pub fn remove_runtime(&self, id: &str) {
3282 let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3283 let before = inner.creds.clone();
3284 inner.creds.retain(|c| !(c.id == id && c.source == Source::Runtime));
3285 if inner.creds.len() == before.len() {
3286 return;
3287 }
3288 self.register(&before, &inner.creds, &[]);
3289 inner.epoch += 1;
3290 self.changed.send_replace(());
3291 self.scheduler
3292 .lock()
3293 .unwrap_or_else(PoisonError::into_inner)
3294 .reconcile(&inner.creds);
3295 }
3296
3297 fn read(&self) -> std::sync::RwLockReadGuard<'_, Inner> {
3298 self.inner.read().unwrap_or_else(PoisonError::into_inner)
3299 }
3300}
3301
3302#[derive(Debug, Clone, Default, PartialEq, Eq)]
3305pub struct CredentialActivity {
3306 pub success: u64,
3307 pub failed: u64,
3308 pub recent: Vec<(i64, u64, u64)>,
3311}
3312
3313pub const RECENT_BUCKET_SECONDS: i64 = 600;
3314const RECENT_BUCKETS: usize = 20;
3315
3316impl CredentialStore {
3317 fn note_activity(&self, id: &str, success: bool) {
3318 let bucket = std::time::SystemTime::now()
3319 .duration_since(std::time::UNIX_EPOCH)
3320 .map_or(0, |d| d.as_secs() as i64 / RECENT_BUCKET_SECONDS);
3321 let mut all = self.activity.lock().unwrap_or_else(PoisonError::into_inner);
3322 let entry = all.entry(id.to_owned()).or_default();
3323 if success {
3324 entry.success += 1;
3325 } else {
3326 entry.failed += 1;
3327 }
3328 match entry.recent.last_mut() {
3329 Some(last) if last.0 == bucket => {
3330 if success {
3331 last.1 += 1;
3332 } else {
3333 last.2 += 1;
3334 }
3335 }
3336 _ => entry.recent.push((bucket, u64::from(success), u64::from(!success))),
3337 }
3338 entry.recent.retain(|(b, _, _)| bucket - b < RECENT_BUCKETS as i64);
3339 }
3340
3341 pub fn activity(&self, id: &str) -> CredentialActivity {
3343 self.activity
3344 .lock()
3345 .unwrap_or_else(PoisonError::into_inner)
3346 .get(id)
3347 .cloned()
3348 .unwrap_or_default()
3349 }
3350
3351 pub fn cooldowns(&self, id: &str) -> Vec<crate::scheduler::CooldownState> {
3353 self.scheduler
3354 .lock()
3355 .unwrap_or_else(PoisonError::into_inner)
3356 .cooldowns_of(id, Instant::now())
3357 }
3358
3359 pub fn reset_cooldowns(&self, id: &str) -> Vec<String> {
3362 let had = !self.cooldowns(id).is_empty();
3363 let models = self
3364 .scheduler
3365 .lock()
3366 .unwrap_or_else(PoisonError::into_inner)
3367 .reset_cooldowns(id);
3368 if had {
3369 self.persist_cooldowns();
3371 }
3372 models
3373 }
3374
3375 pub fn recover_unauthorized(&self, id: &str) {
3380 if self.get(id).is_none_or(|c| c.disabled) {
3381 return;
3382 }
3383 let cleared = self
3384 .scheduler
3385 .lock()
3386 .unwrap_or_else(PoisonError::into_inner)
3387 .clear_unauthorized(id, Instant::now());
3388 if cleared {
3389 self.persist_cooldowns();
3390 }
3391 }
3392}
3393
3394impl Runtime {
3395 pub async fn refresh_credential(&self, id: &str) -> Result<Arc<Credential>, ExecError> {
3398 let cfg = self.config();
3399 let revision = self.store.get(id).map(|c| c.revision);
3400 self.prepare_credential_with(id, &cfg, revision, true, None).await
3401 }
3402}
3403
3404pub(crate) fn commit_refresh(
3408 store: &CredentialStore,
3409 id: &str,
3410 current: &Credential,
3411 mut fence: RefreshFence,
3412 patch: &MetadataPatch,
3413 publish_proxy: impl Fn(&mut Credential) + Copy,
3414) -> Result<Arc<Credential>, PatchError> {
3415 let mut revision = current.revision;
3416 let mut attempt = patch.clone();
3417 loop {
3418 match store.apply_refresh(id, revision, fence, &attempt, publish_proxy, |c| {
3419 credentials_changed(current, c)
3420 }) {
3421 Err(PatchError::Stale { .. }) => {
3429 #[cfg(test)]
3430 if let Some(hook) = tests::AFTER_STALE.with(std::cell::Cell::take) {
3431 hook();
3432 }
3433 let (latest, now) = store.get_registered(id).ok_or(PatchError::NotFound)?;
3437 if now.registration != fence.registered.registration {
3438 return Err(PatchError::NotFound);
3439 }
3440 let (rebased, partition_applies) =
3441 rebase_patch(¤t.provider, ¤t.metadata, &latest.metadata, patch);
3442 let moved = now.credential_version != fence.registered.credential_version;
3446 if (moved || credentials_changed(current, &latest)) && !partition_applies {
3447 return Ok(latest);
3448 }
3449 fence.registered = now;
3451 attempt = rebased;
3452 revision = latest.revision;
3453 if attempt.set.is_empty() && attempt.remove.is_empty() {
3454 let at = (revision, fence.registered.credential_version);
3461 match store.record_refresh(id, fence.registered.registration, Some(at), |c, s| {
3462 if fence.plugin {
3463 crate::plugins::refresh::committed(c, s, chrono::Utc::now());
3464 }
3465 }) {
3466 Err(PatchError::Stale { .. }) => continue,
3467 recorded => return recorded,
3468 }
3469 }
3470 }
3471 other => {
3472 let (committed, rotated) = other?;
3473 if rotated {
3477 store.recover_unauthorized(id);
3478 }
3479 return Ok(committed);
3480 }
3481 }
3482 }
3483}
3484
3485fn rebase_patch(
3495 provider: &str,
3496 base: &Map<String, Value>,
3497 user: &Map<String, Value>,
3498 patch: &MetadataPatch,
3499) -> (MetadataPatch, bool) {
3500 const CODEX_OUTER: [&str; 8] = [
3502 "id_token",
3503 "access_token",
3504 "refresh_token",
3505 "account_id",
3506 "last_refresh",
3507 "email",
3508 "expired",
3509 "plan_type",
3510 ];
3511 let token_payload = |k: &str| {
3513 matches!(
3514 k.trim().to_ascii_lowercase().as_str(),
3515 "access_token"
3516 | "refresh_token"
3517 | "id_token"
3518 | "session_id"
3519 | "expired"
3520 | "last_refresh"
3521 | "expires_in"
3522 | "timestamp"
3523 | "token_type"
3524 | "user_code"
3525 | "verification_uri"
3526 | "verification_uri_complete"
3527 )
3528 };
3529 let codex = provider == "codex";
3530 let kept_by_user = |k: &str| base.get(k) == user.get(k);
3531 let mut out = MetadataPatch::default();
3532 let mut partition_applies = false;
3533 for (k, v) in &patch.set {
3534 let outer = codex && (CODEX_OUTER.contains(&k.as_str()) || k == "siwc");
3535 if !outer && base.get(k) != Some(v) && (kept_by_user(k) || token_payload(k)) {
3536 out.set.insert(k.clone(), v.clone());
3537 }
3538 }
3539 out.remove = patch
3540 .remove
3541 .iter()
3542 .filter(|k| base.contains_key(*k) && kept_by_user(k))
3543 .cloned()
3544 .collect();
3545 if codex {
3546 let fields = |m: &Map<String, Value>, keys: &[&str]| -> Map<String, Value> {
3550 keys.iter()
3551 .filter_map(|k| m.get(*k).map(|v| ((*k).to_owned(), v.clone())))
3552 .collect()
3553 };
3554 let mut updated = base.clone();
3555 patch.apply(&mut updated);
3556 let same = |current: &Map<String, Value>, b: &Map<String, Value>| {
3558 [
3559 "access_token",
3560 "refresh_token",
3561 "id_token",
3562 "account_id",
3563 "client_id",
3564 "device_id",
3565 ]
3566 .iter()
3567 .all(|k| current.get(*k).filter(|v| !v.is_null()) == b.get(*k).filter(|v| !v.is_null()))
3568 };
3569 let changes = |current: &Map<String, Value>, before: &Map<String, Value>, after: &Map<String, Value>| {
3572 [before, after].into_iter().find(|b| same(current, b)).map(|baseline| {
3573 after
3574 .iter()
3575 .filter(|(k, v)| baseline.get(*k) != Some(*v))
3576 .map(|(k, v)| (k.clone(), v.clone()))
3577 .collect::<Map<String, Value>>()
3578 })
3579 };
3580 let (before, after) = (fields(base, &CODEX_OUTER), fields(&updated, &CODEX_OUTER));
3581 if before != after
3582 && let Some(changed) = changes(user, &before, &after)
3583 {
3584 partition_applies = true;
3585 out.set.extend(changed);
3586 }
3587 const CODEX_SIWC: [&str; 9] = [
3589 "access_token",
3590 "refresh_token",
3591 "id_token",
3592 "client_id",
3593 "device_id",
3594 "expired",
3595 "last_refresh",
3596 "scope",
3597 "scopes",
3598 ];
3599 let siwc = |m: &Map<String, Value>| m.get("siwc").and_then(Value::as_object).cloned().unwrap_or_default();
3600 let (before, after) = (fields(&siwc(base), &CODEX_SIWC), fields(&siwc(&updated), &CODEX_SIWC));
3601 let current = siwc(user);
3602 if before != after
3603 && let Some(changed) = changes(¤t, &before, &after)
3604 {
3605 partition_applies = true;
3606 if !changed.is_empty() {
3607 let mut merged = current;
3608 merged.extend(changed);
3609 out.set.insert("siwc".into(), Value::Object(merged));
3610 }
3611 }
3612 }
3613 (out, partition_applies)
3614}
3615
3616#[allow(clippy::too_many_arguments)]
3626async fn refresh_codex_file(
3627 refreshes: &CodexRefreshes,
3628 store: Arc<CredentialStore>,
3629 refresher: cpa_exec::codex_refresh::Refresher,
3630 guard: tokio::sync::OwnedMutexGuard<()>,
3631 current: Arc<Credential>,
3632 registered: crate::plugins::refresh::Registered,
3633 view: Arc<Credential>,
3634 path: std::path::PathBuf,
3635 mode: cpa_exec::codex_refresh::RefreshMode,
3636 refused: Option<Arc<Credential>>,
3637) -> Result<Arc<Credential>, ExecError> {
3638 let local = |e: String| ExecError::local(500, FailureScope::Request, format!("codex executor: {e}"));
3639 let shutdown = refreshes.shutdown.clone();
3640 let transaction = refreshes.tasks.spawn(async move {
3641 let _guard = guard;
3642 let (lock, disk) = read_locked_codex_file(&path, &shutdown).await.map_err(local)?;
3643 let patch = refresher.refresh_locked(&view, &disk, mode).await.inspect_err(|e| {
3645 if let Some(refused) = &refused {
3646 store.reject_refused(refused, registered.registration, e);
3647 }
3648 })?;
3649 if patch.is_empty() {
3650 return Ok(current);
3651 }
3652 let admitted = tokio::task::spawn_blocking(move || {
3653 let _lock = lock;
3654 if patch.granted() {
3657 let bytes = std::fs::read(&path).map_err(|e| {
3658 PatchError::Io(format!("re-read credential file {} before write: {e}", path.display()))
3659 })?;
3660 let out = codex_patched(&bytes, &patch, true).ok_or_else(|| {
3661 PatchError::Io(format!("invalid credential JSON in {} before write", path.display()))
3662 })?;
3663 write_bytes_atomic(&path, &out)
3664 .map_err(|e| PatchError::Io(format!("persist refreshed credentials to {}: {e}", path.display())))?;
3665 }
3666 #[cfg(test)]
3667 tests::before_admit(&path);
3668 store.admit_codex_refresh(¤t, registered, &patch)
3669 })
3670 .await
3671 .map_err(|e| PatchError::Io(e.to_string()))
3672 .and_then(|r| r)
3673 .map_err(|e| {
3674 ExecError::local(
3675 500,
3676 FailureScope::Request,
3677 format!("committing prepared credential: {e:?}"),
3678 )
3679 })?;
3680 Ok(match admitted {
3681 CodexAdmission::Committed(c) | CodexAdmission::Discarded(c) => c,
3682 })
3683 });
3684 transaction.await.map_err(|e| local(e.to_string()))?
3685}
3686
3687fn codex_patched(bytes: &[u8], patch: &cpa_exec::codex_refresh::RefreshPatch, refresh_write: bool) -> Option<Vec<u8>> {
3694 use cpa_common::json::GoValue;
3695 let target = match serde_json::from_slice::<Value>(bytes) {
3698 Ok(Value::Object(target)) if !(refresh_write && target.is_empty()) => target,
3699 _ => return None,
3700 };
3701 let mut doc = GoValue::parse_f64(bytes)?.into_object().ok()?;
3702 patch.apply(&target, &mut doc);
3703 if !refresh_write {
3704 let legacy: Vec<String> = doc
3705 .keys()
3706 .filter(|k| cpa_plugin::auth::canonical_metadata_key(k) != k.as_str())
3707 .cloned()
3708 .collect();
3709 for key in legacy {
3710 let value = doc.remove(&key).expect("listed above");
3711 doc.entry(cpa_plugin::auth::canonical_metadata_key(&key).to_owned())
3712 .or_insert(value);
3713 }
3714 }
3715 Some(GoValue::Object(doc).marshal())
3716}
3717
3718fn save_codex_refresh(
3726 path: &Path,
3727 current: &Credential,
3728 patch: &cpa_exec::codex_refresh::RefreshPatch,
3729) -> (Vec<u8>, bool) {
3730 let memory = || {
3731 let mut metadata = current.metadata.clone();
3732 metadata.insert("disabled".into(), Value::Bool(current.disabled));
3733 let bytes = serde_json::to_vec(&metadata).expect("JSON values serialize");
3734 codex_patched(&bytes, patch, false).expect("a JSON object")
3735 };
3736 let warn = |error: String| {
3737 tracing::warn!(credential = %current.id, %error, "failed to persist updated codex auth");
3738 };
3739 let out = match std::fs::read(path) {
3740 Ok(bytes) => {
3741 codex_patched(&bytes, patch, false).ok_or_else(|| format!("invalid credential JSON in {}", path.display()))
3742 }
3743 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(memory()),
3744 Err(e) => Err(format!("read credential file {}: {e}", path.display())),
3745 };
3746 let saved = out.and_then(|out| match write_bytes_atomic(path, &out) {
3747 Ok(()) => Ok(out),
3748 Err(e) => Err(format!("write credential file {}: {e}", path.display())),
3749 });
3750 match saved {
3751 Ok(out) => (out, true),
3752 Err(error) => {
3753 warn(error);
3754 (memory(), false)
3755 }
3756 }
3757}
3758
3759enum CodexAdmission {
3761 Committed(Arc<Credential>),
3763 Discarded(Arc<Credential>),
3765}
3766
3767fn publish_metadata_proxy(c: &mut Credential) {
3770 match c.str("proxy_url").map(str::trim).filter(|p| !p.is_empty()) {
3771 Some(proxy) => {
3772 let proxy = proxy.to_owned();
3773 c.attributes.insert("proxy_url".into(), proxy)
3774 }
3775 None => c.attributes.remove("proxy_url"),
3776 };
3777}
3778
3779const LOCK_WAIT_STOPPED: &str = "shutting down while waiting for the credential lock";
3781
3782const LOCK_POLL_CEILING: Duration = Duration::from_millis(100);
3784
3785async fn wait_codex_lock(path: &Path, shutdown: &tokio_util::sync::CancellationToken) -> Result<std::fs::File, String> {
3788 #[cfg(test)]
3789 tests::LOCK_ATTEMPTS
3790 .lock()
3791 .unwrap_or_else(PoisonError::into_inner)
3792 .push(path.to_owned());
3793 let stopped = || LOCK_WAIT_STOPPED.to_owned();
3794 if shutdown.is_cancelled() {
3795 return Err(stopped());
3796 }
3797 let lock = cpa_exec::codex_siwc_bind::open_lock_file(path)?;
3798 let mut delay = Duration::from_millis(5);
3802 loop {
3803 if shutdown.is_cancelled() {
3805 return Err(stopped());
3806 }
3807 if cpa_exec::codex_siwc_bind::try_lock(&lock)? {
3808 return Ok(lock);
3809 }
3810 tokio::select! {
3811 biased;
3812 _ = shutdown.cancelled() => return Err(stopped()),
3813 _ = tokio::time::sleep(delay) => {}
3814 }
3815 delay = (delay * 2).min(LOCK_POLL_CEILING);
3816 }
3817}
3818
3819type LockedFile = (std::fs::File, Map<String, Value>);
3821
3822async fn read_locked_codex_file(
3826 path: &Path,
3827 shutdown: &tokio_util::sync::CancellationToken,
3828) -> Result<LockedFile, String> {
3829 let lock = wait_codex_lock(path, shutdown).await?;
3830 let data = std::fs::read(path).map_err(|e| format!("read credential file {}: {e}", path.display()))?;
3831 match serde_json::from_slice(&data) {
3832 Ok(Value::Object(disk)) if !disk.is_empty() => Ok((lock, disk)),
3833 _ => Err(format!("invalid credential JSON in {}", path.display())),
3834 }
3835}
3836
3837pub(crate) fn credentials_changed(a: &Credential, b: &Credential) -> bool {
3839 let text = |c: &Credential, keys: &[&str]| {
3840 keys.iter()
3841 .find_map(|k| {
3842 c.metadata
3843 .get(*k)
3844 .and_then(Value::as_str)
3845 .map(str::trim)
3846 .filter(|v| !v.is_empty())
3847 })
3848 .unwrap_or_default()
3849 .to_owned()
3850 };
3851 let siwc = |c: &Credential, key: &str| {
3852 c.metadata
3853 .get("siwc")
3854 .and_then(|s| s.get(key))
3855 .and_then(Value::as_str)
3856 .map_or("", str::trim)
3857 .to_owned()
3858 };
3859 let api_key = |c: &Credential| {
3860 c.attributes
3861 .get("api_key")
3862 .filter(|k| !k.is_empty())
3863 .cloned()
3864 .unwrap_or_else(|| text(c, &["api_key"]))
3865 };
3866 let parse_bool = |s: &str| match s.trim() {
3868 "1" | "t" | "T" | "TRUE" | "true" | "True" => Some(true),
3869 "0" | "f" | "F" | "FALSE" | "false" | "False" => Some(false),
3870 _ => None,
3871 };
3872 let use_siwc = |c: &Credential| {
3873 let value = c.metadata.get("use_siwc")?;
3874 value
3875 .as_bool()
3876 .or_else(|| parse_bool(value.as_str()?))
3877 .or_else(|| value.as_f64().map(|n| n != 0.0))
3878 };
3879 let effective_access_token = |c: &Credential| {
3882 let attribute = c.attributes.get("use_siwc").and_then(|v| parse_bool(v)) == Some(true);
3883 if attribute || use_siwc(c) == Some(true) {
3884 siwc(c, "access_token")
3885 } else {
3886 text(c, &["access_token", "accessToken"])
3887 }
3888 };
3889 effective_access_token(a) != effective_access_token(b)
3890 || [
3891 &["access_token", "accessToken"][..],
3892 &["refresh_token", "refreshToken"],
3893 &["id_token", "idToken"],
3894 ]
3895 .iter()
3896 .any(|keys| text(a, keys) != text(b, keys))
3897 || api_key(a) != api_key(b)
3898 || ["access_token", "refresh_token", "id_token", "client_id"]
3899 .iter()
3900 .any(|k| siwc(a, k) != siwc(b, k))
3901 || use_siwc(a) != use_siwc(b)
3902}
3903
3904pub(crate) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
3908 #[cfg(test)]
3909 tests::stall_write(path);
3910 static COUNTER: AtomicU64 = AtomicU64::new(0);
3911 let dir = path.parent().unwrap_or(Path::new("."));
3912 let name = path.file_name().unwrap_or_default().to_string_lossy();
3913 let (tmp, mut file) = loop {
3914 let tmp = dir.join(format!(
3915 ".{name}.{}.{}.tmp",
3916 std::process::id(),
3917 COUNTER.fetch_add(1, Ordering::Relaxed)
3918 ));
3919 let mut options = std::fs::OpenOptions::new();
3921 options.write(true).create_new(true);
3922 #[cfg(unix)]
3924 std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
3925 match options.open(&tmp) {
3926 Ok(file) => break (tmp, file),
3927 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
3928 Err(e) => return Err(e),
3929 }
3930 };
3931 let result = file
3932 .write_all(bytes)
3933 .and_then(|()| file.sync_all())
3934 .and_then(|()| std::fs::rename(&tmp, path));
3935 if result.is_err() {
3936 let _ = std::fs::remove_file(&tmp);
3937 }
3938 result
3939}
3940
3941pub struct Completing {
3948 slot: Arc<Mutex<Slot>>,
3949 watcher: Option<tokio::task::JoinHandle<()>>,
3950}
3951
3952struct Slot {
3953 inner: ExecStream,
3954 lease: Option<Lease>,
3955 cancelled: bool,
3957 waker: Option<std::task::Waker>,
3958}
3959
3960impl Completing {
3961 pub fn new(inner: ExecStream, lease: Lease) -> Self {
3962 let cancelled = lease.remote_cancelled();
3963 let slot = Arc::new(Mutex::new(Slot {
3964 inner,
3965 lease: Some(lease),
3966 cancelled: false,
3967 waker: None,
3968 }));
3969 let watcher = cancelled.map(|cancelled| {
3970 let slot = Arc::downgrade(&slot);
3971 tokio::spawn(async move {
3972 cancelled.await;
3973 let Some(slot) = slot.upgrade() else { return };
3974 let (inner, lease, waker) = {
3975 let mut s = slot.lock().unwrap_or_else(PoisonError::into_inner);
3976 let Some(lease) = s.lease.take() else { return };
3977 s.cancelled = true;
3978 let inner = std::mem::replace(&mut s.inner, Box::pin(futures_util::stream::empty()));
3979 (inner, lease, s.waker.take())
3980 };
3981 drop(inner);
3983 lease.complete(Outcome::Cancelled);
3984 if let Some(waker) = waker {
3985 waker.wake();
3986 }
3987 })
3988 });
3989 Self { slot, watcher }
3990 }
3991}
3992
3993impl Drop for Completing {
3994 fn drop(&mut self) {
3995 if let Some(watcher) = &self.watcher {
3996 watcher.abort();
3997 }
3998 }
3999}
4000
4001impl Stream for Completing {
4002 type Item = Result<Bytes, ExecError>;
4003
4004 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
4005 let mut s = self.slot.lock().unwrap_or_else(PoisonError::into_inner);
4006 if std::mem::take(&mut s.cancelled) {
4007 return Poll::Ready(Some(Err(crate::remote::cancelled_error())));
4008 }
4009 if s.lease.is_none() {
4010 return Poll::Ready(None);
4011 }
4012 if self.watcher.is_some() {
4013 s.waker = Some(cx.waker().clone());
4014 }
4015 let item = ready!(s.inner.poll_next_unpin(cx));
4016 match &item {
4017 Some(Ok(_)) => {}
4018 None => s.lease.take().unwrap().complete(Outcome::Success),
4019 Some(Err(e)) => {
4020 s.inner = Box::pin(futures_util::stream::empty());
4023 s.lease.take().unwrap().complete(Outcome::Failure(e.clone()));
4024 }
4025 }
4026 Poll::Ready(item)
4027 }
4028}
4029
4030fn signature_cache_config(cfg: &Config) -> (bool, bool) {
4036 let flag = |key: &str| {
4037 cfg.document
4038 .get("oauth")
4039 .and_then(|o| o.get("providers"))
4040 .and_then(|p| p.get("antigravity"))
4041 .and_then(|a| a.get(key))
4042 .and_then(serde_yaml_ng::Value::as_bool)
4043 };
4044 (
4045 flag("signature-cache-enabled").unwrap_or(true),
4046 flag("signature-bypass-strict").unwrap_or(false),
4047 )
4048}
4049
4050fn effective_access_token(c: &Credential) -> &str {
4055 if cpa_core::registry::dynamic::uses_siwc(c) {
4056 return cpa_exec::codex_siwc::access_token(c);
4057 }
4058 c.str("access_token").map_or("", str::trim)
4059}
4060
4061fn refresh_candidate(c: &Credential) -> bool {
4062 !c.disabled && refreshable(c)
4063}
4064
4065fn refreshable(c: &Credential) -> bool {
4067 matches!(c.source, Source::File(_)) && cpa_core::registry::dynamic::auth_kind(c) != Some("apikey")
4068}
4069
4070fn test_hook(site: &'static str, id: &str) {
4072 #[cfg(test)]
4073 tests::hook(site, id);
4074 let _ = (site, id);
4075}
4076
4077#[cfg(test)]
4078pub(crate) mod tests {
4079 use super::*;
4080 #[cfg(unix)]
4081 use std::os::unix::fs::PermissionsExt;
4082
4083 thread_local! {
4084 pub(super) static AFTER_STALE: std::cell::Cell<Option<Box<dyn FnOnce()>>> = const { std::cell::Cell::new(None) };
4087 }
4088
4089 static PANICS: Mutex<Vec<(&'static str, String)>> = Mutex::new(Vec::new());
4092 static REACHED: Mutex<Vec<(&'static str, String)>> = Mutex::new(Vec::new());
4093
4094 pub(super) fn hook(site: &'static str, id: &str) {
4095 REACHED.lock().unwrap().push((site, id.to_owned()));
4096 let mut panics = PANICS.lock().unwrap();
4097 if let Some(i) = panics.iter().position(|(s, i)| *s == site && i == id) {
4098 panics.remove(i);
4099 drop(panics);
4100 panic!("injected {site} panic");
4101 }
4102 }
4103
4104 fn arm_panic(site: &'static str, id: &str) {
4106 PANICS.lock().unwrap().push((site, id.to_owned()));
4107 }
4108
4109 fn reached(site: &'static str, id: &str) -> usize {
4111 REACHED
4112 .lock()
4113 .unwrap()
4114 .iter()
4115 .filter(|(s, i)| *s == site && i == id)
4116 .count()
4117 }
4118
4119 pub(crate) static LOCK_ATTEMPTS: Mutex<Vec<std::path::PathBuf>> = Mutex::new(Vec::new());
4122
4123 pub(crate) async fn lock_waited(path: &Path) {
4125 let waited = async {
4126 while !LOCK_ATTEMPTS.lock().unwrap().iter().any(|p| p == path) {
4127 tokio::time::sleep(Duration::from_millis(5)).await;
4128 }
4129 };
4130 tokio::time::timeout(Duration::from_secs(5), waited)
4131 .await
4132 .expect("something waits for the credential lock");
4133 }
4134
4135 type AdmitHook = (std::path::PathBuf, Box<dyn FnOnce() + Send>);
4136
4137 static BEFORE_ADMIT: Mutex<Vec<AdmitHook>> = Mutex::new(Vec::new());
4139
4140 pub(super) fn before_admit(path: &Path) {
4141 let hook = {
4142 let mut hooks = BEFORE_ADMIT.lock().unwrap();
4143 hooks.iter().position(|(p, _)| p == path).map(|i| hooks.remove(i).1)
4144 };
4145 if let Some(hook) = hook {
4146 hook();
4147 }
4148 }
4149
4150 #[test]
4155 fn a_refresh_meeting_an_edit_then_a_new_registration_fails_as_stale() {
4156 let dir = std::env::temp_dir().join(format!("cpa-refresh-stale-{}", uuid::Uuid::new_v4()));
4157 std::fs::create_dir_all(&dir).unwrap();
4158 let credential = {
4159 let dir = dir.clone();
4160 move |token: &str| {
4161 let metadata = serde_json::json!({"type":"pp","access_token":token});
4162 let path = dir.join("a.json");
4163 std::fs::write(&path, metadata.to_string()).unwrap();
4164 Credential::from_file(&dir, &path, metadata.as_object().cloned().unwrap()).unwrap()
4165 }
4166 };
4167 let store = Arc::new(CredentialStore::new(vec![credential("a")]));
4168 let (current, registration) = store.get_registered("a.json").unwrap();
4169 let mut note = MetadataPatch::default();
4171 note.set.insert("note".into(), "edited".into());
4172 store.apply_patch("a.json", current.revision, ¬e).unwrap();
4173 AFTER_STALE.with(|hook| {
4174 hook.set(Some(Box::new({
4175 let (store, credential) = (store.clone(), credential.clone());
4176 move || {
4177 store.reconcile(vec![]);
4178 store.reconcile(vec![credential("b")]);
4179 }
4180 })))
4181 });
4182 let mut patch = MetadataPatch::default();
4183 patch.set.insert("access_token".into(), "p2".into());
4184 let fence = RefreshFence {
4185 registered: registration,
4186 plugin: false,
4187 };
4188 let committed = commit_refresh(&store, "a.json", ¤t, fence, &patch, |_| {});
4189 assert!(AFTER_STALE.with(|hook| hook.take()).is_none(), "the hook ran");
4190 assert!(matches!(committed, Err(PatchError::NotFound)), "{committed:?}");
4191 assert_eq!(store.get("a.json").unwrap().metadata["access_token"], "b");
4192 let _ = std::fs::remove_dir_all(&dir);
4193 }
4194
4195 #[test]
4198 fn refresh_skips_api_key_kinds_like_go() {
4199 let fixture: serde_json::Value =
4200 serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
4201 let mut kinds = std::collections::BTreeSet::new();
4202 for case in fixture["auth_kind"].as_array().unwrap() {
4203 let meta = serde_json::json!({"type": "kimi"}).as_object().unwrap().clone();
4204 let mut c = Credential::from_file(Path::new("/a"), Path::new("/a/x.json"), meta).unwrap();
4205 c.metadata = case["metadata"].as_object().cloned().unwrap_or_default();
4206 c.attributes = case["attributes"]
4207 .as_object()
4208 .map(|a| {
4209 a.iter()
4210 .map(|(k, v)| (k.clone(), v.as_str().unwrap().to_owned()))
4211 .collect()
4212 })
4213 .unwrap_or_default();
4214 let kind = case["kind"].as_str().unwrap();
4215 kinds.insert(kind);
4216 assert_eq!(refresh_candidate(&c), kind != "apikey", "case {case}");
4217 c.disabled = true;
4218 assert!(!refresh_candidate(&c));
4219 }
4220 assert!(kinds.contains("apikey") && kinds.contains("oauth"), "{kinds:?}");
4221 }
4222
4223 #[test]
4224 fn signature_cache_config_reads_v8_and_legacy_keys() {
4225 let at = |yaml: &str| signature_cache_config(&Config::parse(yaml).unwrap());
4226 assert_eq!(at("{}\n"), (true, false));
4227 assert_eq!(
4228 at("antigravity-signature-cache-enabled: false\nantigravity-signature-bypass-strict: true\n"),
4229 (false, true)
4230 );
4231 assert_eq!(
4232 at("oauth: {providers: {antigravity: {signature-cache-enabled: false, signature-bypass-strict: true}}}\n"),
4233 (false, true)
4234 );
4235 assert_eq!(
4236 at("oauth: {providers: {antigravity: {signature-cache-enabled: null}}}\n"),
4237 (true, false)
4238 );
4239 }
4240
4241 fn cred(id: &str, provider: &str, disabled: bool) -> Credential {
4242 let mut metadata = Map::new();
4243 metadata.insert("type".into(), provider.into());
4244 metadata.insert("disabled".into(), disabled.into());
4245 Credential::from_file(Path::new("/a"), &Path::new("/a").join(id), metadata).unwrap()
4246 }
4247
4248 fn sel(provider: &str) -> Selection {
4249 Selection {
4250 provider: provider.into(),
4251 ..Selection::default()
4252 }
4253 }
4254
4255 #[test]
4256 fn select_round_robins_enabled_credentials_and_honours_exclusions() {
4257 let store = CredentialStore::new(vec![
4258 cred("c1.json", "claude", false),
4259 cred("off.json", "claude", true),
4260 cred("x.json", "codex", false),
4261 cred("c2.json", "claude", false),
4262 ]);
4263 let picks: Vec<String> = (0..4)
4264 .map(|_| store.select(sel("claude")).unwrap().credential.id.clone())
4265 .collect();
4266 assert_eq!(picks, ["c1.json", "c2.json", "c1.json", "c2.json"]);
4267 assert!(store.select(sel("gemini")).is_none());
4268 let only = Selection {
4269 exclude: vec!["c1.json".into()],
4270 ..sel("claude")
4271 };
4272 for _ in 0..3 {
4273 assert_eq!(store.select(only.clone()).unwrap().credential.id, "c2.json");
4274 }
4275 }
4276
4277 #[test]
4282 fn the_registry_follows_the_latest_credentials() {
4283 let rt = crate::testing::runtime(
4284 Config::parse("").unwrap(),
4285 Vec::new(),
4286 Executors {
4287 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
4288 codex: Default::default(),
4289 devices: Default::default(),
4290 openai: Default::default(),
4291 google: Default::default(),
4292 },
4293 );
4294 let with = |prefix: &str, disabled: bool| {
4295 let mut c = cred("a.json", "claude", disabled);
4296 c.metadata.insert("prefix".into(), prefix.into());
4297 c
4298 };
4299 rt.store.reconcile(vec![with("old", false)]);
4300 let model = cpa_core::registry::dynamic::models_for(&rt.config(), &HashMap::new(), &with("", false), 0)
4301 .first()
4302 .map(|spec| spec.id.clone())
4303 .expect("a Claude catalog model");
4304 let supports = |prefix: &str| rt.registry().client_supports("a.json", &format!("{prefix}/{model}"));
4305 assert!(supports("old"));
4306 let built = rt.registry();
4307 rt.store
4308 .select(Selection::new("claude", &model))
4309 .unwrap()
4310 .complete(Outcome::Success);
4311 assert!(Arc::ptr_eq(&built, &rt.registry()), "a result does not rebuild it");
4312 rt.store.reconcile(vec![with("new", false)]);
4313 assert!(supports("new") && !supports("old"), "a superseded prefix");
4314 rt.store.reconcile(vec![with("new", true)]);
4315 assert!(!rt.registry().has_client("a.json"), "disabled");
4316 rt.store.reconcile(vec![with("new", false)]);
4317 assert!(supports("new"), "enabled again");
4318 rt.store.reconcile(Vec::new());
4319 assert!(!rt.registry().has_client("a.json"), "removed");
4320 rt.store.reconcile(vec![with("fresh", false)]);
4321 assert!(supports("fresh") && !supports("new"), "re-registered");
4322 }
4323
4324 #[test]
4325 fn standalone_admission_reads_attributes_first() {
4326 let mut c = cred("a.json", "claude", false);
4327 c.metadata.insert("prefix".into(), "meta".into());
4328 c.metadata
4329 .insert("excluded_models".into(), serde_json::json!(["claude-sonnet*"]));
4330 c.attributes.insert("prefix".into(), "team".into());
4331 c.attributes.insert("excluded_models".into(), "claude-opus*".into());
4332 let policy = Policy::default();
4333 let admit = |model: &str| {
4334 let s = Selection::new("claude", model);
4335 standalone_admission(&s, &policy)(&c)
4336 };
4337 assert_eq!(
4338 admit("team/claude-sonnet-5(high)").as_deref(),
4339 Some("claude-sonnet-5(high)")
4340 );
4341 assert_eq!(admit("team/claude-opus-5"), None, "attribute exclusions win");
4342 assert_eq!(
4343 admit("meta/claude-sonnet-5"),
4344 None,
4345 "metadata prefix is not authoritative"
4346 );
4347 assert_eq!(admit("claude-sonnet-5").as_deref(), Some("claude-sonnet-5"));
4348 let forced = Policy {
4349 force_model_prefix: true,
4350 ..Policy::default()
4351 };
4352 let s = Selection::new("claude", "claude-sonnet-5");
4353 assert_eq!(standalone_admission(&s, &forced)(&c), None);
4354 assert_eq!(standalone_admission(&Selection::new("codex", "x"), &policy)(&c), None);
4355 }
4356
4357 #[test]
4358 fn patch_persists_atomically_preserves_unknown_fields_and_rejects_stale() {
4359 let dir = std::env::temp_dir().join(format!("cpa-store-{}", std::process::id()));
4360 std::fs::create_dir_all(&dir).unwrap();
4361 let path = dir.join("claude-a.json");
4362 std::fs::write(
4363 &path,
4364 r#"{"type":"claude","access_token":"old","zz_unknown":{"k":[1,2]}}"#,
4365 )
4366 .unwrap();
4367 let victim = dir.join("victim");
4369 std::fs::write(&victim, "keep").unwrap();
4370 #[cfg(unix)]
4371 std::os::unix::fs::symlink(
4372 &victim,
4373 dir.join(format!(".claude-a.json.{}.0.tmp", std::process::id())),
4374 )
4375 .unwrap();
4376 let mut cfg = cpa_core::config::Config::parse("").unwrap();
4377 cfg.auth_dir = dir.clone();
4378 let store = CredentialStore::new(cpa_core::config::credentials::from_auth_dir(&cfg));
4379 let rev = store.get("claude-a.json").unwrap().revision;
4380
4381 let mut patch = MetadataPatch::default();
4382 patch.set.insert("access_token".into(), "new".into());
4383 patch.set.insert("email".into(), "a@x.test".into());
4384 patch.set.insert("disabled".into(), true.into());
4385 let updated = store.apply_patch("claude-a.json", rev, &patch).unwrap();
4386 assert!(updated.revision > rev);
4387 assert!(updated.disabled);
4388 assert_eq!(updated.label, "a@x.test", "derived label follows the new email");
4389 assert_eq!(
4390 std::fs::read_to_string(&path).unwrap(),
4391 "{\"type\":\"claude\",\"access_token\":\"new\",\"zz_unknown\":{\"k\":[1,2]},\"email\":\"a@x.test\",\"disabled\":true}\n"
4392 );
4393 #[cfg(unix)]
4394 assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600);
4395 assert_eq!(
4396 std::fs::read_to_string(&victim).unwrap(),
4397 "keep",
4398 "symlink target untouched"
4399 );
4400 assert!(
4401 store.select(sel("claude")).is_none(),
4402 "patched-disabled credential leaves rotation"
4403 );
4404
4405 assert_eq!(
4406 store.apply_patch("claude-a.json", rev, &patch).unwrap_err(),
4407 PatchError::Stale {
4408 current: updated.revision
4409 }
4410 );
4411 assert_eq!(
4412 store.apply_patch("nope.json", 0, &patch).unwrap_err(),
4413 PatchError::NotFound
4414 );
4415 let mut retype = MetadataPatch::default();
4416 retype.set.insert("type".into(), "codex".into());
4417 assert_eq!(
4418 store
4419 .apply_patch("claude-a.json", updated.revision, &retype)
4420 .unwrap_err(),
4421 PatchError::TypeIsImmutable
4422 );
4423 let leftovers: Vec<_> = std::fs::read_dir(&dir)
4424 .unwrap()
4425 .map(|e| e.unwrap().file_name().into_string().unwrap())
4426 .filter(|n| n.ends_with(".tmp") && !n.ends_with(".0.tmp"))
4427 .collect();
4428 assert!(leftovers.is_empty(), "no temp files left behind: {leftovers:?}");
4429 std::fs::remove_dir_all(&dir).unwrap();
4430 }
4431
4432 #[test]
4435 fn upsert_never_undoes_a_concurrent_patch() {
4436 let dir = std::env::temp_dir().join(format!("cpa-upsert-{}", std::process::id()));
4437 let _ = std::fs::remove_dir_all(&dir);
4438 std::fs::create_dir_all(&dir).unwrap();
4439 std::fs::write(dir.join("refreshed.json"), r#"{"type":"claude","refresh_token":"r0"}"#).unwrap();
4440 std::fs::write(dir.join("saved.json"), r#"{"type":"claude"}"#).unwrap();
4441 let mut cfg = cpa_core::config::Config::parse("").unwrap();
4442 cfg.auth_dir = dir.clone();
4443 let store = Arc::new(CredentialStore::new(cpa_core::config::credentials::from_auth_dir(&cfg)));
4444 let saved = (*store.get("saved.json").unwrap()).clone();
4445 let saver = {
4446 let store = store.clone();
4447 std::thread::spawn(move || {
4448 for n in 0..500 {
4449 let mut next = saved.clone();
4450 next.metadata.insert("n".into(), n.into());
4451 store.upsert(next);
4452 }
4453 })
4454 };
4455 let mut revision = store.get("refreshed.json").unwrap().revision;
4456 for n in 1..=500 {
4457 let mut patch = MetadataPatch::default();
4458 patch.set.insert("refresh_token".into(), format!("r{n}").into());
4459 revision = store
4460 .apply_patch("refreshed.json", revision, &patch)
4461 .unwrap_or_else(|e| panic!("refresh {n} was undone: {e:?}"))
4462 .revision;
4463 }
4464 saver.join().unwrap();
4465 let refreshed = store.get("refreshed.json").unwrap();
4466 assert_eq!(refreshed.revision, revision);
4467 assert_eq!(refreshed.metadata["refresh_token"], "r500");
4468 assert_eq!(store.get("saved.json").unwrap().metadata["n"], 499);
4469 assert_eq!(store.snapshot().len(), 2);
4470 std::fs::remove_dir_all(&dir).unwrap();
4471 }
4472
4473 #[test]
4476 fn replace_config_keeps_concurrent_changes_to_other_credentials() {
4477 let keys = |key: &str| {
4478 let cfg = cpa_core::config::Config::parse(&format!(
4479 "codex-api-key:\n - api-key: {key}\n base-url: http://127.0.0.1:9\n"
4480 ))
4481 .unwrap();
4482 cpa_core::config::credentials::from_config(&cfg)
4483 };
4484 let store = Arc::new(CredentialStore::new(keys("sk-a")));
4485 let mut saved = Credential::relay_session("saved");
4486 saved.source = Source::File("saved.json".into());
4487 saved.id = "saved.json".into();
4488 let changer = {
4490 let store = store.clone();
4491 std::thread::spawn(move || {
4492 for n in 0..2000 {
4493 if n > 0 {
4494 assert_eq!(
4495 store.get("saved.json").unwrap().metadata["n"],
4496 n - 1,
4497 "a save was undone"
4498 );
4499 assert!(store.get("relay").is_none(), "a closed relay session came back");
4500 }
4501 let mut next = saved.clone();
4502 next.metadata.insert("n".into(), n.into());
4503 store.upsert(next);
4504 store.add_runtime(Credential::relay_session("relay"));
4505 store.remove_runtime("relay");
4506 }
4507 })
4508 };
4509 let mut n = 0;
4510 while !changer.is_finished() {
4511 store.replace_config(keys(if n % 2 == 0 { "sk-b" } else { "sk-a" }));
4512 n += 1;
4513 }
4514 changer.join().unwrap();
4515 store.replace_config(keys("sk-a"));
4516 let revision = store.snapshot().iter().find(|c| c.id != "saved.json").unwrap().revision;
4517 store.replace_config(keys("sk-a"));
4518 let ids: Vec<_> = store.snapshot().iter().map(|c| (c.id.clone(), c.revision)).collect();
4519 assert_eq!(ids.len(), 2, "{ids:?}");
4520 assert!(ids.contains(&(keys("sk-a")[0].id.clone(), revision)), "{ids:?}");
4521 assert_eq!(store.get("saved.json").unwrap().metadata["n"], 1999);
4522 }
4523
4524 fn stall_next_write(path: &Path) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
4527 let (entered_tx, entered) = std::sync::mpsc::channel();
4528 let (release, release_rx) = std::sync::mpsc::channel();
4529 STALLS
4530 .lock()
4531 .unwrap()
4532 .insert(path.to_path_buf(), (entered_tx, release_rx));
4533 (entered, release)
4534 }
4535
4536 type Stall = (std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>);
4537 static STALLS: std::sync::LazyLock<Mutex<HashMap<std::path::PathBuf, Stall>>> =
4538 std::sync::LazyLock::new(Mutex::default);
4539
4540 pub(super) fn stall_write(path: &Path) {
4541 let Some((entered, release)) = STALLS.lock().unwrap().remove(path) else {
4542 return;
4543 };
4544 let _ = entered.send(());
4545 let _ = release.recv();
4546 }
4547
4548 fn spawn_bounded<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> std::sync::mpsc::Receiver<T> {
4551 let (tx, rx) = std::sync::mpsc::channel();
4552 std::thread::spawn(move || {
4553 let _ = tx.send(f());
4554 });
4555 rx
4556 }
4557
4558 const BOUND: Duration = Duration::from_secs(5);
4559 const BLOCKED: Duration = Duration::from_millis(300);
4560
4561 fn stalled_store(
4562 name: &str,
4563 files: &[&str],
4564 ) -> (std::path::PathBuf, cpa_core::config::Config, Arc<CredentialStore>) {
4565 let dir = std::env::temp_dir().join(format!("cpa-{name}-{}", std::process::id()));
4566 let _ = std::fs::remove_dir_all(&dir);
4567 std::fs::create_dir_all(&dir).unwrap();
4568 for file in files {
4569 std::fs::write(dir.join(file), r#"{"type":"claude","access_token":"old"}"#).unwrap();
4570 }
4571 let mut cfg = cpa_core::config::Config::parse("").unwrap();
4572 cfg.auth_dir = dir.clone();
4573 let store = CredentialStore::new(cpa_core::config::credentials::from_auth_dir(&cfg));
4574 (dir, cfg, store)
4575 }
4576
4577 fn token_patch(token: &str) -> MetadataPatch {
4578 let mut patch = MetadataPatch::default();
4579 patch.set.insert("access_token".into(), token.into());
4580 patch
4581 }
4582
4583 #[test]
4590 fn a_stalled_credential_write_blocks_only_its_reload() {
4591 let (dir, cfg, store) = stalled_store("stalled-write", &["a.json", "b.json"]);
4592 let before = store.get("a.json").unwrap().revision;
4593 let (entered, release) = stall_next_write(&dir.join("a.json"));
4594 let writer = spawn_bounded({
4595 let store = store.clone();
4596 move || {
4597 store
4598 .apply_patch("a.json", before, &token_patch("new"))
4599 .map(|c| c.revision)
4600 }
4601 });
4602 entered.recv_timeout(BOUND).expect("the write reached the disk");
4603 let mut leases = Vec::new();
4605 let read_and_select = |leases: &mut Vec<Lease>, when: &str| {
4606 let reader = spawn_bounded({
4607 let store = store.clone();
4608 move || store.snapshot().len()
4609 });
4610 assert_eq!(reader.recv_timeout(BOUND).ok(), Some(2), "a snapshot {when}");
4611 let selector = spawn_bounded({
4612 let store = store.clone();
4613 move || {
4614 store.select(Selection {
4615 exclude: vec!["a.json".into()],
4616 ..sel("claude")
4617 })
4618 }
4619 });
4620 let lease = selector
4621 .recv_timeout(BOUND)
4622 .unwrap_or_else(|_| panic!("a selection {when}"));
4623 leases.push(lease.expect("b.json"));
4624 };
4625 read_and_select(&mut leases, "during the write");
4626 let other = spawn_bounded({
4627 let store = store.clone();
4628 move || {
4629 let revision = store.get("b.json").unwrap().revision;
4630 store.apply_patch("b.json", revision, &token_patch("b2")).map(|_| ())
4631 }
4632 });
4633 assert!(
4634 matches!(other.recv_timeout(BOUND), Ok(Ok(()))),
4635 "another credential's write"
4636 );
4637 let mut rescan = cpa_core::config::credentials::from_auth_dir(&cfg);
4641 let in_scan = rescan.iter_mut().find(|c| c.id == "a.json").unwrap();
4642 in_scan.metadata.insert("access_token".into(), "new".into());
4643 in_scan.metadata.insert("reloaded".into(), true.into());
4644 let reload = spawn_bounded({
4645 let store = store.clone();
4646 move || store.reconcile(rescan)
4647 });
4648 assert!(
4649 reload.recv_timeout(BLOCKED).is_err(),
4650 "the reload overtook the stalled write"
4651 );
4652 read_and_select(&mut leases, "while a reload is pending");
4653 let same = spawn_bounded({
4654 let store = store.clone();
4655 move || store.apply_patch("a.json", before, &token_patch("other")).map(|_| ())
4656 });
4657 assert!(
4658 same.recv_timeout(BLOCKED).is_err(),
4659 "a write of the same credential overtook it"
4660 );
4661 release.send(()).unwrap();
4662 let published = writer
4663 .recv_timeout(BOUND)
4664 .expect("the write finished")
4665 .expect("and published");
4666 assert!(published > before);
4667 reload.recv_timeout(BOUND).expect("the reload finished");
4668 assert!(
4669 matches!(same.recv_timeout(BOUND), Ok(Err(PatchError::Stale { .. }))),
4670 "the same-credential write saw the published revision"
4671 );
4672 drop(leases);
4673 let file: Map<String, Value> = serde_json::from_slice(&std::fs::read(dir.join("a.json")).unwrap()).unwrap();
4674 assert_eq!(file["access_token"], "new");
4675 let a = store.get("a.json").unwrap();
4676 assert_eq!(
4677 a.metadata.get("reloaded"),
4678 Some(&Value::Bool(true)),
4679 "the reload landed last"
4680 );
4681 assert!(a.revision > published);
4682 assert_eq!(store.get("b.json").unwrap().metadata["access_token"], "b2");
4683 std::fs::remove_dir_all(&dir).unwrap();
4684 }
4685
4686 #[test]
4689 fn an_identity_change_waits_for_a_write_of_the_credential_it_replaces() {
4690 let (dir, _cfg, store) = stalled_store("identity-change", &["a.json", "b.json"]);
4691 let b = store.get("b.json").unwrap().revision;
4692 let a = store.get("a.json").unwrap();
4693 let (entered, release) = stall_next_write(&dir.join("b.json"));
4694 let writer = spawn_bounded({
4695 let store = store.clone();
4696 move || store.apply_patch("b.json", b, &token_patch("b2")).map(|c| c.revision)
4697 });
4698 entered.recv_timeout(BOUND).expect("B's write reached the disk");
4699 let mut renamed = Credential::clone(&a);
4700 renamed.id = "b.json".into();
4701 renamed.metadata.insert("access_token".into(), "renamed".into());
4702 let replace = spawn_bounded({
4703 let store = store.clone();
4704 move || {
4705 let bytes = br#"{"type":"claude","access_token":"renamed"}"#;
4706 store.replace_file_backed("a.json", a.revision, bytes, Some(renamed))
4707 }
4708 });
4709 assert!(replace.recv_timeout(BLOCKED).is_err(), "the rename overtook B's write");
4710 release.send(()).unwrap();
4711 let written = writer.recv_timeout(BOUND).expect("B's write finished");
4712 assert!(written.is_ok(), "B's write was published, not dropped: {written:?}");
4713 assert!(matches!(replace.recv_timeout(BOUND), Ok(Ok(()))));
4714 let ids: Vec<String> = store.snapshot().iter().map(|c| c.id.clone()).collect();
4715 assert_eq!(ids, ["b.json"]);
4716 assert_eq!(store.get("b.json").unwrap().metadata["access_token"], "renamed");
4717 std::fs::remove_dir_all(&dir).unwrap();
4718 }
4719
4720 #[test]
4724 fn a_stale_scan_cannot_undo_a_later_write() {
4725 let dir = std::env::temp_dir().join(format!("cpa-stale-scan-{}", std::process::id()));
4726 let _ = std::fs::remove_dir_all(&dir);
4727 std::fs::create_dir_all(&dir).unwrap();
4728 let path = dir.join("synthetic.json");
4729 let write = |prefix: &str| std::fs::write(&path, format!(r#"{{"type":"codex","prefix":"{prefix}"}}"#)).unwrap();
4730 let mut cfg = cpa_core::config::Config::parse("").unwrap();
4731 cfg.auth_dir = dir.clone();
4732 let scan = || cpa_core::config::credentials::scan_auth_dir(&cfg);
4733 write("pfx-8");
4734 let store = CredentialStore::new(scan().credentials);
4735 let stale = scan();
4736 let revision = store.get("synthetic.json").unwrap().revision;
4737 let mut patch = MetadataPatch::default();
4738 patch.set.insert("prefix".into(), "pfx-9".into());
4739 store.apply_patch("synthetic.json", revision, &patch).unwrap();
4740 let current = || store.get("synthetic.json").unwrap();
4741 let before = current();
4742 store.reconcile_scanned(stale.credentials, &stale.read);
4743 assert_eq!(
4744 current().metadata["prefix"],
4745 "pfx-9",
4746 "the stale scan overwrote the write"
4747 );
4748 assert_eq!(current().revision, before.revision);
4749 write("manual-edit");
4750 let manual = scan();
4751 store.reconcile_scanned(manual.credentials, &manual.read);
4752 assert_eq!(current().metadata["prefix"], "manual-edit", "a real edit was lost");
4753 let before = current();
4754 write("deleted-edit");
4755 let deleted = scan();
4756 std::fs::remove_file(&path).unwrap();
4757 store.reconcile_scanned(deleted.credentials.clone(), &deleted.read);
4758 assert_eq!(
4759 current().revision,
4760 before.revision,
4761 "a deleted file's snapshot was applied"
4762 );
4763 std::fs::create_dir(&path).unwrap();
4764 store.reconcile_scanned(deleted.credentials.clone(), &deleted.read);
4765 assert_eq!(
4766 current().revision,
4767 before.revision,
4768 "an unreadable file's snapshot was applied"
4769 );
4770 #[cfg(unix)]
4772 {
4773 std::fs::remove_dir(&path).unwrap();
4774 let fifo = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
4775 assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o600) }, 0);
4777 let done = spawn_bounded({
4778 let store = store.clone();
4779 move || store.reconcile_scanned(deleted.credentials, &deleted.read)
4780 });
4781 done.recv_timeout(BOUND).expect("the freshness check opened the FIFO");
4782 assert_eq!(current().revision, before.revision, "a FIFO's snapshot was applied");
4783 }
4784 std::fs::remove_dir_all(&dir).unwrap();
4785 }
4786
4787 #[test]
4788 fn reconcile_keeps_unchanged_revisions_and_never_reuses_old_ones() {
4789 let store = CredentialStore::new(vec![cred("a.json", "claude", false), cred("b.json", "claude", false)]);
4790 let a = store.get("a.json").unwrap();
4791 let b = store.get("b.json").unwrap();
4792 store.reconcile(vec![cred("a.json", "claude", false)]);
4793 assert!(
4794 Arc::ptr_eq(&store.get("a.json").unwrap(), &a),
4795 "unchanged credential kept as is"
4796 );
4797 assert!(store.get("b.json").is_none());
4798 store.reconcile(vec![cred("a.json", "claude", false), cred("b.json", "claude", false)]);
4799 assert!(
4800 store.get("b.json").unwrap().revision > b.revision,
4801 "re-created credential gets a new revision"
4802 );
4803 }
4804
4805 #[test]
4806 fn attribute_only_reload_changes_priority_and_rejects_stale_outcome() {
4807 let a = cred("a.json", "claude", false);
4808 let mut b = cred("b.json", "claude", false);
4809 b.attributes.insert("priority".into(), "1".into());
4810 let store = CredentialStore::new(vec![a.clone(), b.clone()]);
4811 let old = store.select(sel("claude")).unwrap();
4812 assert_eq!(old.credential.id, "b.json");
4813 let revision = old.credential.revision;
4814 b.attributes.insert("priority".into(), "2".into());
4815 store.reconcile(vec![a, b]);
4816 assert!(store.get("b.json").unwrap().revision > revision);
4817 old.complete(Outcome::Failure(ExecError::local(
4818 401,
4819 FailureScope::Credential,
4820 "expired",
4821 )));
4822 let current = store.select(sel("claude")).unwrap();
4823 assert_eq!(
4824 current.credential.id, "b.json",
4825 "stale outcome must not cool new revision"
4826 );
4827 assert_eq!(current.credential.attributes["priority"], "2");
4828 current.complete(Outcome::Success);
4829 assert_eq!(store.stats().failure, 1);
4830 assert_eq!(store.stats().success, 1);
4831 }
4832
4833 #[test]
4834 fn stale_lease_cannot_poison_deleted_and_recreated_credential() {
4835 let credential = cred("a.json", "claude", false);
4836 let store = CredentialStore::new(vec![credential.clone()]);
4837 let old = store.select(sel("claude")).unwrap();
4838 store.reconcile(Vec::new());
4839 store.reconcile(vec![credential]);
4840 old.complete(Outcome::Failure(ExecError::local(
4841 429,
4842 FailureScope::Credential,
4843 "quota",
4844 )));
4845 store.select(sel("claude")).unwrap().complete(Outcome::Success);
4846 assert_eq!(store.stats().failure, 1);
4847 assert_eq!(store.stats().success, 1);
4848 }
4849
4850 #[test]
4856 fn retry_wait_skips_cooldowns_beyond_the_cap_and_non_retryable_statuses() {
4857 let now = Instant::now();
4858 let selection = sel("claude");
4859 let admit_all = |_: &Credential| Some(String::new());
4860 let wait = |cap: u64, cooldowns: &[(u16, u64)]| {
4862 let policy = Policy {
4863 request_retry: 1,
4864 max_retry_interval: Duration::from_secs(cap),
4865 ..Policy::default()
4866 };
4867 let ids: Vec<String> = (0..cooldowns.len()).map(|i| format!("c{i}.json")).collect();
4868 let store = CredentialStore::new(ids.iter().map(|id| cred(id, "claude", false)).collect());
4869 for (id, (status, seconds)) in ids.iter().zip(cooldowns) {
4870 let mut error = ExecError::local(*status, FailureScope::Model, "cooling");
4871 error.retry_after = Some(Duration::from_secs(*seconds));
4872 let credential = store.get(id).unwrap();
4873 store.scheduler.lock().unwrap().record(
4874 &credential,
4875 &selection.model,
4876 &Outcome::Failure(error),
4877 &policy,
4878 now,
4879 );
4880 }
4881 store.retry_wait_at(&selection, &policy, 429, &[], &admit_all, now)
4882 };
4883 let short_and_long = wait(30, &[(429, 10), (429, 60)]);
4884 assert!(
4885 short_and_long.is_some_and(|w| !w.is_zero() && w <= Duration::from_secs(10)),
4886 "{short_and_long:?}"
4887 );
4888 assert_eq!(wait(30, &[(429, 60)]), None, "only a cooldown beyond the cap");
4889 assert!(wait(3600, &[(429, 60)]).is_some(), "rate limit");
4890 assert_eq!(wait(3600, &[(402, 60)]), None, "payment required");
4891 assert_eq!(wait(3600, &[(404, 60)]), None, "not found");
4892 }
4893
4894 #[test]
4895 fn retry_wait_obeys_exact_cap_and_attempted_quota_floor() {
4896 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
4897 let now = Instant::now();
4898 let mut selection = sel("claude");
4899 let admit_all = |_: &Credential| Some(String::new());
4900 let tried = ["a.json".to_owned()];
4901 let mut policy = Policy {
4902 request_retry: 1,
4903 ..Policy::default()
4904 };
4905 assert_eq!(
4907 store.retry_wait_at(&selection, &policy, 0, &tried, &admit_all, now),
4908 Some(Duration::ZERO)
4909 );
4910 assert_eq!(
4913 store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, now),
4914 None
4915 );
4916 assert_eq!(
4917 store.retry_wait_at(&selection, &policy, 429, &[], &admit_all, now),
4918 Some(Duration::ZERO)
4919 );
4920 policy.max_retry_interval = Duration::from_secs(10);
4921 assert_eq!(
4922 store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, now),
4923 Some(Duration::from_secs(10))
4924 );
4925 let quota = ExecError::local(429, FailureScope::Model, "quota");
4926 let credential = store.get("a.json").unwrap();
4927 store.scheduler.lock().unwrap().record(
4928 &credential,
4929 &selection.model,
4930 &Outcome::Failure(quota.clone()),
4931 &policy,
4932 now,
4933 );
4934 let later = now + Duration::from_millis(1);
4935 assert_eq!(
4936 store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, later),
4937 Some(Duration::from_secs(10))
4938 );
4939 policy.max_retry_interval = Duration::from_secs(9);
4940 assert_eq!(
4941 store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, later),
4942 None
4943 );
4944 policy.disable_cooling = true;
4945 store.scheduler.lock().unwrap().record(
4946 &credential,
4947 &selection.model,
4948 &Outcome::Failure(quota.clone()),
4949 &policy,
4950 later,
4951 );
4952 policy.max_retry_interval = Duration::ZERO;
4953 assert_eq!(
4954 store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, later),
4955 Some(Duration::ZERO),
4956 "disabled cooling has no floor"
4957 );
4958 selection.retry_round = 1;
4959 assert_eq!(
4960 store.retry_wait_at(&selection, &policy, 0, &tried, &admit_all, later),
4961 None
4962 );
4963 }
4964
4965 fn weekly_limit(scope: FailureScope) -> Outcome {
4967 let mut error = ExecError::local(429, scope, "usage_limit_reached");
4968 error.retry_after = Some(Duration::from_secs(6 * 24 * 3600));
4969 Outcome::Failure(error)
4970 }
4971
4972 #[test]
4975 fn a_stale_probe_cancellation_keeps_the_newer_reservation() {
4976 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
4977 let selection = Selection::new("claude", "m");
4978 let credential = store.get("a.json").unwrap();
4979 let policy = Policy {
4981 max_trusted_cooldown: Duration::from_secs(10),
4982 ..Policy::default()
4983 };
4984 let opened = Instant::now().checked_sub(Duration::from_secs(11)).unwrap();
4985 store
4986 .scheduler
4987 .lock()
4988 .unwrap()
4989 .record(&credential, "m", &weekly_limit(FailureScope::Model), &policy, opened);
4990 let first = store.select(selection.clone()).expect("the probe");
4991 assert_eq!(first.execution_model, "m");
4992 assert!(store.select(selection.clone()).is_none(), "reserved");
4993 let key = ("a.json".to_owned(), "m".to_owned());
4995 store
4996 .scheduler
4997 .lock()
4998 .unwrap()
4999 .cooldowns
5000 .get_mut(&key)
5001 .unwrap()
5002 .trust
5003 .probe = Some(Instant::now());
5004 let second = store.select(selection.clone()).expect("the second probe");
5005 first.complete(Outcome::Cancelled);
5006 assert!(store.select(selection.clone()).is_none(), "still reserved");
5007 second.complete(Outcome::Cancelled);
5008 assert!(store.select(selection).is_some(), "its own cancellation frees it");
5009 }
5010
5011 #[tokio::test]
5016 async fn successes_dispatched_before_a_cooldown_keep_it() {
5017 let rt = Arc::new(crate::testing::runtime(
5018 Config::parse("").unwrap(),
5019 vec![cred("a.json", "claude", false)],
5020 Executors {
5021 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5022 codex: Default::default(),
5023 devices: Default::default(),
5024 openai: Default::default(),
5025 google: Default::default(),
5026 },
5027 ));
5028 let selection = Selection::new("claude", "m");
5029 let cfg = rt.config();
5030 let in_flight = rt.store.select(selection.clone()).unwrap();
5031 let credits = rt
5032 .lease_credential("a.json", selection.clone(), &cfg, Credential::clone)
5033 .await
5034 .unwrap();
5035 let mut retried = rt.store.select(selection.clone()).unwrap();
5036 let mut quota = ExecError::local(429, FailureScope::Model, "rate limited");
5037 quota.retry_after = Some(Duration::from_secs(3600));
5038 rt.store
5039 .select(selection.clone())
5040 .unwrap()
5041 .complete(Outcome::Failure(quota));
5042 assert!(rt.store.select(selection.clone()).is_none(), "cooling");
5043 in_flight.complete(Outcome::Success);
5044 assert!(rt.store.select(selection.clone()).is_none(), "a stale success keeps it");
5045 credits.complete(Outcome::Success);
5046 assert!(
5047 rt.store.select(selection.clone()).is_none(),
5048 "a stale credits success keeps it"
5049 );
5050 retried.redispatch();
5051 retried.complete(Outcome::Success);
5052 assert!(rt.store.select(selection).is_some(), "a later dispatch clears it");
5053 }
5054
5055 #[test]
5059 fn a_late_success_keeps_a_newer_window() {
5060 let selection = Selection::new("claude", "m");
5061 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
5063 let stream = store.select(selection.clone()).unwrap();
5064 let other = store.select(selection.clone()).unwrap();
5065 other.complete(weekly_limit(FailureScope::Model));
5066 assert!(store.select(selection.clone()).is_none(), "cooling");
5067 stream.accepted();
5068 stream.complete(Outcome::Success);
5069 assert!(store.select(selection.clone()).is_none(), "the window stands");
5070 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
5072 let stream = store.select(selection.clone()).unwrap();
5073 let policy = Policy {
5074 max_trusted_cooldown: Duration::from_secs(10),
5075 ..Policy::default()
5076 };
5077 let opened = Instant::now().checked_sub(Duration::from_secs(11)).unwrap();
5078 store.scheduler.lock().unwrap().record(
5079 &store.get("a.json").unwrap(),
5080 "m",
5081 &weekly_limit(FailureScope::Credential),
5082 &policy,
5083 opened,
5084 );
5085 stream.accepted();
5086 stream.complete(Outcome::Success);
5087 let _probe = store.select(selection.clone()).expect("the window's probe");
5088 assert!(store.select(selection).is_none(), "one probe");
5089 }
5090
5091 #[test]
5092 fn config_routing_drives_policy_at_startup_and_on_publish() {
5093 let executors = || Executors {
5094 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5095 codex: Default::default(),
5096 devices: Default::default(),
5097 openai: Default::default(),
5098 google: Default::default(),
5099 };
5100 let rt = crate::testing::runtime(
5102 Config::parse("request-retry: 2\nrouting:\n strategy: ff\n session-affinity-ttl: 250ms\n").unwrap(),
5103 Vec::new(),
5104 executors(),
5105 );
5106 let policy = rt.policy();
5107 assert_eq!(policy.strategy, crate::scheduler::Strategy::FillFirst);
5108 assert_eq!(policy.request_retry, 2);
5109 assert_eq!(policy.session_affinity_ttl, Duration::from_secs(1));
5110
5111 rt.publish_config(Config::parse("routing:\n strategy: wrr\n").unwrap());
5112 let policy = rt.policy();
5113 assert_eq!(policy.strategy, crate::scheduler::Strategy::WeightedRoundRobin);
5114 assert_eq!(policy.request_retry, 0);
5115 assert_eq!(policy.session_affinity_ttl, Duration::from_secs(3600));
5116 }
5117
5118 #[test]
5121 fn pool_offsets_rotate_per_key_like_go() {
5122 let rt = crate::testing::runtime(
5123 Config::parse("").unwrap(),
5124 Vec::new(),
5125 Executors {
5126 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5127 codex: Default::default(),
5128 devices: Default::default(),
5129 openai: Default::default(),
5130 google: Default::default(),
5131 },
5132 );
5133 let offsets: Vec<usize> = (0..4).map(|_| rt.next_pool_offset("a|openai|m", 3)).collect();
5134 assert_eq!(offsets, [0, 1, 2, 0]);
5135 assert_eq!(rt.next_pool_offset(" a|openai|m ", 3), 1, "keys are trimmed");
5136 assert_eq!(rt.next_pool_offset("b|openai|m", 3), 0, "keys rotate independently");
5137 assert_eq!(rt.next_pool_offset("c", 1), 0);
5138 assert_eq!(rt.next_pool_offset("c", 2), 0, "a single-entry pool does not advance");
5139 assert_eq!(rt.next_pool_offset(" ", 2), 0);
5140 rt.pool_offsets.lock().unwrap().insert("d".into(), 2_147_483_641);
5141 assert_eq!(rt.next_pool_offset("d", 3), 0, "the guard resets before use");
5142 assert_eq!(rt.next_pool_offset("d", 3), 1);
5143 }
5144
5145 #[test]
5149 fn first_use_persists_disabled_in_go_marshal_form() {
5150 let dir = std::env::temp_dir().join(format!(
5151 "persist-{}-{}",
5152 std::process::id(),
5153 std::time::SystemTime::now()
5154 .duration_since(std::time::UNIX_EPOCH)
5155 .unwrap()
5156 .as_nanos()
5157 ));
5158 std::fs::create_dir_all(&dir).unwrap();
5159 let path = dir.join("a.json");
5160 let original = "{\"type\":\"claude\",\"email\":\"a<b>@x.invalid\",\"n\":1.50,\"big\":1e3,\"nested\":{\"z\":1,\"a\":[true,null]}}\n";
5161 std::fs::write(&path, original).unwrap();
5162 let metadata: Map<String, Value> = serde_json::from_str(original).unwrap();
5163 let store = CredentialStore::new(vec![Credential::from_file(&dir, &path, metadata).unwrap()]);
5164 let revision = store.get("a.json").unwrap().revision;
5165 store
5166 .select(Selection::new("claude", "m"))
5167 .unwrap()
5168 .complete(Outcome::Success);
5169 let written = std::fs::read_to_string(&path).unwrap();
5170 assert_eq!(
5171 written,
5172 r#"{"big":1000,"disabled":false,"email":"a\u003cb\u003e@x.invalid","n":1.5,"nested":{"a":[true,null],"z":1},"type":"claude"}"#
5173 );
5174 let current = store.get("a.json").unwrap();
5175 assert_eq!(current.revision, revision, "a neutral write keeps the revision");
5176 assert_eq!(current.metadata.get("disabled"), Some(&Value::Bool(false)));
5177 assert_eq!(current.metadata.get("big"), Some(&Value::from(1000)));
5178 let reloaded: Map<String, Value> = serde_json::from_str(&written).unwrap();
5180 store.reconcile(vec![Credential::from_file(&dir, &path, reloaded).unwrap()]);
5181 assert_eq!(store.get("a.json").unwrap().revision, revision);
5182 std::fs::write(&path, "{\"type\":\"claude\", \"disabled\":false}").unwrap();
5184 let mut metadata = Map::new();
5185 metadata.insert("type".into(), "claude".into());
5186 metadata.insert("disabled".into(), false.into());
5187 store.reconcile(vec![Credential::from_file(&dir, &path, metadata).unwrap()]);
5188 store
5189 .select(Selection::new("claude", "m"))
5190 .unwrap()
5191 .complete(Outcome::Success);
5192 assert_eq!(
5193 std::fs::read_to_string(&path).unwrap(),
5194 "{\"type\":\"claude\", \"disabled\":false}"
5195 );
5196 let _ = std::fs::remove_dir_all(&dir);
5197 }
5198
5199 #[test]
5202 fn stale_result_does_not_overwrite_a_refreshed_credential() {
5203 let dir = std::env::temp_dir().join(format!(
5204 "stale-{}-{}",
5205 std::process::id(),
5206 std::time::SystemTime::now()
5207 .duration_since(std::time::UNIX_EPOCH)
5208 .unwrap()
5209 .as_nanos()
5210 ));
5211 std::fs::create_dir_all(&dir).unwrap();
5212 let path = dir.join("a.json");
5213 std::fs::write(&path, r#"{"type":"claude","access_token":"old"}"#).unwrap();
5214 let metadata: Map<String, Value> = serde_json::from_str(r#"{"type":"claude","access_token":"old"}"#).unwrap();
5215 let store = CredentialStore::new(vec![Credential::from_file(&dir, &path, metadata).unwrap()]);
5216 let stale = store.get("a.json").unwrap();
5217 let mut patch = MetadataPatch::default();
5218 patch.set.insert("access_token".into(), "new".into());
5219 store.apply_patch("a.json", stale.revision, &patch).unwrap();
5220 let refreshed = std::fs::read_to_string(&path).unwrap();
5221 store.persist_credential(&stale);
5222 assert_eq!(std::fs::read_to_string(&path).unwrap(), refreshed);
5223 assert!(refreshed.contains("\"new\"") && refreshed.contains("\"disabled\":false"));
5224 let _ = std::fs::remove_dir_all(&dir);
5225 }
5226
5227 #[test]
5231 fn a_plugin_virtual_refresh_never_rewrites_the_shared_source_file() {
5232 let dir = std::env::temp_dir().join(format!("virtual-{}-{}", std::process::id(), line!()));
5233 std::fs::create_dir_all(&dir).unwrap();
5234 let path = dir.join("multi.json");
5235 let source = r#"{"type":"claude","accounts":[{"access_token":"a"},{"access_token":"b"}]}"#;
5236 std::fs::write(&path, source).unwrap();
5237 let metadata: Map<String, Value> = serde_json::from_str(r#"{"type":"claude","access_token":"a"}"#).unwrap();
5238 let mut credential = Credential::from_file(&dir, &path, metadata).unwrap();
5239 credential.id = "multi-a".into();
5240 credential.attributes.insert("plugin_virtual".into(), "true".into());
5241 let store = CredentialStore::new(vec![credential]);
5242 let current = store.get("multi-a").unwrap();
5243 let mut patch = MetadataPatch::default();
5244 patch.set.insert("access_token".into(), "refreshed".into());
5245 let committed = store.apply_patch("multi-a", current.revision, &patch).unwrap();
5246 assert_eq!(committed.metadata["access_token"], "refreshed");
5247 assert_eq!(store.get("multi-a").unwrap().metadata["access_token"], "refreshed");
5248 store.persist_credential(&committed);
5249 assert_eq!(std::fs::read_to_string(&path).unwrap(), source);
5250 let _ = std::fs::remove_dir_all(&dir);
5251 }
5252
5253 #[test]
5256 fn cooldowns_persist_restore_and_reset_through_cds_files() {
5257 let dir = std::env::temp_dir().join(format!(
5258 "cds-rt-{}-{}",
5259 std::process::id(),
5260 std::time::SystemTime::now()
5261 .duration_since(std::time::UNIX_EPOCH)
5262 .unwrap()
5263 .as_nanos()
5264 ));
5265 std::fs::create_dir_all(&dir).unwrap();
5266 let config = || {
5267 Config::parse(&format!(
5268 "auth-dir: {}\nrouting:\n cooldown:\n save-cooldown-status: true\n",
5269 dir.display()
5270 ))
5271 .unwrap()
5272 };
5273 let creds = || {
5274 let metadata = |p: &str| {
5275 let mut m = Map::new();
5276 m.insert("type".into(), p.into());
5277 m
5278 };
5279 vec![
5280 Credential::from_file(&dir, &dir.join("a.json"), metadata("claude")).unwrap(),
5281 Credential::from_file(&dir, &dir.join("b.json"), metadata("claude")).unwrap(),
5282 ]
5283 };
5284 let executors = || Executors {
5285 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5286 codex: Default::default(),
5287 openai: Default::default(),
5288 google: Default::default(),
5289 devices: Default::default(),
5290 };
5291 let rt = crate::testing::runtime(config(), creds(), executors());
5292 let mut selection = Selection::new("claude", "m1");
5293 selection.exclude.push("b.json".into());
5294 let lease = rt.store.select(selection.clone()).unwrap();
5295 let mut quota = ExecError::local(429, FailureScope::Model, "rate limited");
5296 quota.retry_after = Some(Duration::from_secs(120));
5297 lease.complete(Outcome::Failure(quota));
5298 let file = dir.join("a.cds");
5299 let written = std::fs::read_to_string(&file).expect("cooldown written");
5300 #[cfg(unix)]
5301 assert_eq!(
5302 std::fs::metadata(&file).unwrap().permissions().mode() & 0o777,
5303 0o600,
5304 "Go os.CreateTemp mode"
5305 );
5306 assert!(
5307 written.contains("\"model\": \"m1\"") && written.contains("\"reason\": \"quota\""),
5308 "{written}"
5309 );
5310 assert!(!dir.join("b.cds").exists());
5311 let modified = std::fs::metadata(&file).unwrap().modified().unwrap();
5313 let mut other = Selection::new("claude", "m1");
5314 other.exclude.push("a.json".into());
5315 rt.store.select(other).unwrap().complete(Outcome::Success);
5316 assert_eq!(std::fs::metadata(&file).unwrap().modified().unwrap(), modified);
5317
5318 drop(rt);
5320 let rt = crate::testing::runtime(config(), creds(), executors());
5321 let a = rt.store.get("a.json").unwrap();
5322 let wait = rt
5323 .store
5324 .scheduler
5325 .lock()
5326 .unwrap()
5327 .wait(&a, "m1", Instant::now())
5328 .unwrap();
5329 assert!(
5330 wait > Duration::from_secs(100) && wait <= Duration::from_secs(120),
5331 "{wait:?}"
5332 );
5333 assert_eq!(rt.store.reset_cooldown("a.json"), ["m1"]);
5334 assert!(!file.exists(), "reset removes the stale file");
5335
5336 let at = |secs: i64| {
5339 (chrono::Utc::now() + chrono::Duration::seconds(secs))
5340 .with_timezone(&chrono::FixedOffset::east_opt(7200).unwrap())
5341 .to_rfc3339_opts(chrono::SecondsFormat::Nanos, false)
5342 };
5343 let go = format!(
5344 r#"{{"version":1,"auth_id":"b.json","provider":"claude","updated_at":"{now}","records":[
5345{{"provider":"claude","auth_id":"b.json","status":"cooling","next_retry_after":"{late}","reason":"credential_quota","quota":{{"exceeded":true,"reason":"credential_quota","next_recover_at":"{late}","observed_at":"0001-01-01T00:00:00Z"}},"last_error":{{"message":"credential quota","retryable":false,"http_status":429}},"updated_at":"{now}"}},
5346{{"provider":"claude","auth_id":"b.json","model":"m2","status":"cooling","next_retry_after":"{soon}","reason":"unauthorized","quota":{{"exceeded":false,"next_recover_at":"0001-01-01T00:00:00Z","observed_at":"0001-01-01T00:00:00Z"}},"last_error":{{"message":"unauthorized","retryable":false,"http_status":401}},"updated_at":"{now}"}},
5347{{"provider":"claude","auth_id":"b.json","model":"m4","status":"cooling","next_retry_after":"{soon}","reason":"quota","quota":{{"exceeded":true,"reason":"quota","next_recover_at":"{late}","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}},
5348{{"provider":"claude","auth_id":"b.json","model":"m5","status":"cooling","next_retry_after":"{past}","reason":"quota","quota":{{"exceeded":true,"reason":"quota","next_recover_at":"{late}","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}},
5349{{"provider":"claude","auth_id":"b.json","model":"m3","status":"cooling","next_retry_after":"{past}","reason":"boom","quota":{{"exceeded":false,"next_recover_at":"0001-01-01T00:00:00Z","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}},
5350{{"provider":"claude","auth_id":"gone.json","model":"m1","status":"cooling","next_retry_after":"{late}","quota":{{"exceeded":false,"next_recover_at":"0001-01-01T00:00:00Z","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}}]}}"#,
5351 now = at(0),
5352 late = at(600),
5353 soon = at(60),
5354 past = at(-5),
5355 );
5356 std::fs::write(dir.join("b.cds"), go).unwrap();
5357 drop(rt);
5358 let rt = crate::testing::runtime(config(), creds(), executors());
5359 let b = rt.store.get("b.json").unwrap();
5360 let now = Instant::now();
5361 let scheduler = rt.store.scheduler.lock().unwrap();
5362 let wait = |m: &str| scheduler.wait(&b, m, now).unwrap_or_default().as_secs();
5363 assert!((590..=600).contains(&wait("m9")), "credential quota blocks every model");
5364 assert!(scheduler.quota_cooling(&b, "m9", now));
5365 let records = scheduler.records(&b, now, std::time::SystemTime::now());
5366 drop(scheduler);
5367 let models: Vec<&str> = records.iter().map(|r| r.model.as_str()).collect();
5368 assert_eq!(
5369 models,
5370 ["", "m2", "m4"],
5371 "expired (by retry deadline) and unknown-credential records are dropped"
5372 );
5373 let m4 = records.iter().find(|r| r.model == "m4").unwrap();
5374 let left = m4
5375 .next_retry_after
5376 .unwrap()
5377 .duration_since(std::time::SystemTime::now())
5378 .unwrap();
5379 assert!(
5380 left > Duration::from_secs(590),
5381 "the later quota recovery wins: {left:?}"
5382 );
5383 let rewritten = std::fs::read_to_string(dir.join("b.cds")).unwrap();
5384 assert!(!rewritten.contains("gone.json") && !rewritten.contains("\"m3\""));
5385
5386 rt.publish_config(Config::parse(&format!("auth-dir: {}\n", dir.display())).unwrap());
5388 rt.store.reset_cooldown("b.json");
5389 assert!(dir.join("b.cds").exists(), "no writes once disabled");
5390 let _ = std::fs::remove_dir_all(&dir);
5391 }
5392
5393 #[tokio::test]
5394 async fn preparation_waiter_observes_deletion_and_refresh_loop_is_replaceable() {
5395 let rt = Arc::new(crate::testing::runtime(
5396 Config::parse("").unwrap(),
5397 vec![cred("a.json", "claude", false)],
5398 Executors {
5399 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5400 codex: Default::default(),
5401 devices: Default::default(),
5402 openai: Default::default(),
5403 google: Default::default(),
5404 },
5405 ));
5406 let lock = rt.store.prepare_lock("a.json");
5407 let guard = lock.lock().await;
5408 let worker = {
5409 let rt = rt.clone();
5410 tokio::spawn(async move { rt.prepare_credential("a.json", &rt.config(), None).await })
5411 };
5412 tokio::task::yield_now().await;
5413 assert!(!worker.is_finished());
5414 rt.store.reconcile(Vec::new());
5415 drop(guard);
5416 let error = worker.await.unwrap().unwrap_err();
5417 assert_eq!(error.scope, FailureScope::Request);
5418 assert!(rt.store.snapshot().is_empty());
5419 rt.start_auto_refresh();
5420 let first = rt.refresh_task.lock().unwrap().clone().unwrap();
5421 rt.start_auto_refresh();
5422 tokio::task::yield_now().await;
5423 assert!(first.is_finished());
5424 let second = rt.refresh_task.lock().unwrap().clone().unwrap();
5425 rt.stop_auto_refresh();
5426 tokio::task::yield_now().await;
5427 assert!(second.is_finished());
5428 assert!(rt.refresh_task.lock().unwrap().is_none());
5429 }
5430
5431 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5435 async fn a_cancelled_refresh_holds_the_preparation_lock_through_its_commit() {
5436 let (hit_tx, mut hit) = tokio::sync::mpsc::unbounded_channel();
5437 let answer = Arc::new(tokio::sync::Semaphore::new(0));
5438 let app = axum::Router::new().route(
5439 "/oauth/token",
5440 axum::routing::post({
5441 let answer = answer.clone();
5442 move || async move {
5443 hit_tx.send(()).unwrap();
5444 answer.acquire().await.unwrap().forget();
5445 r#"{"access_token":"rotated","refresh_token":"rotated-rt","expires_in":3600}"#
5446 }
5447 }),
5448 );
5449 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5450 let token_url = format!("http://{}/oauth/token", listener.local_addr().unwrap());
5451 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5452 let dir = std::env::temp_dir().join(format!("cpa-prepare-cancel-{}", uuid::Uuid::new_v4()));
5453 std::fs::create_dir_all(&dir).unwrap();
5454 let path = dir.join("codex.json");
5455 std::fs::write(
5456 &path,
5457 r#"{"type":"codex","access_token":"outer","refresh_token":"outer-rt"}"#,
5458 )
5459 .unwrap();
5460 let mut cfg = Config::parse("").unwrap();
5461 cfg.auth_dir = dir.clone();
5462 let client = wreq::Client::new();
5463 let oauth =
5464 cpa_exec::codex_oauth::CodexOAuth::with_endpoints(client.clone(), &token_url, &token_url, &token_url);
5465 let rt = Arc::new(crate::testing::runtime(
5466 cfg.clone(),
5467 cpa_core::config::credentials::from_auth_dir(&cfg)
5468 .into_iter()
5469 .map(crate::testing::local)
5470 .collect(),
5471 Executors {
5472 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5473 codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
5474 devices: Default::default(),
5475 openai: Default::default(),
5476 google: Default::default(),
5477 },
5478 ));
5479 let refresh = tokio::spawn({
5480 let rt = rt.clone();
5481 async move { rt.refresh_credential("codex.json").await }
5482 });
5483 if tokio::time::timeout(Duration::from_secs(5), hit.recv()).await.is_err() {
5484 panic!("no refresh request: {:?}", refresh.await);
5485 }
5486 let (held_tx, held) = std::sync::mpsc::channel();
5489 let (release, release_rx) = std::sync::mpsc::channel::<()>();
5490 let holder = std::thread::spawn({
5491 let store = rt.store.clone();
5492 move || {
5493 let _inner = store.inner.write().unwrap();
5494 held_tx.send(()).unwrap();
5495 release_rx.recv().unwrap();
5496 }
5497 });
5498 held.recv().unwrap();
5499 answer.add_permits(1);
5500 tokio::time::sleep(Duration::from_millis(500)).await;
5501 refresh.abort();
5502 assert!(refresh.await.unwrap_err().is_cancelled());
5503 let lock = rt.store.prepare_lock("codex.json");
5504 assert!(
5505 tokio::time::timeout(Duration::from_millis(300), lock.clone().lock_owned())
5506 .await
5507 .is_err(),
5508 "the detached commit still holds the lock"
5509 );
5510 release.send(()).unwrap();
5511 holder.join().unwrap();
5512 let _binding = tokio::time::timeout(Duration::from_secs(5), lock.lock_owned())
5513 .await
5514 .unwrap();
5515 let current = rt.store.get("codex.json").unwrap();
5516 assert_eq!(current.str("access_token"), Some("rotated"));
5517 let mut patch = MetadataPatch::default();
5518 patch.set.insert("use_siwc".into(), true.into());
5519 let bound = rt.store.apply_patch("codex.json", current.revision, &patch).unwrap();
5520 let disk: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5521 for meta in [&disk, &Value::Object(bound.metadata.clone())] {
5522 assert_eq!(
5523 (&meta["access_token"], &meta["refresh_token"], &meta["use_siwc"]),
5524 (
5525 &serde_json::json!("rotated"),
5526 &serde_json::json!("rotated-rt"),
5527 &serde_json::json!(true)
5528 ),
5529 "{meta}"
5530 );
5531 }
5532 std::fs::remove_dir_all(dir).unwrap();
5533 }
5534
5535 #[tokio::test]
5541 async fn an_in_flight_refresh_does_not_overwrite_newer_credentials() {
5542 for (edit, want) in [
5543 (
5544 serde_json::json!({"access_token": "saved", "refresh_token": "saved-rt"}),
5545 ("saved", "saved-rt", None, "free"),
5546 ),
5547 (
5548 serde_json::json!({"note": "kept"}),
5549 ("rotated", "rotated-rt", Some("kept"), "free"),
5550 ),
5551 (
5553 serde_json::json!({"plan_type": "team"}),
5554 ("rotated", "rotated-rt", None, "team"),
5555 ),
5556 ] {
5557 let (hit_tx, mut hit) = tokio::sync::mpsc::unbounded_channel();
5558 let answer = Arc::new(tokio::sync::Semaphore::new(0));
5559 let app = axum::Router::new().route(
5560 "/oauth/token",
5561 axum::routing::post({
5562 let answer = answer.clone();
5563 move || async move {
5564 hit_tx.send(()).unwrap();
5565 answer.acquire().await.unwrap().forget();
5566 r#"{"access_token":"rotated","refresh_token":"rotated-rt","expires_in":3600}"#
5567 }
5568 }),
5569 );
5570 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5571 let token_url = format!("http://{}/oauth/token", listener.local_addr().unwrap());
5572 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5573 let dir = std::env::temp_dir().join(format!("cpa-refresh-stale-{}", uuid::Uuid::new_v4()));
5574 std::fs::create_dir_all(&dir).unwrap();
5575 let path = dir.join("codex.json");
5576 std::fs::write(
5577 &path,
5578 r#"{"type":"codex","access_token":"outer","refresh_token":"outer-rt","plan_type":"free"}"#,
5579 )
5580 .unwrap();
5581 let mut cfg = Config::parse("").unwrap();
5582 cfg.auth_dir = dir.clone();
5583 let client = wreq::Client::new();
5584 let oauth =
5585 cpa_exec::codex_oauth::CodexOAuth::with_endpoints(client.clone(), &token_url, &token_url, &token_url);
5586 let rt = Arc::new(crate::testing::runtime(
5587 cfg.clone(),
5588 cpa_core::config::credentials::from_auth_dir(&cfg)
5589 .into_iter()
5590 .map(crate::testing::local)
5591 .collect(),
5592 Executors {
5593 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5594 codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
5595 devices: Default::default(),
5596 openai: Default::default(),
5597 google: Default::default(),
5598 },
5599 ));
5600 let refresh = tokio::spawn({
5601 let rt = rt.clone();
5602 async move { rt.refresh_credential("codex.json").await }
5603 });
5604 tokio::time::timeout(Duration::from_secs(5), hit.recv())
5605 .await
5606 .expect("a refresh request");
5607 let mut patch = MetadataPatch::default();
5608 for (k, v) in edit.as_object().unwrap() {
5609 patch.set.insert(k.clone(), v.clone());
5610 }
5611 let before = rt.store.get("codex.json").unwrap().revision;
5612 rt.store.apply_patch("codex.json", before, &patch).unwrap();
5613 answer.add_permits(1);
5614 let answered = refresh.await.unwrap().expect("the refresh answers");
5615 let current = rt.store.get("codex.json").unwrap();
5616 let disk: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5617 for meta in [
5618 &disk,
5619 &Value::Object(current.metadata.clone()),
5620 &Value::Object(answered.metadata.clone()),
5621 ] {
5622 assert_eq!(
5623 (
5624 meta["access_token"].as_str().unwrap(),
5625 meta["refresh_token"].as_str().unwrap(),
5626 meta["note"].as_str(),
5627 meta["plan_type"].as_str().unwrap()
5628 ),
5629 want,
5630 "{edit}: {meta}"
5631 );
5632 }
5633 std::fs::remove_dir_all(dir).unwrap();
5634 }
5635 }
5636
5637 struct CodexDisk {
5640 rt: Arc<Runtime>,
5641 dir: std::path::PathBuf,
5642 grants: Arc<Mutex<Grants>>,
5643 gate: Arc<tokio::sync::Semaphore>,
5645 base: String,
5646 }
5647
5648 type Grants = Vec<(&'static str, String, String)>;
5649
5650 impl CodexDisk {
5651 async fn new(files: &[(&str, &Value)]) -> Self {
5652 Self::gated(files, 1000).await
5653 }
5654
5655 async fn gated(files: &[(&str, &Value)], permits: usize) -> Self {
5656 let grants = Arc::new(Mutex::new(Vec::new()));
5657 let gate = Arc::new(tokio::sync::Semaphore::new(permits));
5658 let route = |partition: &'static str, answer: &'static str| {
5659 let (grants, gate) = (grants.clone(), gate.clone());
5660 axum::routing::post(move |body: String| async move {
5661 let field = |key: &str| {
5662 url::form_urlencoded::parse(body.as_bytes())
5663 .find(|(k, _)| k == key)
5664 .map(|(_, v)| v.into_owned())
5665 .unwrap_or_default()
5666 };
5667 grants
5668 .lock()
5669 .unwrap()
5670 .push((partition, field("refresh_token"), field("client_id")));
5671 gate.acquire().await.unwrap().forget();
5672 answer
5673 })
5674 };
5675 let app = axum::Router::new()
5676 .route(
5677 "/oauth/token",
5678 route(
5679 "outer",
5680 r#"{"access_token":"outer-access-refreshed","refresh_token":"outer-ref-refreshed","expires_in":864000}"#,
5681 ),
5682 )
5683 .route(
5684 "/siwc/token",
5685 route(
5686 "siwc",
5687 r#"{"access_token":"siwc-access-refreshed","refresh_token":"siwc-ref-refreshed","scope":"openid chatgpt.tokens.use.direct","expires_in":3600}"#,
5688 ),
5689 );
5690 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5691 let base = format!("http://{}", listener.local_addr().unwrap());
5692 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5693 let dir = std::env::temp_dir().join(format!("cpa-codex-disk-{}", uuid::Uuid::new_v4()));
5694 std::fs::create_dir_all(&dir).unwrap();
5695 for (name, body) in files {
5696 std::fs::write(dir.join(name), body.to_string()).unwrap();
5697 }
5698 let rt = Self::runtime(&dir, &base);
5699 Self {
5700 rt,
5701 dir,
5702 grants,
5703 gate,
5704 base,
5705 }
5706 }
5707
5708 fn other_process(&self) -> Arc<Runtime> {
5711 Self::runtime(&self.dir, &self.base)
5712 }
5713
5714 fn runtime(dir: &Path, base: &str) -> Arc<Runtime> {
5715 let mut cfg = Config::parse("").unwrap();
5716 cfg.auth_dir = dir.to_owned();
5717 let client = wreq::Client::new();
5718 let oauth = cpa_exec::codex_oauth::CodexOAuth::with_endpoints(
5719 client.clone(),
5720 &format!("{base}/oauth/token"),
5721 "",
5722 "",
5723 )
5724 .with_siwc_token_url(&format!("{base}/siwc/token"))
5725 .with_retry_delay(Duration::from_millis(1));
5726 Arc::new(crate::testing::runtime(
5727 cfg.clone(),
5728 cpa_core::config::credentials::from_auth_dir(&cfg)
5729 .into_iter()
5730 .map(crate::testing::local)
5731 .collect(),
5732 Executors {
5733 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5734 codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
5735 devices: Default::default(),
5736 openai: Default::default(),
5737 google: Default::default(),
5738 },
5739 ))
5740 }
5741
5742 async fn lock_attempts(&self, name: &str, n: usize) {
5744 let path = self.dir.join(name);
5745 let attempted = async {
5746 while LOCK_ATTEMPTS.lock().unwrap().iter().filter(|p| **p == path).count() < n {
5747 tokio::time::sleep(Duration::from_millis(5)).await;
5748 }
5749 };
5750 tokio::time::timeout(Duration::from_secs(5), attempted)
5751 .await
5752 .expect("the refresh went to lock the file");
5753 }
5754
5755 async fn granted(&self) {
5757 let granted = async {
5758 while self.grants.lock().unwrap().is_empty() {
5759 tokio::time::sleep(Duration::from_millis(5)).await;
5760 }
5761 };
5762 tokio::time::timeout(Duration::from_secs(5), granted)
5763 .await
5764 .expect("a grant");
5765 }
5766
5767 fn grants(&self) -> Grants {
5768 std::mem::take(&mut *self.grants.lock().unwrap())
5769 }
5770
5771 fn tokens(&self, name: &str, fields: &[&str]) -> Vec<Value> {
5773 let disk: Value = serde_json::from_slice(&std::fs::read(self.dir.join(name)).unwrap()).unwrap();
5774 let stored = Value::Object(self.rt.store.get(name).unwrap().metadata.clone());
5775 let pick = |m: &Value| {
5776 fields
5777 .iter()
5778 .map(|f| m.pointer(f).cloned().unwrap_or(Value::Null))
5779 .collect()
5780 };
5781 let (disk, stored): (Vec<Value>, Vec<Value>) = (pick(&disk), pick(&stored));
5782 assert_eq!(disk, stored, "file and store differ");
5783 disk
5784 }
5785 }
5786
5787 fn codex_file(outer: (&str, chrono::Duration), siwc: (&str, &str, chrono::Duration)) -> Value {
5788 let now = chrono::Utc::now();
5789 serde_json::json!({
5790 "type": "codex",
5791 "access_token": format!("outer-{}", outer.0),
5792 "refresh_token": format!("outer-ref-{}", outer.0),
5793 "expired": (now + outer.1).to_rfc3339(),
5794 "use_siwc": true,
5795 "siwc": {
5796 "access_token": format!("siwc-{}", siwc.0),
5797 "refresh_token": format!("siwc-ref-{}", siwc.0),
5798 "client_id": siwc.1,
5799 "device_id": "e61bbe28-07ef-466d-8e5d-a344f94ab305",
5800 "expired": (now + siwc.2).to_rfc3339(),
5801 },
5802 })
5803 }
5804
5805 #[tokio::test(start_paused = true)]
5809 async fn a_lock_wait_stops_at_shutdown_even_when_the_lock_frees() {
5810 let dir = std::env::temp_dir().join(format!("cpa-lock-wait-{}", uuid::Uuid::new_v4()));
5811 std::fs::create_dir_all(&dir).unwrap();
5812 let path = dir.join("codex.json");
5813 let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
5814 for _ in 0..32 {
5815 let held = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
5816 let shutdown = tokio_util::sync::CancellationToken::new();
5817 let mut wait = std::pin::pin!(wait_codex_lock(&path, &shutdown));
5818 assert!(wait.as_mut().poll(&mut cx).is_pending(), "the lock is held");
5819 drop(held);
5820 tokio::time::advance(LOCK_POLL_CEILING).await;
5821 shutdown.cancel();
5822 let std::task::Poll::Ready(waited) = wait.as_mut().poll(&mut cx) else {
5823 panic!("the wait ends");
5824 };
5825 assert_eq!(waited.map(drop), Err(LOCK_WAIT_STOPPED.to_owned()));
5826 }
5827 std::fs::remove_dir_all(dir).unwrap();
5828 }
5829
5830 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5835 async fn a_codex_refresh_waits_for_the_file_lock_and_adopts_what_another_process_rotated() {
5836 let (days, minutes) = (chrono::Duration::days(5), chrono::Duration::minutes(2));
5837 let f = CodexDisk::new(&[(
5838 "codex.json",
5839 &codex_file(("keep", days), ("old", "oaiapp_shared", minutes)),
5840 )])
5841 .await;
5842 let path = f.dir.join("codex.json");
5843 let (locked_tx, locked) = std::sync::mpsc::channel();
5844 let (release_tx, release) = std::sync::mpsc::channel::<()>();
5845 let other = std::thread::spawn({
5846 let path = path.clone();
5847 move || {
5848 let lock = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
5849 locked_tx.send(()).unwrap();
5850 release.recv().unwrap();
5851 let rotated = codex_file(("keep", days), ("other", "oaiapp_shared", chrono::Duration::hours(1)));
5852 std::fs::write(&path, rotated.to_string()).unwrap();
5853 drop(lock);
5854 }
5855 });
5856 locked.recv().unwrap();
5857 let refresh = tokio::spawn({
5858 let rt = f.rt.clone();
5859 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
5860 });
5861 f.lock_attempts("codex.json", 1).await;
5862 tokio::time::sleep(Duration::from_millis(50)).await;
5863 assert!(!refresh.is_finished(), "the refresh waits for the lock");
5864 assert_eq!(f.grants(), [], "nothing refreshes under another process's lock");
5865 release_tx.send(()).unwrap();
5866 other.join().unwrap();
5867 tokio::time::timeout(Duration::from_secs(5), refresh)
5868 .await
5869 .unwrap()
5870 .unwrap()
5871 .unwrap();
5872 assert_eq!(f.grants(), [], "the rotated token is not refreshed again");
5873 assert_eq!(
5874 f.tokens(
5875 "codex.json",
5876 &["/access_token", "/siwc/access_token", "/siwc/refresh_token"]
5877 ),
5878 ["outer-keep", "siwc-other", "siwc-ref-other"]
5879 );
5880 std::fs::remove_dir_all(&f.dir).unwrap();
5881 }
5882
5883 #[tokio::test]
5887 async fn a_codex_refresh_spends_the_tokens_on_disk_not_the_stale_snapshot() {
5888 let hour = chrono::Duration::hours(1);
5889 let minute = chrono::Duration::minutes(1);
5890 let f = CodexDisk::new(&[("codex.json", &codex_file(("A", hour), ("A", "oaiapp_A", minute)))]).await;
5891 let disk_b = codex_file(("B", hour), ("B", "oaiapp_B", minute));
5892 std::fs::write(f.dir.join("codex.json"), disk_b.to_string()).unwrap();
5893 f.rt.prepare_credential("codex.json", &f.rt.config(), None)
5894 .await
5895 .unwrap();
5896 assert_eq!(
5897 f.grants(),
5898 [
5899 (
5900 "outer",
5901 "outer-ref-B".to_owned(),
5902 cpa_exec::codex_oauth::CLIENT_ID.to_owned()
5903 ),
5904 ("siwc", "siwc-ref-B".to_owned(), "oaiapp_B".to_owned()),
5905 ]
5906 );
5907 assert_eq!(
5908 f.tokens(
5909 "codex.json",
5910 &[
5911 "/access_token",
5912 "/refresh_token",
5913 "/siwc/access_token",
5914 "/siwc/client_id"
5915 ]
5916 ),
5917 [
5918 "outer-access-refreshed",
5919 "outer-ref-refreshed",
5920 "siwc-access-refreshed",
5921 "oaiapp_B"
5922 ]
5923 );
5924 std::fs::remove_dir_all(&f.dir).unwrap();
5925 }
5926
5927 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5931 async fn a_codex_refresh_keeps_its_result_when_the_watcher_loads_the_adopted_tokens() {
5932 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
5933 let f = CodexDisk::gated(
5934 &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_A", minute)))],
5935 0,
5936 )
5937 .await;
5938 let disk_b = codex_file(("keep", days), ("B", "oaiapp_B", minute));
5939 std::fs::write(f.dir.join("codex.json"), disk_b.to_string()).unwrap();
5940 let refresh = tokio::spawn({
5941 let rt = f.rt.clone();
5942 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
5943 });
5944 f.granted().await;
5945 let mut reload = MetadataPatch::default();
5947 reload.set.insert("siwc".into(), disk_b["siwc"].clone());
5948 let revision = f.rt.store.get("codex.json").unwrap().revision;
5949 f.rt.store.apply_patch("codex.json", revision, &reload).unwrap();
5950 f.gate.add_permits(1);
5951 tokio::time::timeout(Duration::from_secs(5), refresh)
5952 .await
5953 .unwrap()
5954 .unwrap()
5955 .unwrap();
5956 assert_eq!(f.grants(), [("siwc", "siwc-ref-B".to_owned(), "oaiapp_B".to_owned())]);
5957 assert_eq!(
5958 f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
5959 ["siwc-access-refreshed", "siwc-ref-refreshed"]
5960 );
5961 std::fs::remove_dir_all(&f.dir).unwrap();
5962 }
5963
5964 #[tokio::test]
5967 async fn a_codex_refresh_adopts_a_rotated_file_whatever_its_type_spelling() {
5968 let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
5969 let mut file = codex_file(("A", days), ("keep", "oaiapp_x", hour));
5970 file["type"] = "Codex".into();
5971 file["use_siwc"] = false.into();
5972 let f = CodexDisk::new(&[("codex.json", &file)]).await;
5973 let mut rotated = file.clone();
5974 rotated["access_token"] = "outer-B".into();
5975 rotated["refresh_token"] = "outer-ref-B".into();
5976 std::fs::write(f.dir.join("codex.json"), rotated.to_string()).unwrap();
5977 let current = f.rt.store.get("codex.json").unwrap();
5978 let cfg = f.rt.config();
5979 assert!(f.rt.refresh_after_unauthorized(¤t, &cfg).await.is_some());
5980 assert_eq!(f.grants(), []);
5981 assert_eq!(
5982 f.tokens("codex.json", &["/type", "/access_token", "/refresh_token"]),
5983 ["Codex", "outer-B", "outer-ref-B"]
5984 );
5985 std::fs::remove_dir_all(&f.dir).unwrap();
5986 }
5987
5988 #[tokio::test]
5993 async fn a_rejected_codex_token_adopts_a_rotated_file_and_a_corrupt_file_refreshes_nothing() {
5994 let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
5995 let f = CodexDisk::new(&[(
5996 "codex.json",
5997 &codex_file(("keep", days), ("rejected", "oaiapp_x", hour)),
5998 )])
5999 .await;
6000 let path = f.dir.join("codex.json");
6001 std::fs::write(
6002 &path,
6003 codex_file(("keep", days), ("other", "oaiapp_x", hour)).to_string(),
6004 )
6005 .unwrap();
6006 let cfg = f.rt.config();
6007 let current = f.rt.store.get("codex.json").unwrap();
6008 let refreshed = f.rt.refresh_after_unauthorized(¤t, &cfg).await.expect("adopted");
6009 assert_eq!(f.grants(), []);
6010 assert_eq!(refreshed.metadata["siwc"]["access_token"], "siwc-other");
6011 assert_eq!(
6012 f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6013 ["siwc-other", "siwc-ref-other"]
6014 );
6015 std::fs::write(&path, "{corrupt").unwrap();
6016 assert!(f.rt.refresh_after_unauthorized(&refreshed, &cfg).await.is_none());
6017 assert_eq!(f.grants(), []);
6018 std::fs::remove_dir_all(&f.dir).unwrap();
6019 }
6020
6021 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6027 async fn a_codex_refresh_of_an_adopted_partition_lands_after_an_edit_to_the_snapshot() {
6028 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6029 let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))]).await;
6030 let path = f.dir.join("codex.json");
6031 let (locked_tx, locked) = std::sync::mpsc::channel();
6032 let (release_tx, release) = std::sync::mpsc::channel::<()>();
6033 let other = std::thread::spawn({
6034 let path = path.clone();
6035 move || {
6036 let lock = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
6037 locked_tx.send(()).unwrap();
6038 release.recv().unwrap();
6039 let mut rotated = codex_file(("keep", days), ("B", "oaiapp_x", minute));
6040 rotated["priority"] = 7.into();
6041 std::fs::write(&path, rotated.to_string()).unwrap();
6042 drop(lock);
6043 }
6044 });
6045 locked.recv().unwrap();
6046 let refresh = tokio::spawn({
6047 let rt = f.rt.clone();
6048 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6049 });
6050 f.lock_attempts("codex.json", 1).await;
6051 let mut edit = MetadataPatch::default();
6053 edit.set.insert("priority".into(), 7.into());
6054 let revision = f.rt.store.get("codex.json").unwrap().revision;
6055 f.rt.store.apply_patch("codex.json", revision, &edit).unwrap();
6056 release_tx.send(()).unwrap();
6057 other.join().unwrap();
6058 tokio::time::timeout(Duration::from_secs(5), refresh)
6059 .await
6060 .unwrap()
6061 .unwrap()
6062 .unwrap();
6063 assert_eq!(f.grants(), [("siwc", "siwc-ref-B".to_owned(), "oaiapp_x".to_owned())]);
6064 assert_eq!(
6065 f.tokens(
6066 "codex.json",
6067 &["/siwc/access_token", "/siwc/refresh_token", "/priority"]
6068 ),
6069 [
6070 Value::from("siwc-access-refreshed"),
6071 Value::from("siwc-ref-refreshed"),
6072 Value::from(7)
6073 ]
6074 );
6075 std::fs::remove_dir_all(&f.dir).unwrap();
6076 }
6077
6078 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6083 async fn a_cancelled_codex_refresh_writes_its_tokens_before_another_process_reads_the_file() {
6084 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6085 let f = CodexDisk::gated(
6086 &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6087 0,
6088 )
6089 .await;
6090 let other = f.other_process();
6091 let refresh = tokio::spawn({
6092 let rt = f.rt.clone();
6093 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6094 });
6095 f.granted().await;
6096 refresh.abort();
6097 assert!(refresh.await.unwrap_err().is_cancelled());
6098 let waiting = tokio::spawn({
6099 let other = other.clone();
6100 async move { other.prepare_credential("codex.json", &other.config(), None).await }
6101 });
6102 f.lock_attempts("codex.json", 2).await;
6103 f.gate.add_permits(10);
6104 let adopted = tokio::time::timeout(Duration::from_secs(5), waiting)
6105 .await
6106 .unwrap()
6107 .unwrap()
6108 .unwrap();
6109 assert_eq!(f.grants(), [("siwc", "siwc-ref-A".to_owned(), "oaiapp_x".to_owned())]);
6110 assert_eq!(adopted.metadata["siwc"]["refresh_token"], "siwc-ref-refreshed");
6111 assert_eq!(
6112 f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6113 ["siwc-access-refreshed", "siwc-ref-refreshed"]
6114 );
6115 std::fs::remove_dir_all(&f.dir).unwrap();
6116 }
6117
6118 #[tokio::test]
6123 async fn a_codex_refresh_keeps_a_disabled_flag_it_finds_in_the_file() {
6124 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6125 let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))]).await;
6126 let mut disk_b = codex_file(("keep", days), ("B", "oaiapp_x", minute));
6127 disk_b["disabled"] = true.into();
6128 std::fs::write(f.dir.join("codex.json"), disk_b.to_string()).unwrap();
6129 let refreshed =
6130 f.rt.prepare_credential("codex.json", &f.rt.config(), None)
6131 .await
6132 .unwrap();
6133 assert_eq!(f.grants(), [("siwc", "siwc-ref-B".to_owned(), "oaiapp_x".to_owned())]);
6134 assert!(refreshed.disabled);
6135 assert_eq!(
6136 f.tokens("codex.json", &["/disabled", "/siwc/access_token"]),
6137 [Value::Bool(true), Value::from("siwc-access-refreshed")]
6138 );
6139 std::fs::remove_dir_all(&f.dir).unwrap();
6140 }
6141
6142 #[tokio::test]
6147 async fn a_rejected_codex_token_refreshes_when_the_file_rotated_only_its_refresh_token() {
6148 let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
6149 for use_siwc in [true, false] {
6150 let mut file = codex_file(("A", days), ("A", "oaiapp_x", hour));
6151 file["use_siwc"] = use_siwc.into();
6152 let f = CodexDisk::new(&[("codex.json", &file)]).await;
6153 let (prefix, client) = match use_siwc {
6154 true => ("siwc", "oaiapp_x"),
6155 false => ("outer", cpa_exec::codex_oauth::CLIENT_ID),
6156 };
6157 let pointer = if use_siwc {
6158 "/siwc/refresh_token"
6159 } else {
6160 "/refresh_token"
6161 };
6162 *file.pointer_mut(pointer).unwrap() = format!("{prefix}-ref-B").into();
6163 std::fs::write(f.dir.join("codex.json"), file.to_string()).unwrap();
6164 let current = f.rt.store.get("codex.json").unwrap();
6165 let cfg = f.rt.config();
6166 let refreshed = f.rt.refresh_after_unauthorized(¤t, &cfg).await;
6167 assert_eq!(
6168 f.grants(),
6169 [(
6170 if use_siwc { "siwc" } else { "outer" },
6171 format!("{prefix}-ref-B"),
6172 client.to_owned()
6173 )],
6174 "use_siwc {use_siwc}"
6175 );
6176 let access = if use_siwc {
6177 "/siwc/access_token"
6178 } else {
6179 "/access_token"
6180 };
6181 let refreshed = Value::Object(refreshed.expect("refreshed").metadata.clone());
6182 assert_eq!(
6183 refreshed.pointer(access).unwrap(),
6184 &format!("{prefix}-access-refreshed")
6185 );
6186 assert_eq!(
6187 f.tokens("codex.json", &[access]),
6188 [format!("{prefix}-access-refreshed")]
6189 );
6190 std::fs::remove_dir_all(&f.dir).unwrap();
6191 }
6192 }
6193
6194 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6197 async fn a_codex_refresh_fails_when_its_file_is_removed_while_it_waits_for_the_lock() {
6198 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6199 let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))]).await;
6200 let path = f.dir.join("codex.json");
6201 let (locked_tx, locked) = std::sync::mpsc::channel();
6202 let (release_tx, release) = std::sync::mpsc::channel::<()>();
6203 let other = std::thread::spawn({
6204 let path = path.clone();
6205 move || {
6206 let lock = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
6207 locked_tx.send(()).unwrap();
6208 release.recv().unwrap();
6209 std::fs::remove_file(&path).unwrap();
6210 drop(lock);
6211 }
6212 });
6213 locked.recv().unwrap();
6214 let refresh = tokio::spawn({
6215 let rt = f.rt.clone();
6216 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6217 });
6218 f.lock_attempts("codex.json", 1).await;
6219 release_tx.send(()).unwrap();
6220 other.join().unwrap();
6221 let result = tokio::time::timeout(Duration::from_secs(5), refresh)
6222 .await
6223 .unwrap()
6224 .unwrap();
6225 assert!(result.is_err(), "{result:?}");
6226 assert_eq!(f.grants(), []);
6227 std::fs::remove_dir_all(&f.dir).unwrap();
6228 }
6229
6230 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6236 async fn a_codex_refresh_lands_over_the_snapshot_tokens_restored_while_it_ran() {
6237 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6238 let file_a = codex_file(("A", minute), ("A", "oaiapp_x", days));
6239 let f = CodexDisk::gated(&[("codex.json", &file_a)], 0).await;
6240 let path = f.dir.join("codex.json");
6241 let mut file_b = codex_file(("A", minute), ("B", "oaiapp_x", days));
6242 file_b["siwc"]["expired"] = serde_json::from_str("9007199254740993").unwrap();
6243 file_b["siwc"]["last_refresh"] = serde_json::from_str("2000000000.0").unwrap();
6244 std::fs::write(&path, serde_json::to_string(&file_b).unwrap()).unwrap();
6245 let refresh = tokio::spawn({
6246 let rt = f.rt.clone();
6247 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6248 });
6249 f.granted().await;
6250 let mut restored = file_a.clone();
6251 restored["openapi-location"] = "x".into();
6252 std::fs::write(&path, restored.to_string()).unwrap();
6253 f.gate.add_permits(1);
6254 tokio::time::timeout(Duration::from_secs(5), refresh)
6255 .await
6256 .unwrap()
6257 .unwrap()
6258 .unwrap();
6259 assert_eq!(
6260 f.grants(),
6261 [(
6262 "outer",
6263 "outer-ref-A".to_owned(),
6264 cpa_exec::codex_oauth::CLIENT_ID.to_owned()
6265 )]
6266 );
6267 assert_eq!(
6268 f.tokens(
6269 "codex.json",
6270 &["/access_token", "/siwc/access_token", "/siwc/refresh_token"]
6271 ),
6272 ["outer-access-refreshed", "siwc-B", "siwc-ref-B"]
6273 );
6274 let written = std::fs::read_to_string(&path).unwrap();
6275 for want in [
6276 r#""expired":9007199254740992,"#,
6277 r#""last_refresh":2000000000,"#,
6278 r#""openapi_location":"x""#,
6279 ] {
6280 assert!(written.contains(want), "{want} in {written}");
6281 }
6282 std::fs::remove_dir_all(&f.dir).unwrap();
6283 }
6284
6285 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6289 async fn a_codex_refresh_is_discarded_when_the_credential_was_replaced_meanwhile() {
6290 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6291 let f = CodexDisk::gated(
6292 &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6293 0,
6294 )
6295 .await;
6296 let refresh = tokio::spawn({
6297 let rt = f.rt.clone();
6298 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6299 });
6300 f.granted().await;
6301 let mut replace = MetadataPatch::default();
6302 replace.set.insert(
6303 "siwc".into(),
6304 codex_file(("keep", days), ("D", "oaiapp_x", days))["siwc"].clone(),
6305 );
6306 let before = f.rt.store.get("codex.json").unwrap().revision;
6307 let replaced = f.rt.store.apply_patch("codex.json", before, &replace).unwrap();
6308 f.gate.add_permits(1);
6309 let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6310 .await
6311 .unwrap()
6312 .unwrap()
6313 .unwrap();
6314 assert_eq!(f.grants(), [("siwc", "siwc-ref-A".to_owned(), "oaiapp_x".to_owned())]);
6315 assert_eq!(answered.revision, replaced.revision);
6316 assert_eq!(f.rt.store.get("codex.json").unwrap().revision, replaced.revision);
6317 assert_eq!(
6318 f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6319 ["siwc-D", "siwc-ref-D"]
6320 );
6321 std::fs::remove_dir_all(&f.dir).unwrap();
6322 }
6323
6324 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6329 async fn a_codex_refresh_is_discarded_when_the_credential_version_moved_meanwhile() {
6330 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6331 let mut file = codex_file(("A", minute), ("keep", "oaiapp_x", days));
6332 file["use_siwc"] = false.into();
6333 file["account_id"] = "acct-A".into();
6334 let f = CodexDisk::gated(&[("codex.json", &file)], 0).await;
6335 let refresh = tokio::spawn({
6336 let rt = f.rt.clone();
6337 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6338 });
6339 f.granted().await;
6340 let mut last = None;
6341 for edit in [
6342 serde_json::json!({"access_token": "outer-B", "account_id": "acct-X"}),
6343 serde_json::json!({"access_token": "outer-A"}),
6344 ] {
6345 let mut patch = MetadataPatch::default();
6346 for (k, v) in edit.as_object().unwrap() {
6347 patch.set.insert(k.clone(), v.clone());
6348 }
6349 let revision = f.rt.store.get("codex.json").unwrap().revision;
6350 last = Some(f.rt.store.apply_patch("codex.json", revision, &patch).unwrap());
6351 }
6352 let edited = last.unwrap();
6353 f.gate.add_permits(1);
6354 let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6355 .await
6356 .unwrap()
6357 .unwrap()
6358 .unwrap();
6359 assert_eq!(f.grants().len(), 1);
6360 assert_eq!(answered.revision, edited.revision);
6361 assert_eq!(f.rt.store.get("codex.json").unwrap().revision, edited.revision);
6362 assert_eq!(
6363 f.tokens("codex.json", &["/access_token", "/account_id"]),
6364 ["outer-A", "acct-X"]
6365 );
6366 std::fs::remove_dir_all(&f.dir).unwrap();
6367 }
6368
6369 #[tokio::test]
6374 async fn a_discarded_adopt_only_codex_refresh_changes_nothing() {
6375 let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
6376 let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", hour)))]).await;
6377 let path = f.dir.join("codex.json");
6378 let select = |model: &str| f.rt.store.select(Selection::new("codex", model)).unwrap();
6379 select("m").complete(Outcome::Failure(ExecError::local(
6380 401,
6381 FailureScope::Credential,
6382 "unauthorized",
6383 )));
6384 let mut file_b = codex_file(("keep", days), ("B", "oaiapp_x", hour));
6386 file_b["request-retry"] = 2.into();
6387 let bytes_b = file_b.to_string();
6388 std::fs::write(&path, &bytes_b).unwrap();
6389 let cooldowns = Arc::new(Mutex::new(None));
6392 BEFORE_ADMIT.lock().unwrap().push((
6393 path.clone(),
6394 Box::new({
6395 let (store, dir, path, cooldowns) =
6396 (f.rt.store.clone(), f.dir.clone(), path.clone(), cooldowns.clone());
6397 move || {
6398 let d = codex_file(("keep", days), ("D", "oaiapp_x", days));
6399 let d = Credential::from_file(&dir, &path, d.as_object().cloned().unwrap()).unwrap();
6400 store.reconcile(vec![crate::testing::local(d)]);
6401 let rejected = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
6405 let (d, policy) = (store.get("codex.json").unwrap(), store.policy.read().unwrap().clone());
6406 let mut scheduler = store.scheduler.lock().unwrap();
6407 scheduler.record(&d, "m", &rejected, &policy, Instant::now());
6408 drop(scheduler);
6409 *cooldowns.lock().unwrap() = Some(store.cooldowns("codex.json").len());
6410 }
6411 }),
6412 ));
6413 let current = f.rt.store.get("codex.json").unwrap();
6414 let cfg = f.rt.config();
6415 assert!(f.rt.refresh_after_unauthorized(¤t, &cfg).await.is_some());
6416 assert_eq!(f.grants(), []);
6417 let stored = f.rt.store.get("codex.json").unwrap();
6418 assert_eq!(stored.metadata["siwc"]["access_token"], "siwc-D");
6419 assert_ne!(stored.revision, current.revision);
6420 assert_eq!(std::fs::read_to_string(&path).unwrap(), bytes_b);
6421 let before = cooldowns.lock().unwrap().expect("the hook ran");
6422 assert_eq!(before, 1, "the 401 cooldown is still there");
6423 assert_eq!(f.rt.store.cooldowns("codex.json").len(), before);
6424 assert!(
6425 !f.rt
6426 .store
6427 .unauthorized
6428 .lock()
6429 .unwrap()
6430 .refreshed
6431 .contains_key("codex.json")
6432 );
6433 std::fs::remove_dir_all(&f.dir).unwrap();
6434 }
6435
6436 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6440 async fn a_codex_refresh_is_admitted_over_tokens_put_back_meanwhile() {
6441 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6442 let file_a = codex_file(("keep", days), ("A", "oaiapp_x", minute));
6443 let f = CodexDisk::gated(&[("codex.json", &file_a)], 0).await;
6444 let refresh = tokio::spawn({
6445 let rt = f.rt.clone();
6446 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6447 });
6448 f.granted().await;
6449 let version = || f.rt.store.get_registered("codex.json").unwrap().1.credential_version;
6450 let started = version();
6451 for siwc in [
6452 codex_file(("keep", days), ("D", "oaiapp_x", days))["siwc"].clone(),
6453 file_a["siwc"].clone(),
6454 ] {
6455 let mut patch = MetadataPatch::default();
6456 patch.set.insert("siwc".into(), siwc);
6457 let revision = f.rt.store.get("codex.json").unwrap().revision;
6458 f.rt.store.apply_patch("codex.json", revision, &patch).unwrap();
6459 }
6460 assert_ne!(version(), started, "the put-back moved the credential version");
6461 f.gate.add_permits(1);
6462 let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6463 .await
6464 .unwrap()
6465 .unwrap()
6466 .unwrap();
6467 assert_eq!(answered.metadata["siwc"]["access_token"], "siwc-access-refreshed");
6468 assert_eq!(
6469 f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6470 ["siwc-access-refreshed", "siwc-ref-refreshed"]
6471 );
6472 std::fs::remove_dir_all(&f.dir).unwrap();
6473 }
6474
6475 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6480 async fn a_codex_refresh_publishes_its_tokens_when_the_save_fails() {
6481 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6482 for truncate in [false, true] {
6483 let f = CodexDisk::gated(
6484 &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6485 0,
6486 )
6487 .await;
6488 let path = f.dir.join("codex.json");
6489 BEFORE_ADMIT.lock().unwrap().push((
6490 path.clone(),
6491 Box::new({
6492 let path = path.clone();
6493 move || match truncate {
6494 true => std::fs::write(&path, r#"{"type":"#).unwrap(),
6495 false => std::fs::remove_file(&path).unwrap(),
6496 }
6497 }),
6498 ));
6499 let refresh = tokio::spawn({
6500 let rt = f.rt.clone();
6501 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6502 });
6503 f.granted().await;
6504 f.gate.add_permits(1);
6505 let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6506 .await
6507 .unwrap()
6508 .unwrap()
6509 .expect("the refresh succeeds");
6510 assert_eq!(answered.metadata["siwc"]["refresh_token"], "siwc-ref-refreshed");
6511 let stored = f.rt.store.get("codex.json").unwrap();
6512 assert_eq!(stored.metadata["siwc"]["refresh_token"], "siwc-ref-refreshed");
6513 let disk = std::fs::read_to_string(&path).unwrap();
6514 if truncate {
6515 assert_eq!(disk, r#"{"type":"#);
6516 } else {
6517 let disk: Value = serde_json::from_str(&disk).unwrap();
6518 assert_eq!(
6519 (&disk["siwc"]["refresh_token"], &disk["disabled"]),
6520 (&"siwc-ref-refreshed".into(), &false.into())
6521 );
6522 }
6523 std::fs::remove_dir_all(&f.dir).unwrap();
6524 }
6525 }
6526
6527 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6531 async fn shutdown_stops_codex_lock_waits_and_drains_started_grants() {
6532 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6533 let f = CodexDisk::gated(
6534 &[
6535 ("locked.json", &codex_file(("keep", days), ("L", "oaiapp_x", minute))),
6536 ("granting.json", &codex_file(("keep", days), ("G", "oaiapp_x", minute))),
6537 ],
6538 0,
6539 )
6540 .await;
6541 let held = cpa_exec::codex_siwc_bind::lock_file(&f.dir.join("locked.json")).unwrap();
6542 let refresh = |id: &'static str| {
6543 let rt = f.rt.clone();
6544 tokio::spawn(async move { rt.prepare_credential(id, &rt.config(), None).await })
6545 };
6546 let (waiting, granting) = (refresh("locked.json"), refresh("granting.json"));
6547 f.lock_attempts("locked.json", 1).await;
6548 f.granted().await;
6549 let shutdown = tokio::spawn({
6550 let rt = f.rt.clone();
6551 async move { rt.shutdown_codex_refreshes().await }
6552 });
6553 let waited = tokio::time::timeout(Duration::from_secs(5), waiting)
6554 .await
6555 .unwrap()
6556 .unwrap();
6557 assert!(waited.is_err(), "{waited:?}");
6558 tokio::time::sleep(Duration::from_millis(100)).await;
6559 assert!(!shutdown.is_finished(), "shutdown waits for the started grant");
6560 f.gate.add_permits(1);
6561 tokio::time::timeout(Duration::from_secs(5), shutdown)
6562 .await
6563 .unwrap()
6564 .unwrap();
6565 assert!(granting.await.unwrap().is_ok());
6566 assert_eq!(f.grants(), [("siwc", "siwc-ref-G".to_owned(), "oaiapp_x".to_owned())]);
6567 assert_eq!(
6568 f.tokens("granting.json", &["/siwc/refresh_token"]),
6569 ["siwc-ref-refreshed"]
6570 );
6571 drop(held);
6572 std::fs::remove_dir_all(&f.dir).unwrap();
6573 }
6574
6575 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6580 async fn a_codex_refresh_writes_its_tokens_when_the_id_was_registered_again_meanwhile() {
6581 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6582 let mut file_a = codex_file(("keep", days), ("A", "oaiapp_x", minute));
6583 file_a["request-retry"] = 2.into();
6584 let f = CodexDisk::gated(&[("codex.json", &file_a)], 0).await;
6585 let path = f.dir.join("codex.json");
6586 let refresh = tokio::spawn({
6587 let rt = f.rt.clone();
6588 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6589 });
6590 f.granted().await;
6591 std::fs::remove_file(&path).unwrap();
6592 std::fs::write(&path, file_a.to_string()).unwrap();
6593 let cfg = f.rt.config();
6594 let load = || {
6595 cpa_core::config::credentials::from_auth_dir(&cfg)
6596 .into_iter()
6597 .map(crate::testing::local)
6598 .collect::<Vec<_>>()
6599 };
6600 f.rt.store.reconcile(vec![]);
6601 f.rt.store.reconcile(load());
6602 f.gate.add_permits(1);
6603 let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6604 .await
6605 .unwrap()
6606 .unwrap();
6607 assert!(answered.is_err(), "{answered:?}");
6608 let disk: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6609 assert_eq!(disk["siwc"]["refresh_token"], "siwc-ref-refreshed");
6610 assert_eq!(
6612 (&disk["request-retry"], &disk["request_retry"]),
6613 (&2.into(), &Value::Null)
6614 );
6615 assert_eq!(
6616 f.rt.store.get("codex.json").unwrap().metadata["siwc"]["refresh_token"],
6617 "siwc-ref-A"
6618 );
6619 f.rt.store.reconcile(load());
6621 assert_eq!(
6622 f.rt.store.get("codex.json").unwrap().metadata["siwc"]["refresh_token"],
6623 "siwc-ref-refreshed"
6624 );
6625 std::fs::remove_dir_all(&f.dir).unwrap();
6626 }
6627
6628 #[tokio::test]
6631 async fn a_codex_refresh_routes_through_the_proxy_of_the_file_it_adopted() {
6632 let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
6633 let mut file = codex_file(("keep", days), ("A", "oaiapp_x", hour));
6634 file["proxy_url"] = "http://127.0.0.1:9/".into();
6635 let f = CodexDisk::new(&[("codex.json", &file)]).await;
6636 let cfg = f.rt.config();
6637 let route = || cpa_exec::proxy::Proxy::effective_url(&f.rt.store.get("codex.json").unwrap(), &cfg);
6638 assert_eq!(route(), "http://127.0.0.1:9/");
6639 for (version, proxy) in [("B", Some("http://127.0.0.1:8/")), ("C", None)] {
6640 let mut rotated = codex_file(("keep", days), (version, "oaiapp_x", hour));
6641 if let Some(proxy) = proxy {
6642 rotated["proxy_url"] = proxy.into();
6643 }
6644 std::fs::write(f.dir.join("codex.json"), rotated.to_string()).unwrap();
6645 let current = f.rt.store.get("codex.json").unwrap();
6646 assert!(f.rt.refresh_after_unauthorized(¤t, &cfg).await.is_some());
6647 assert_eq!(route(), proxy.unwrap_or(""), "version {version}");
6648 }
6649 assert_eq!(f.grants(), []);
6650 std::fs::remove_dir_all(&f.dir).unwrap();
6651 }
6652
6653 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6657 async fn a_codex_refresh_fails_when_its_file_is_emptied_while_the_grant_runs() {
6658 let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6659 let f = CodexDisk::gated(
6660 &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6661 0,
6662 )
6663 .await;
6664 let path = f.dir.join("codex.json");
6665 let refresh = tokio::spawn({
6666 let rt = f.rt.clone();
6667 async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6668 });
6669 f.granted().await;
6670 std::fs::write(&path, "{}").unwrap();
6671 f.gate.add_permits(1);
6672 let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6673 .await
6674 .unwrap()
6675 .unwrap();
6676 assert!(answered.is_err(), "{answered:?}");
6677 assert_eq!(std::fs::read_to_string(&path).unwrap(), "{}");
6678 assert_eq!(
6679 f.rt.store.get("codex.json").unwrap().metadata["siwc"]["refresh_token"],
6680 "siwc-ref-A"
6681 );
6682 std::fs::remove_dir_all(&f.dir).unwrap();
6683 }
6684
6685 #[tokio::test]
6690 async fn a_rejected_token_is_refreshed_in_the_background() {
6691 let (hit_tx, mut hit) = tokio::sync::mpsc::unbounded_channel();
6692 let app = axum::Router::new().route(
6693 "/oauth/token",
6694 axum::routing::post(move || async move {
6695 hit_tx.send(()).unwrap();
6696 r#"{"access_token":"rotated","refresh_token":"rotated-rt","expires_in":864000}"#
6697 }),
6698 );
6699 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6700 let token_url = format!("http://{}/oauth/token", listener.local_addr().unwrap());
6701 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6702 let dir = std::env::temp_dir().join(format!("cpa-refresh-401-{}", uuid::Uuid::new_v4()));
6703 std::fs::create_dir_all(&dir).unwrap();
6704 let expires = (chrono::Utc::now() + chrono::Duration::days(9)).to_rfc3339();
6705 std::fs::write(
6706 dir.join("codex.json"),
6707 serde_json::json!({"type": "codex", "access_token": "outer", "refresh_token": "outer-rt", "expired": expires})
6708 .to_string(),
6709 )
6710 .unwrap();
6711 let mut cfg = Config::parse("").unwrap();
6712 cfg.auth_dir = dir.clone();
6713 let client = wreq::Client::new();
6714 let oauth =
6715 cpa_exec::codex_oauth::CodexOAuth::with_endpoints(client.clone(), &token_url, &token_url, &token_url);
6716 let rt = Arc::new(crate::testing::runtime(
6717 cfg.clone(),
6718 cpa_core::config::credentials::from_auth_dir(&cfg)
6719 .into_iter()
6720 .map(crate::testing::local)
6721 .collect(),
6722 Executors {
6723 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
6724 codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
6725 devices: Default::default(),
6726 openai: Default::default(),
6727 google: Default::default(),
6728 },
6729 ));
6730 rt.start_auto_refresh();
6731 async fn quiet(hit: &mut tokio::sync::mpsc::UnboundedReceiver<()>) {
6732 let pending = tokio::time::timeout(Duration::from_millis(1500), hit.recv());
6733 assert!(pending.await.is_err(), "no refresh");
6734 }
6735 quiet(&mut hit).await; let mut quota = ExecError::local(429, FailureScope::Model, "rate limited");
6737 quota.retry_after = Some(Duration::from_secs(3600));
6738 let select = |model: &str| rt.store.select(Selection::new("codex", model)).unwrap();
6739 select("q").complete(Outcome::Failure(quota));
6740 let unauthorized = || Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
6741 select("m").complete(unauthorized());
6742 assert_eq!(rt.store.cooldowns("codex.json").len(), 2);
6743 tokio::time::timeout(Duration::from_secs(5), hit.recv())
6744 .await
6745 .expect("a refresh of the rejected token");
6746 let rotated = async {
6747 loop {
6748 let c = rt.store.get("codex.json").unwrap();
6749 if c.str("access_token") == Some("rotated") {
6750 break c;
6751 }
6752 tokio::time::sleep(Duration::from_millis(20)).await;
6753 }
6754 };
6755 let rotated = tokio::time::timeout(Duration::from_secs(5), rotated).await.unwrap();
6756 let left = rt.store.cooldowns("codex.json");
6757 assert_eq!(
6758 left.iter().map(|c| (c.model.as_str(), c.status)).collect::<Vec<_>>(),
6759 [("q", 429)]
6760 );
6761 rt.store
6763 .select(Selection::new("codex", "m"))
6764 .unwrap()
6765 .complete(unauthorized());
6766 assert_eq!(
6767 rt.store.unauthorized().pending.get("codex.json").map(|(r, _)| *r),
6768 Some(rotated.revision)
6769 );
6770 quiet(&mut hit).await;
6771 std::fs::remove_dir_all(dir).unwrap();
6772 }
6773
6774 #[test]
6779 fn which_401s_schedule_a_background_refresh() {
6780 let codex = |note: &str| {
6781 let mut c = cred("codex.json", "codex", false);
6782 c.metadata.insert("refresh_token".into(), "rt".into());
6783 c.metadata.insert("note".into(), note.into());
6784 c.metadata.insert(
6785 "request_scoped_errors".into(),
6786 serde_json::json!([{"status": 401, "match": ["revoked"], "action": "stop-and-cooldown"}]),
6787 );
6788 c
6789 };
6790 let mut claude = cred("claude.json", "claude", false);
6791 claude.metadata.insert("refresh_token".into(), "rt".into());
6792 let store = CredentialStore::new(vec![codex("a"), claude]);
6793 let fail = |provider: &str, model: &str, scope, text: &str| {
6794 store
6795 .select(Selection::new(provider, model))
6796 .unwrap()
6797 .complete(Outcome::Failure(ExecError::local(401, scope, text)));
6798 };
6799 let pending = || {
6800 let mut ids: Vec<_> = store
6801 .unauthorized()
6802 .pending
6803 .into_iter()
6804 .map(|(id, (r, _))| (id, r))
6805 .collect();
6806 ids.sort();
6807 ids
6808 };
6809 fail("claude", "m", FailureScope::Credential, "unauthorized");
6810 fail("codex", "m", FailureScope::Request, "bad request");
6811 assert_eq!(pending(), [], "Claude, and an unforced request-scoped 401");
6812 fail("codex", "m", FailureScope::Request, "token revoked");
6813 let first = store.get("codex.json").unwrap().revision;
6814 assert_eq!(pending(), [("codex.json".to_owned(), first)], "a forced cooldown");
6815 let old = store.snapshot();
6818 store.reconcile(vec![
6819 codex("b"),
6820 store.get("claude.json").map(|c| (*c).clone()).unwrap(),
6821 ]);
6822 let second = store.get("codex.json").unwrap().revision;
6823 assert!(second > first);
6824 fail("codex", "n", FailureScope::Request, "token revoked");
6825 store.forget_stale_unauthorized(&store.unauthorized(), &old);
6826 assert_eq!(
6827 pending(),
6828 [("codex.json".to_owned(), second)],
6829 "a stale snapshot keeps it"
6830 );
6831 store.forget_stale_unauthorized(&store.unauthorized(), &store.snapshot());
6832 assert_eq!(pending(), [("codex.json".to_owned(), second)], "still current");
6833 let seen = store.unauthorized();
6834 store.reconcile(vec![
6835 codex("c"),
6836 store.get("claude.json").map(|c| (*c).clone()).unwrap(),
6837 ]);
6838 store.forget_stale_unauthorized(&seen, &store.snapshot());
6839 assert_eq!(pending(), [], "replaced");
6840 }
6841
6842 #[test]
6845 fn a_stale_refresh_merges_like_go() {
6846 use serde_json::json;
6847 let map = |v: Value| v.as_object().unwrap().clone();
6848 let codex = json!({"type": "codex", "access_token": "a", "refresh_token": "r", "account_id": "acct", "email": "e@x", "plan_type": "plus"});
6849 let refresh = |plan: &str, email: &str| json!({"type": "codex", "access_token": "a2", "refresh_token": "r2", "account_id": "acct", "email": email, "plan_type": plan});
6850 let edit = |base: &Value, k: &str, v: Value| {
6851 let mut m = map(base.clone());
6852 m.insert(k.into(), v);
6853 Value::Object(m)
6854 };
6855 let kimi = json!({"type": "kimi", "note": "a", "expired": "t0", "gone": "a"});
6856 let siwc = json!({"type": "codex", "access_token": "a", "refresh_token": "r", "account_id": "acct",
6857 "siwc": {"access_token": "s", "refresh_token": "sr", "client_id": "c", "expired": "t0", "note": "n"}});
6858 let siwc_refresh = json!({"siwc": {"access_token": "s2", "refresh_token": "sr2", "client_id": "c", "expired": "t1", "note": "n"}});
6859 let siwc_edit = |k: &str, v: Value| {
6860 let mut m = map(siwc.clone());
6861 m["siwc"][k] = v;
6862 Value::Object(m)
6863 };
6864 for (name, provider, base, user, set, remove, want_set, want_remove) in [
6865 (
6866 "an unchanged value keeps the edit, a changed token is applied",
6867 "kimi",
6868 &kimi,
6869 edit(&kimi, "note", json!("b")),
6870 json!({"type": "kimi", "note": "a", "access_token": "new"}),
6871 vec![],
6872 json!({"access_token": "new"}),
6873 vec![],
6874 ),
6875 (
6876 "both changed a normal field: the edit wins",
6877 "kimi",
6878 &kimi,
6879 edit(&kimi, "note", json!("b")),
6880 json!({"note": "x"}),
6881 vec![],
6882 json!({}),
6883 vec![],
6884 ),
6885 (
6886 "both changed a token payload: the refresh wins",
6887 "kimi",
6888 &kimi,
6889 edit(&kimi, "expired", json!("t2")),
6890 json!({"expired": "t1"}),
6891 vec![],
6892 json!({"expired": "t1"}),
6893 vec![],
6894 ),
6895 (
6896 "a removal of an edited key is dropped, of an untouched key kept",
6897 "kimi",
6898 &kimi,
6899 edit(&kimi, "expired", json!("t2")),
6900 json!({}),
6901 vec!["expired", "gone"],
6902 json!({}),
6903 vec!["gone"],
6904 ),
6905 (
6906 "JSON null is an edit of a missing key",
6907 "kimi",
6908 &kimi,
6909 edit(&kimi, "fresh", Value::Null),
6910 json!({"fresh": "x"}),
6911 vec![],
6912 json!({}),
6913 vec![],
6914 ),
6915 (
6916 "Codex: an unchanged plan and email keep the edit",
6917 "codex",
6918 &codex,
6919 edit(&codex, "plan_type", json!("team")),
6920 refresh("plus", "e@x"),
6921 vec![],
6922 json!({"access_token": "a2", "refresh_token": "r2"}),
6923 vec![],
6924 ),
6925 (
6926 "Codex: a changed plan and email are applied over the edit",
6927 "codex",
6928 &codex,
6929 edit(&edit(&codex, "plan_type", json!("team")), "email", json!("u@x")),
6930 refresh("pro", "n@x"),
6931 vec![],
6932 json!({"access_token": "a2", "refresh_token": "r2", "plan_type": "pro", "email": "n@x"}),
6933 vec![],
6934 ),
6935 (
6936 "Codex SIWC: the refreshed fields land over an edit inside the partition",
6937 "codex",
6938 &siwc,
6939 siwc_edit("note", json!("edited")),
6940 siwc_refresh.clone(),
6941 vec![],
6942 json!({"siwc": {"access_token": "s2", "refresh_token": "sr2", "client_id": "c", "expired": "t1", "note": "edited"}}),
6943 vec![],
6944 ),
6945 (
6946 "Codex SIWC: another client drops the partition",
6947 "codex",
6948 &siwc,
6949 siwc_edit("client_id", json!("other")),
6950 siwc_refresh.clone(),
6951 vec![],
6952 json!({}),
6953 vec![],
6954 ),
6955 (
6956 "Codex SIWC: an edit already holding the result keeps it",
6957 "codex",
6958 &siwc,
6959 siwc_edit("access_token", json!("s2")),
6960 json!({"siwc": {"access_token": "s2", "refresh_token": "sr", "client_id": "c", "expired": "t0", "note": "n"}}),
6961 vec![],
6962 json!({}),
6963 vec![],
6964 ),
6965 (
6966 "Codex: another account drops the whole partition",
6967 "codex",
6968 &codex,
6969 edit(&codex, "account_id", json!("other")),
6970 refresh("pro", "n@x"),
6971 vec![],
6972 json!({}),
6973 vec![],
6974 ),
6975 ] {
6976 let patch = MetadataPatch {
6977 set: map(set),
6978 remove: remove.into_iter().map(str::to_owned).collect(),
6979 };
6980 let (out, _) = rebase_patch(provider, &map(base.clone()), &map(user), &patch);
6981 assert_eq!(Value::Object(out.set), want_set, "{name}");
6982 assert_eq!(out.remove, want_remove, "{name}");
6983 }
6984 }
6985
6986 #[test]
6989 fn a_matching_codex_partition_applies_after_the_other_changed() {
6990 use serde_json::json;
6991 let map = |v: Value| v.as_object().unwrap().clone();
6992 let base = json!({"type": "codex", "access_token": "a", "refresh_token": "r",
6993 "siwc": {"access_token": "s", "refresh_token": "sr", "client_id": "c"}});
6994 let patch = MetadataPatch {
6995 set: map(json!({"siwc": {"access_token": "s2", "refresh_token": "sr2", "client_id": "c"}})),
6996 remove: vec![],
6997 };
6998 let mut outer_changed = map(base.clone());
6999 outer_changed.insert("access_token".into(), "other".into());
7000 let (out, applies) = rebase_patch("codex", &map(base.clone()), &outer_changed, &patch);
7001 assert!(applies);
7002 assert_eq!(out.set["siwc"]["access_token"], "s2");
7003 let mut siwc_changed = map(base.clone());
7004 siwc_changed["siwc"]["refresh_token"] = "rotated-elsewhere".into();
7005 let (out, applies) = rebase_patch("codex", &map(base.clone()), &siwc_changed, &patch);
7006 assert!(!applies && out.set.is_empty());
7007 assert!(!rebase_patch("claude", &map(base.clone()), &outer_changed, &patch).1);
7009 }
7010
7011 #[test]
7014 fn switching_to_the_siwc_token_changes_the_credentials() {
7015 let mut a = cred("codex.json", "codex", false);
7016 a.metadata.insert("access_token".into(), "outer".into());
7017 a.metadata
7018 .insert("siwc".into(), serde_json::json!({"access_token": "nested"}));
7019 let mut b = a.clone();
7020 assert!(!credentials_changed(&a, &b));
7021 b.attributes.insert("use_siwc".into(), "true".into());
7022 assert!(credentials_changed(&a, &b));
7023 }
7024
7025 #[derive(Clone, Default)]
7027 struct Logs(Arc<Mutex<Vec<u8>>>);
7028 impl std::io::Write for Logs {
7029 fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
7030 self.0.lock().unwrap().extend_from_slice(bytes);
7031 Ok(bytes.len())
7032 }
7033 fn flush(&mut self) -> std::io::Result<()> {
7034 Ok(())
7035 }
7036 }
7037 impl Logs {
7038 fn install(&self) -> tracing::subscriber::DefaultGuard {
7039 let writer = self.clone();
7040 let subscriber = tracing_subscriber::fmt()
7041 .with_ansi(false)
7042 .without_time()
7043 .with_writer(move || writer.clone())
7044 .finish();
7045 tracing::subscriber::set_default(subscriber)
7046 }
7047 fn text(&self) -> String {
7048 String::from_utf8(self.0.lock().unwrap().clone()).unwrap()
7049 }
7050 async fn wait_for(&self, needle: &str, count: usize) {
7051 let found = tokio::time::timeout(Duration::from_secs(10), async {
7052 while self.text().matches(needle).count() < count {
7053 tokio::time::sleep(Duration::from_millis(20)).await;
7054 }
7055 });
7056 found
7057 .await
7058 .unwrap_or_else(|_| panic!("{needle:?} not logged {count} times:\n{}", self.text()));
7059 }
7060 }
7061
7062 #[tokio::test]
7068 async fn a_failed_background_refresh_is_logged_and_retried_after_its_backoff() {
7069 let answers = Arc::new(Mutex::new(std::collections::VecDeque::from([
7070 (
7071 400,
7072 r#"{"error":"invalid_grant","error_description":"Refresh token sk-ant-ort01-leaked is invalid"}"#,
7073 ),
7074 (500, r#"{"error":"server_error"}"#),
7075 (500, r#"{"error":"server_error"}"#),
7076 (500, r#"{"error":"server_error"}"#),
7077 (
7078 200,
7079 r#"{"access_token":"sk-ant-oat-new-fake","refresh_token":"fake-rt-2","expires_in":28800}"#,
7080 ),
7081 ])));
7082 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7083 let app = axum::Router::new().route(
7084 "/token",
7085 axum::routing::post({
7086 let (answers, calls) = (answers.clone(), calls.clone());
7087 move || async move {
7088 calls.fetch_add(1, Ordering::SeqCst);
7089 let (status, body) = answers.lock().unwrap().pop_front().expect("an unexpected exchange");
7090 (axum::http::StatusCode::from_u16(status).unwrap(), body)
7091 }
7092 }),
7093 );
7094 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7095 let base = format!("http://{}", listener.local_addr().unwrap());
7096 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7097 let dir = std::env::temp_dir().join(format!("cpa-refresh-fail-{}", uuid::Uuid::new_v4()));
7098 std::fs::create_dir_all(&dir).unwrap();
7099 let path = dir.join("claude-backoff.json");
7100 std::fs::write(
7101 &path,
7102 serde_json::json!({"type": "claude", "access_token": "sk-ant-oat-old-fake", "refresh_token": "fake-rt-1",
7103 "expired": (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc3339(),
7104 "account_uuid": "acct-fake", "claude_device_ids": ["b".repeat(64)]})
7105 .to_string(),
7106 )
7107 .unwrap();
7108 let mut cfg = Config::parse("").unwrap();
7109 cfg.auth_dir = dir.clone();
7110 let (token, profile) = (format!("{base}/token"), format!("{base}/profile"));
7111 let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), &token, &profile, &profile);
7112 let rt = Arc::new(crate::testing::runtime(
7113 cfg.clone(),
7114 cpa_core::config::credentials::from_auth_dir(&cfg)
7115 .into_iter()
7116 .map(crate::testing::local)
7117 .collect(),
7118 Executors {
7119 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7120 .unwrap()
7121 .with_oauth(oauth),
7122 codex: Default::default(),
7123 devices: Default::default(),
7124 openai: Default::default(),
7125 google: Default::default(),
7126 },
7127 ));
7128 let logs = Logs::default();
7129 let _guard = logs.install();
7130 let failed = "credential refresh failed for claude (claude-backoff.json): status";
7131 let retry_in = |rt: &Runtime| {
7132 let c = rt.store.get("claude-backoff.json").unwrap();
7133 let at = rt.refresh_state.lock().unwrap().retry_at(&c).unwrap();
7134 at.saturating_duration_since(Instant::now())
7135 };
7136 let expire = |rt: &Runtime| {
7138 let c = rt.store.get("claude-backoff.json").unwrap();
7139 rt.refresh_state.lock().unwrap().expire(&c);
7140 rt.config_changed.send_replace(());
7141 };
7142 rt.start_auto_refresh();
7143 logs.wait_for(failed, 1).await;
7144 assert!(
7145 logs.text().contains(&format!(
7146 "{failed} 400: Claude OAuth acquisition failed: invalid_grant; next attempt in 60s"
7147 )),
7148 "{}",
7149 logs.text()
7150 );
7151 assert!(
7152 (55..=60).contains(&retry_in(&rt).as_secs()),
7153 "Go invalidGrantBackoffBase"
7154 );
7155 tests::arm_panic("prepare", "claude-backoff.json");
7157 expire(&rt);
7158 logs.wait_for(failed, 2).await;
7159 assert!(
7160 logs.text()
7161 .contains(&format!("{failed} 500: refresh panicked: injected prepare panic"))
7162 );
7163 expire(&rt);
7164 logs.wait_for(failed, 3).await;
7165 assert!(
7166 logs.text().contains(&format!(
7167 "{failed} 500: Claude OAuth acquisition failed: server_error; next attempt in 300s"
7168 )),
7169 "{}",
7170 logs.text()
7171 );
7172 assert!(
7173 (295..=300).contains(&retry_in(&rt).as_secs()),
7174 "Go refreshFailureBackoff"
7175 );
7176 assert_eq!(
7177 calls.load(Ordering::SeqCst),
7178 4,
7179 "one invalid_grant, then three 5xx tries in place"
7180 );
7181 tests::arm_panic("scan", "claude-backoff.json");
7183 expire(&rt);
7184 logs.wait_for("credential auto-refresh loop panicked (injected scan panic)", 1)
7185 .await;
7186 let committed = tokio::time::timeout(Duration::from_secs(10), async {
7187 loop {
7188 let c = rt.store.get("claude-backoff.json").unwrap();
7189 if c.str("access_token") == Some("sk-ant-oat-new-fake") {
7190 break c;
7191 }
7192 tokio::time::sleep(Duration::from_millis(20)).await;
7193 }
7194 })
7195 .await
7196 .expect("the retry after the backoff commits");
7197 assert_eq!(committed.str("refresh_token"), Some("fake-rt-2"));
7198 let saved: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
7199 assert_eq!(saved["access_token"], "sk-ant-oat-new-fake");
7200 assert_eq!(logs.text().matches(failed).count(), 3, "{}", logs.text());
7201 let text = logs.text();
7202 assert!(
7203 !text.contains("fake-rt") && !text.contains("sk-ant-o"),
7204 "no token in the logs: {text}"
7205 );
7206 rt.stop_auto_refresh();
7207 std::fs::remove_dir_all(dir).unwrap();
7208 }
7209
7210 async fn claude_refusing(
7213 name: &str,
7214 file: Value,
7215 answer: (u16, &'static str),
7216 ) -> (Arc<Runtime>, std::path::PathBuf, Arc<std::sync::atomic::AtomicUsize>) {
7217 claude_gated(name, file, answer, None).await
7218 }
7219
7220 #[derive(Clone)]
7223 struct Gate {
7224 entered: Arc<tokio::sync::Notify>,
7225 release: Arc<tokio::sync::Semaphore>,
7226 }
7227
7228 impl Gate {
7229 fn new() -> Self {
7230 Self {
7231 entered: Arc::new(tokio::sync::Notify::new()),
7232 release: Arc::new(tokio::sync::Semaphore::new(0)),
7233 }
7234 }
7235 }
7236
7237 async fn claude_gated(
7239 name: &str,
7240 file: Value,
7241 answer: (u16, &'static str),
7242 gate: Option<Gate>,
7243 ) -> (Arc<Runtime>, std::path::PathBuf, Arc<std::sync::atomic::AtomicUsize>) {
7244 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7245 let app = axum::Router::new().route(
7246 "/token",
7247 axum::routing::post({
7248 let calls = calls.clone();
7249 move || async move {
7250 calls.fetch_add(1, Ordering::SeqCst);
7251 if let Some(gate) = gate {
7252 gate.entered.notify_one();
7253 gate.release.acquire().await.unwrap().forget();
7254 }
7255 (axum::http::StatusCode::from_u16(answer.0).unwrap(), answer.1)
7256 }
7257 }),
7258 );
7259 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7260 let base = format!("http://{}", listener.local_addr().unwrap());
7261 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7262 let dir = std::env::temp_dir().join(format!("cpa-{name}-{}", uuid::Uuid::new_v4()));
7263 std::fs::create_dir_all(&dir).unwrap();
7264 std::fs::write(dir.join(format!("{name}.json")), file.to_string()).unwrap();
7265 let mut cfg = Config::parse("").unwrap();
7266 cfg.auth_dir = dir.clone();
7267 let (token, profile) = (format!("{base}/token"), format!("{base}/profile"));
7268 let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), &token, &profile, &profile);
7269 let rt = Arc::new(crate::testing::runtime(
7270 cfg.clone(),
7271 cpa_core::config::credentials::from_auth_dir(&cfg)
7272 .into_iter()
7273 .map(crate::testing::local)
7274 .collect(),
7275 Executors {
7276 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7277 .unwrap()
7278 .with_oauth(oauth),
7279 codex: Default::default(),
7280 devices: Default::default(),
7281 openai: Default::default(),
7282 google: Default::default(),
7283 },
7284 ));
7285 (rt, dir, calls)
7286 }
7287
7288 fn claude_file(access: &str, expires_in: chrono::Duration, disabled: bool) -> Value {
7289 serde_json::json!({"type": "claude", "access_token": access, "refresh_token": "fake-rt-1",
7290 "expired": (chrono::Utc::now() + expires_in).to_rfc3339(), "disabled": disabled,
7291 "account_uuid": "acct-fake", "claude_device_ids": ["b".repeat(64)]})
7292 }
7293
7294 #[tokio::test]
7299 async fn an_invalid_grant_after_a_401_blocks_the_credential_until_its_tokens_change() {
7300 let name = "claude-revoked";
7301 let id = format!("{name}.json");
7302 let refused = (400, r#"{"error":"invalid_grant","error_description":"revoked"}"#);
7303 let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7304 let (rt, dir, calls) = claude_refusing(name, file, refused).await;
7305 let cfg = rt.config();
7306 let claude = || Selection {
7307 provider: "claude".into(),
7308 ..Selection::default()
7309 };
7310 let rejected = rt.store.get(&id).unwrap();
7311 assert!(rt.store.select(claude()).is_some(), "selectable before the 401");
7312 let both = [0, 1].map(|_| {
7315 let (rt, rejected, cfg) = (rt.clone(), rejected.clone(), cfg.clone());
7316 tokio::spawn(async move { rt.refresh_after_unauthorized(&rejected, &cfg).await })
7317 });
7318 for job in both {
7319 assert!(job.await.unwrap().is_none());
7320 }
7321 assert_eq!(calls.load(Ordering::SeqCst), 1, "one token request for both 401s");
7322 let current = rt.store.get(&id).unwrap();
7323 assert!(rt.store.tokens_rejected(¤t));
7324 assert!(rt.store.blocked(¤t, ""));
7325 assert!(rt.store.select(claude()).is_none(), "no longer selected");
7326 assert_eq!(rt.prepare_credential(&id, &cfg, None).await.unwrap_err().status, 401);
7328 assert_eq!(calls.load(Ordering::SeqCst), 1);
7329 assert!(rt.store.rejected_ids(&rt.store.snapshot()).contains(&id));
7331 let lease = rt.lease_credential(&id, claude(), &cfg, |c| c.clone()).await;
7332 assert!(lease.is_none(), "the credits fallback leased a refused credential");
7333 let reload = |rt: &Runtime| {
7335 rt.store.reconcile(
7336 cpa_core::config::credentials::from_auth_dir(&cfg)
7337 .into_iter()
7338 .map(crate::testing::local)
7339 .collect(),
7340 )
7341 };
7342 let bytes = std::fs::read(dir.join(&id)).unwrap();
7343 std::fs::remove_file(dir.join(&id)).unwrap();
7344 reload(&rt);
7345 assert!(rt.store.get(&id).is_none());
7346 std::fs::write(dir.join(&id), &bytes).unwrap();
7347 reload(&rt);
7348 let imported = rt.store.get(&id).unwrap();
7349 assert!(
7350 !rt.store.tokens_rejected(&imported),
7351 "a re-import inherited the rejection"
7352 );
7353 assert!(rt.store.select(claude()).is_some());
7354 assert!(rt.refresh_after_unauthorized(&imported, &cfg).await.is_none());
7355 assert!(rt.store.tokens_rejected(&rt.store.get(&id).unwrap()));
7356 let file = claude_file("sk-ant-oat-new-fake", chrono::Duration::hours(1), false);
7358 std::fs::write(dir.join(&id), file.to_string()).unwrap();
7359 rt.store.reconcile(
7360 cpa_core::config::credentials::from_auth_dir(&cfg)
7361 .into_iter()
7362 .map(crate::testing::local)
7363 .collect(),
7364 );
7365 let replaced = rt.store.get(&id).unwrap();
7366 assert!(!rt.store.tokens_rejected(&replaced));
7367 assert_eq!(rt.store.select(claude()).unwrap().credential.id, id);
7368 std::fs::remove_dir_all(dir).unwrap();
7369 }
7370
7371 #[tokio::test]
7375 async fn refreshes_queued_behind_a_refused_one_send_nothing() {
7376 let name = "claude-queued";
7377 let id = format!("{name}.json");
7378 let refused = (400, r#"{"error":"invalid_grant"}"#);
7379 let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7380 let gate = Gate::new();
7381 let (rt, dir, calls) = claude_gated(name, file, refused, Some(gate.clone())).await;
7382 let cfg = rt.config();
7383 let credential = rt.store.get(&id).unwrap();
7384 let first = {
7385 let (rt, c, cfg) = (rt.clone(), credential.clone(), cfg.clone());
7386 tokio::spawn(async move { rt.refresh_after_unauthorized(&c, &cfg).await })
7387 };
7388 gate.entered.notified().await;
7389 let queued = {
7390 let (rt, id, cfg) = (rt.clone(), id.clone(), cfg.clone());
7391 tokio::spawn(async move { rt.prepare_credential(&id, &cfg, None).await })
7392 };
7393 let job = tokio::spawn(run_refresh_job(
7394 rt.clone(),
7395 cfg.clone(),
7396 Arc::new(tokio::sync::Semaphore::new(1)),
7397 credential.clone(),
7398 true,
7399 ));
7400 tokio::time::sleep(Duration::from_millis(300)).await;
7402 gate.release.add_permits(3);
7403 assert!(first.await.unwrap().is_none());
7404 assert_eq!(queued.await.unwrap().unwrap_err().status, 401);
7405 job.await.unwrap();
7406 assert_eq!(
7407 calls.load(Ordering::SeqCst),
7408 1,
7409 "a queued refresh sent another exchange"
7410 );
7411 assert!(rt.store.tokens_rejected(&rt.store.get(&id).unwrap()));
7412 std::fs::remove_dir_all(dir).unwrap();
7413 }
7414
7415 #[tokio::test]
7419 async fn a_background_401_refresh_records_the_rejection() {
7420 let name = "claude-revoked-later";
7421 let id = format!("{name}.json");
7422 let refused = (400, r#"{"error":"invalid_grant"}"#);
7423 let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7424 let (rt, dir, calls) = claude_refusing(name, file, refused).await;
7425 let credential = rt.store.get(&id).unwrap();
7426 let workers = Arc::new(tokio::sync::Semaphore::new(1));
7427 run_refresh_job(rt.clone(), rt.config(), workers, credential, true).await;
7428 assert_eq!(calls.load(Ordering::SeqCst), 1);
7429 assert!(
7430 rt.store.tokens_rejected(&rt.store.get(&id).unwrap()),
7431 "the scheduled 401 refresh recorded nothing"
7432 );
7433 std::fs::remove_dir_all(dir).unwrap();
7434 }
7435
7436 #[tokio::test]
7440 async fn a_late_refusal_never_blocks_a_reimported_credential() {
7441 let name = "claude-reimported";
7442 let id = format!("{name}.json");
7443 let refused = (400, r#"{"error":"invalid_grant"}"#);
7444 let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7445 let gate = Gate::new();
7446 let (rt, dir, calls) = claude_gated(name, file, refused, Some(gate.clone())).await;
7447 let cfg = rt.config();
7448 let claude = || Selection {
7449 provider: "claude".into(),
7450 ..Selection::default()
7451 };
7452 let credential = rt.store.get(&id).unwrap();
7453 let job = {
7454 let (rt, c, cfg) = (rt.clone(), credential.clone(), cfg.clone());
7455 tokio::spawn(async move { rt.refresh_after_unauthorized(&c, &cfg).await })
7456 };
7457 gate.entered.notified().await;
7458 let reload = |rt: &Runtime| {
7459 rt.store.reconcile(
7460 cpa_core::config::credentials::from_auth_dir(&cfg)
7461 .into_iter()
7462 .map(crate::testing::local)
7463 .collect(),
7464 )
7465 };
7466 let bytes = std::fs::read(dir.join(&id)).unwrap();
7467 std::fs::remove_file(dir.join(&id)).unwrap();
7468 reload(&rt);
7469 assert!(rt.store.get(&id).is_none());
7470 std::fs::write(dir.join(&id), &bytes).unwrap();
7471 reload(&rt);
7472 gate.release.add_permits(1);
7473 assert!(job.await.unwrap().is_none());
7474 assert_eq!(calls.load(Ordering::SeqCst), 1);
7475 let imported = rt.store.get(&id).unwrap();
7476 assert!(
7477 !rt.store.tokens_rejected(&imported),
7478 "the old registration's refusal blocked the import"
7479 );
7480 assert_eq!(rt.store.select(claude()).unwrap().credential.id, id);
7481 std::fs::remove_dir_all(dir).unwrap();
7482 }
7483
7484 #[tokio::test]
7488 async fn a_stale_rejection_never_blocks_or_unblocks_rotated_tokens() {
7489 let name = "claude-rotated";
7490 let id = format!("{name}.json");
7491 let file = claude_file("sk-ant-oat-a-fake", chrono::Duration::hours(1), false);
7492 let (rt, dir, _) = claude_refusing(name, file, (500, "{}")).await;
7493 let cfg = rt.config();
7494 let invalid = ExecError::local(400, FailureScope::Request, r#"{"error":"invalid_grant"}"#);
7495 let a = rt.store.get(&id).unwrap();
7496 let registration = rt.store.get_registered(&id).unwrap().1.registration;
7497 let other = ExecError::local(400, FailureScope::Request, r#"{"error":"server_error"}"#);
7498 assert!(!rt.store.reject_refused(&a, registration, &other), "not invalid_grant");
7499 std::fs::write(
7500 dir.join(&id),
7501 claude_file("sk-ant-oat-b-fake", chrono::Duration::hours(1), false).to_string(),
7502 )
7503 .unwrap();
7504 rt.store.reconcile(
7505 cpa_core::config::credentials::from_auth_dir(&cfg)
7506 .into_iter()
7507 .map(crate::testing::local)
7508 .collect(),
7509 );
7510 let b = rt.store.get(&id).unwrap();
7511 assert!(!rt.store.reject_refused(&a, registration, &invalid));
7512 assert!(!rt.store.tokens_rejected(&b), "a stale failure blocked the new tokens");
7513 assert!(rt.store.reject_refused(&b, registration, &invalid));
7514 assert!(!rt.store.reject_refused(&a, registration, &invalid));
7515 assert!(
7516 rt.store.tokens_rejected(&b),
7517 "a stale failure lifted the new tokens' rejection"
7518 );
7519 std::fs::write(
7521 dir.join(&id),
7522 claude_file(" ", chrono::Duration::hours(1), false).to_string(),
7523 )
7524 .unwrap();
7525 rt.store.reconcile(
7526 cpa_core::config::credentials::from_auth_dir(&cfg)
7527 .into_iter()
7528 .map(crate::testing::local)
7529 .collect(),
7530 );
7531 let empty = rt.store.get(&id).unwrap();
7532 assert_eq!(empty.str("access_token"), Some(" "));
7533 assert!(
7534 !rt.store.reject_refused(&empty, registration, &invalid),
7535 "empty access token"
7536 );
7537 assert!(!rt.store.tokens_rejected(&empty));
7538 std::fs::remove_dir_all(dir).unwrap();
7539 }
7540
7541 #[tokio::test]
7544 async fn a_skipped_refresh_of_a_disabled_credential_is_logged_once() {
7545 let name = "claude-off";
7546 let expires = chrono::Duration::hours(4) + chrono::Duration::seconds(3);
7549 let file = claude_file("sk-ant-oat-old-fake", expires, true);
7550 let (rt, dir, calls) = claude_refusing(name, file, (500, "{}")).await;
7551 let logs = Logs::default();
7552 let _guard = logs.install();
7553 let skipped = "credential refresh for claude (claude-off.json) is due but the credential is disabled";
7554 rt.start_auto_refresh();
7555 logs.wait_for(skipped, 1).await;
7556 for _ in 0..2 {
7557 rt.config_changed.send_replace(());
7558 tokio::time::sleep(Duration::from_millis(1500)).await;
7559 }
7560 assert_eq!(logs.text().matches(skipped).count(), 1, "{}", logs.text());
7561 assert_eq!(calls.load(Ordering::SeqCst), 0);
7562 rt.stop_auto_refresh();
7563 std::fs::remove_dir_all(dir).unwrap();
7564 }
7565
7566 async fn claude_refresh_fixture<F, Fut>(ids: &[&str], yaml: &str, answer: F) -> (Arc<Runtime>, std::path::PathBuf)
7570 where
7571 F: Fn(String) -> Fut + Clone + Send + Sync + 'static,
7572 Fut: std::future::Future<Output = (axum::http::StatusCode, &'static str)> + Send + 'static,
7573 {
7574 let app = axum::Router::new().route("/token", axum::routing::post(move |body: String| answer(body)));
7575 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7576 let base = format!("http://{}", listener.local_addr().unwrap());
7577 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7578 let dir = std::env::temp_dir().join(format!("cpa-refresh-{}", uuid::Uuid::new_v4()));
7579 std::fs::create_dir_all(&dir).unwrap();
7580 for id in ids {
7581 std::fs::write(
7582 dir.join(id),
7583 serde_json::json!({"type": "claude", "access_token": "sk-ant-oat-old-fake",
7584 "refresh_token": format!("fake-rt-{id}"),
7585 "expired": (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc3339(),
7586 "account_uuid": "acct-fake", "claude_device_ids": ["b".repeat(64)]})
7587 .to_string(),
7588 )
7589 .unwrap();
7590 }
7591 let mut cfg = Config::parse(yaml).unwrap();
7592 cfg.auth_dir = dir.clone();
7593 let (token, profile) = (format!("{base}/token"), format!("{base}/profile"));
7594 let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), &token, &profile, &profile);
7595 let rt = Arc::new(crate::testing::runtime(
7596 cfg.clone(),
7597 cpa_core::config::credentials::from_auth_dir(&cfg)
7598 .into_iter()
7599 .map(crate::testing::local)
7600 .collect(),
7601 Executors {
7602 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7603 .unwrap()
7604 .with_oauth(oauth),
7605 codex: Default::default(),
7606 devices: Default::default(),
7607 openai: Default::default(),
7608 google: Default::default(),
7609 },
7610 ));
7611 (rt, dir)
7612 }
7613
7614 async fn refreshed(rt: &Runtime, id: &str) -> Arc<Credential> {
7616 tokio::time::timeout(Duration::from_secs(10), async {
7617 loop {
7618 let c = rt.store.get(id).unwrap();
7619 if c.str("access_token") == Some("sk-ant-oat-new-fake") {
7620 break c;
7621 }
7622 tokio::time::sleep(Duration::from_millis(20)).await;
7623 }
7624 })
7625 .await
7626 .unwrap_or_else(|_| panic!("{id} was not refreshed"))
7627 }
7628
7629 #[tokio::test]
7637 async fn a_job_past_its_bound_stays_the_only_one_and_wakes_the_loop_when_it_ends() {
7638 let (slow, quick) = ("claude-slow.json", "claude-quick.json");
7639 let release = Arc::new(tokio::sync::Notify::new());
7640 let slow_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7641 let answer = {
7642 let (release, slow_calls) = (release.clone(), slow_calls.clone());
7643 move |body: String| {
7644 let (release, slow_calls) = (release.clone(), slow_calls.clone());
7645 async move {
7646 if body.contains(&format!("fake-rt-{slow}")) && slow_calls.fetch_add(1, Ordering::SeqCst) == 0 {
7647 release.notified().await;
7648 return (axum::http::StatusCode::BAD_REQUEST, r#"{"error":"invalid_request"}"#);
7649 }
7650 let ok = r#"{"access_token":"sk-ant-oat-new-fake","refresh_token":"fake-rt-2","expires_in":28800}"#;
7651 (axum::http::StatusCode::OK, ok)
7652 }
7653 }
7654 };
7655 let workers = "oauth:\n auth-auto-refresh-workers: 1\n";
7656 let (rt, dir) = claude_refresh_fixture(&[slow, quick], workers, answer).await;
7657 let set_expiry = |id: &str, ahead: chrono::Duration| {
7659 let mut patch = MetadataPatch::default();
7660 patch
7661 .set
7662 .insert("expired".into(), (chrono::Utc::now() + ahead).to_rfc3339().into());
7663 let c = rt.store.get(id).unwrap();
7664 rt.store.apply_patch(id, c.revision, &patch).unwrap()
7665 };
7666 set_expiry(quick, chrono::Duration::days(9));
7667 let c = set_expiry(slow, chrono::Duration::days(9));
7668 rt.store
7669 .unauthorized
7670 .lock()
7671 .unwrap()
7672 .pending
7673 .insert(slow.into(), (c.revision, Instant::now()));
7674 crate::refresh::override_backoff(slow, Duration::from_secs(1));
7675 let logs = Logs::default();
7676 let _guard = logs.install();
7677 rt.start_auto_refresh();
7678 logs.wait_for(
7679 &format!("credential refresh for claude ({slow}) still running after 2s"),
7680 1,
7681 )
7682 .await;
7683 set_expiry(quick, chrono::Duration::hours(1));
7685 refreshed(&rt, quick).await;
7686 for _ in 0..2 {
7688 let scans = tests::reached("scan", slow);
7689 rt.refresh_state.lock().unwrap().expire(&c);
7690 rt.config_changed.send_replace(());
7691 tokio::time::timeout(Duration::from_secs(10), async {
7692 while tests::reached("scan", slow) == scans {
7693 tokio::time::sleep(Duration::from_millis(20)).await;
7694 }
7695 })
7696 .await
7697 .expect("a forced scan");
7698 }
7699 assert_eq!(slow_calls.load(Ordering::SeqCst), 1);
7700 assert_eq!(tests::reached("prepare", slow), 1, "one job while it runs");
7701 release.notify_one();
7702 logs.wait_for(
7703 &format!("credential refresh failed for claude ({slow}) after an upstream 401: status 400"),
7704 1,
7705 )
7706 .await;
7707 let committed = refreshed(&rt, slow).await;
7709 assert_eq!(slow_calls.load(Ordering::SeqCst), 2);
7710 tokio::time::timeout(Duration::from_secs(5), async {
7712 while rt.refresh_state.lock().unwrap().running(&committed) {
7713 tokio::time::sleep(Duration::from_millis(10)).await;
7714 }
7715 })
7716 .await
7717 .expect("the job ends its reservation");
7718 let text = logs.text();
7719 assert_eq!(text.matches("still running after").count(), 1, "{text}");
7720 assert_eq!(
7721 text.matches("its previous refresh is still running").count(),
7722 1,
7723 "{text}"
7724 );
7725 assert_eq!(text.matches("credential refresh failed").count(), 1, "{text}");
7726 rt.stop_auto_refresh();
7727 std::fs::remove_dir_all(dir).unwrap();
7728 }
7729
7730 #[tokio::test]
7735 async fn no_reservation_is_stranded_by_a_panicking_scan_or_job() {
7736 let ids = ["claude-strand-a.json", "claude-strand-b.json"];
7737 let answer = |_| async {
7738 let ok = r#"{"access_token":"sk-ant-oat-new-fake","refresh_token":"fake-rt-2","expires_in":28800}"#;
7739 (axum::http::StatusCode::OK, ok)
7740 };
7741 let (rt, dir) = claude_refresh_fixture(&ids, "", answer).await;
7742 let order: Vec<String> = rt.store.snapshot().iter().map(|c| c.id.clone()).collect();
7743 let (first, second) = (order[0].as_str(), order[1].as_str());
7744 crate::refresh::override_backoff(first, Duration::from_secs(1));
7745 tests::arm_panic("scan", second);
7746 tests::arm_panic("prepare", first);
7747 let logs = Logs::default();
7748 let _guard = logs.install();
7749 rt.start_auto_refresh();
7750 refreshed(&rt, first).await;
7751 refreshed(&rt, second).await;
7752 assert_eq!(tests::reached("prepare", first), 2, "the panicked job, then its retry");
7753 assert!(
7754 logs.text()
7755 .contains("credential auto-refresh loop panicked (injected scan panic)")
7756 );
7757 assert!(logs.text().contains(&format!(
7758 "credential refresh failed for claude ({first}): status 500: refresh panicked"
7759 )));
7760 rt.stop_auto_refresh();
7761 std::fs::remove_dir_all(dir).unwrap();
7762 }
7763
7764 #[cfg(unix)]
7768 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7769 #[cfg_attr(feature = "media-relay", ignore = "needs the Go plugin toolchain")]
7770 async fn a_panicking_plugin_refresh_backs_off() {
7771 let tmp = std::env::temp_dir();
7772 let Some(built) = cpa_plugin::testing::built_plugins(&tmp, "a_panicking_plugin_refresh") else {
7773 return;
7774 };
7775 let dir = cpa_plugin::testing::scratch(&tmp, "plugin-refresh-panic");
7776 let (auths, plugins, records) = (dir.join("auths"), dir.join("plugins"), dir.join("records"));
7777 for d in [&auths, &plugins, &records] {
7778 std::fs::create_dir_all(d).unwrap();
7779 }
7780 std::fs::copy(built.join("recorder.so"), plugins.join("acme.so")).unwrap();
7781 std::fs::write(
7782 auths.join("p.json"),
7783 r#"{"type":"acme","access_token":"p","refresh_interval":"1h"}"#,
7784 )
7785 .unwrap();
7786 let yaml = format!(
7787 "auth-dir: {}\nplugins:\n enabled: true\n dir: {}\n configs:\n acme:\n enabled: true\n label: acme\n record: {}\n caps: auth_provider\n",
7788 auths.display(),
7789 plugins.display(),
7790 records.display()
7791 );
7792 let cfg = Config::parse(&yaml).unwrap();
7793 let credentials = cpa_core::config::credentials::load(&cfg);
7794 let rt = Arc::new(crate::testing::runtime(
7795 cfg,
7796 credentials,
7797 Executors {
7798 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
7799 codex: Default::default(),
7800 devices: Default::default(),
7801 openai: Default::default(),
7802 google: Default::default(),
7803 },
7804 ));
7805 crate::plugins::start(&rt).await;
7806 rt.store.end_bootstrap();
7808 let c = rt.store.get("p.json").unwrap();
7809 assert!(
7810 crate::plugins::refresh::route(&rt, &c, &rt.config()).is_some(),
7811 "plugin-routed"
7812 );
7813 tests::arm_panic("prepare", "p.json");
7814 rt.start_auto_refresh();
7815 let backed_off = tokio::time::timeout(Duration::from_secs(10), async {
7816 loop {
7817 let c = rt.store.get("p.json").unwrap();
7818 let retry = rt.refresh_state.lock().unwrap().retry_at(&c);
7819 if retry.is_some_and(|at| at > Instant::now() + Duration::from_secs(250)) {
7820 break;
7821 }
7822 tokio::time::sleep(Duration::from_millis(20)).await;
7823 }
7824 });
7825 if backed_off.await.is_err() {
7826 let c = rt.store.get("p.json").unwrap();
7827 panic!(
7828 "no backoff: reached {}, retry {:?}, due {:?}, schedule {:?}",
7829 tests::reached("prepare", "p.json"),
7830 rt.refresh_state
7831 .lock()
7832 .unwrap()
7833 .retry_at(&c)
7834 .map(|at| at.saturating_duration_since(Instant::now())),
7835 crate::plugins::refresh::due_at(&rt, &c, &rt.config()),
7836 rt.store.refresh_schedules().lock().unwrap().get("p.json").copied()
7837 );
7838 }
7839 tokio::time::sleep(Duration::from_millis(1500)).await;
7840 assert_eq!(
7841 tests::reached("prepare", "p.json"),
7842 1,
7843 "no second attempt before the backoff"
7844 );
7845 rt.stop_auto_refresh();
7846 std::fs::remove_dir_all(dir).unwrap();
7847 }
7848
7849 #[test]
7853 fn a_plugin_refresh_failure_logs_no_plugin_message() {
7854 let logs = Logs::default();
7855 let _guard = logs.install();
7856 let c = cred("p.json", "acme", false);
7857 for code in ["invalid_grant", "rejected"] {
7858 let error = crate::plugins::refresh::call_error(cpa_plugin::rpc::CallError::Rpc {
7859 code: code.into(),
7860 message: "rejected abc123xyz456opaque".into(),
7861 status: 401,
7862 });
7863 warn_refresh_failed(&c, "", &error, None);
7864 }
7865 let text = logs.text();
7866 assert!(
7867 text.contains(
7868 "credential refresh failed for acme (p.json): status 401: plugin refresh failed: invalid_grant;"
7869 ),
7870 "{text}"
7871 );
7872 assert!(
7873 text.contains("status 401: plugin refresh failed; next attempt"),
7874 "an unknown code, which could be a token, is left out: {text}"
7875 );
7876 assert!(!text.contains("abc123"), "{text}");
7877 }
7878
7879 #[tokio::test]
7882 async fn refresh_loop_sleeps_until_the_store_changes() {
7883 let unreachable = "http://127.0.0.1:1/token";
7884 let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), unreachable, unreachable, unreachable);
7885 let rt = Arc::new(crate::testing::runtime(
7886 Config::parse("").unwrap(),
7887 Vec::new(),
7888 Executors {
7889 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7890 .unwrap()
7891 .with_oauth(oauth),
7892 codex: Default::default(),
7893 devices: Default::default(),
7894 openai: Default::default(),
7895 google: Default::default(),
7896 },
7897 ));
7898 rt.start_auto_refresh();
7899 tokio::time::sleep(Duration::from_millis(50)).await;
7900 let mut due = cred("a.json", "claude", false);
7901 due.metadata.insert("refresh_token".into(), "fake-refresh".into());
7902 due.metadata.insert(
7903 "expired".into(),
7904 (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339().into(),
7905 );
7906 rt.store.reconcile(vec![due]);
7907 let added = rt.store.get("a.json").unwrap();
7908 tokio::time::timeout(Duration::from_secs(20), async {
7910 loop {
7911 let retry = rt.refresh_state.lock().unwrap().retry_at(&added);
7912 if retry.is_some_and(|at| at > Instant::now() + Duration::from_secs(200)) {
7913 break;
7914 }
7915 tokio::time::sleep(Duration::from_millis(20)).await;
7916 }
7917 })
7918 .await
7919 .expect("the added credential was tried and backed off");
7920 rt.stop_auto_refresh();
7921 }
7922
7923 #[tokio::test]
7924 async fn every_lease_reports_exactly_once() {
7925 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
7926 let stats = |s, f, c| AttemptStats {
7927 success: s,
7928 failure: f,
7929 cancelled: c,
7930 };
7931
7932 drop(store.select(sel("claude")).unwrap());
7934 assert_eq!(store.stats(), stats(0, 0, 1));
7935 store.select(sel("claude")).unwrap().complete(Outcome::Success);
7937 assert_eq!(store.stats(), stats(1, 0, 1));
7938
7939 let err = ExecError::local(502, FailureScope::Transport, "boom");
7941 let inner = futures_util::stream::iter(vec![Ok(Bytes::from_static(b"a")), Err(err)])
7942 .chain(futures_util::stream::pending())
7943 .boxed();
7944 let mut s = Completing::new(inner, store.select(sel("claude")).unwrap());
7945 assert!(s.next().await.unwrap().is_ok());
7946 assert!(s.next().await.unwrap().is_err());
7947 assert!(s.next().await.is_none(), "fused after the first error");
7948 drop(s);
7949 assert_eq!(store.stats(), stats(1, 1, 1));
7950
7951 let mut done = Completing::new(
7952 futures_util::stream::empty().boxed(),
7953 store.select(sel("claude")).unwrap(),
7954 );
7955 assert!(done.next().await.is_none());
7956 drop(done);
7957 assert_eq!(store.stats(), stats(2, 1, 1));
7958
7959 let mut mid = Completing::new(
7961 futures_util::stream::iter(vec![Ok(Bytes::new())])
7962 .chain(futures_util::stream::pending())
7963 .boxed(),
7964 store.select(sel("claude")).unwrap(),
7965 );
7966 assert!(mid.next().await.is_some());
7967 drop(mid);
7968 assert_eq!(store.stats(), stats(2, 1, 2));
7969 }
7970
7971 #[tokio::test]
7974 async fn remote_leases_end_after_the_upstream_body_closes() {
7975 use std::sync::atomic::AtomicBool;
7976 struct Probe(Arc<AtomicBool>);
7977 impl Drop for Probe {
7978 fn drop(&mut self) {
7979 self.0.store(true, Ordering::SeqCst);
7980 }
7981 }
7982 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
7983 let dropped = Arc::new(AtomicBool::new(false));
7984 let observed = Arc::new(Mutex::new(Vec::new()));
7985 let mut lease = store.select(sel("claude")).unwrap();
7986 let (seen, probe) = (observed.clone(), dropped.clone());
7987 lease.remote = Some(crate::remote::RemoteEnd {
7988 end: Some(Box::new(move || {
7989 seen.lock().unwrap().push(probe.load(Ordering::SeqCst));
7990 None
7991 })),
7992 releases: Default::default(),
7993 cancel: None,
7994 private: Default::default(),
7995 });
7996 let guard = Probe(dropped.clone());
7997 let err = ExecError::local(502, FailureScope::Transport, "boom");
7998 let inner = futures_util::stream::iter(vec![Ok(Bytes::from_static(b"a")), Err(err)])
7999 .chain(futures_util::stream::pending())
8000 .map(move |item| {
8001 let _ = &guard;
8002 item
8003 })
8004 .boxed();
8005 let mut stream = Completing::new(inner, lease);
8006 assert!(stream.next().await.unwrap().is_ok());
8007 assert!(observed.lock().unwrap().is_empty(), "still streaming");
8008 assert!(stream.next().await.unwrap().is_err());
8009 assert_eq!(
8010 *observed.lock().unwrap(),
8011 vec![true],
8012 "body dropped first, then one end"
8013 );
8014 drop(stream);
8015 assert_eq!(observed.lock().unwrap().len(), 1);
8016 assert_eq!(
8017 store.stats(),
8018 AttemptStats::default(),
8019 "the local scheduler saw nothing"
8020 );
8021 }
8022 #[tokio::test]
8026 async fn a_drain_ends_a_remote_stream_the_client_stopped_reading() {
8027 use std::sync::atomic::AtomicBool;
8028 struct Probe(Arc<AtomicBool>);
8029 impl Drop for Probe {
8030 fn drop(&mut self) {
8031 self.0.store(true, Ordering::SeqCst);
8032 }
8033 }
8034 let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
8035 let dropped = Arc::new(AtomicBool::new(false));
8036 let observed = Arc::new(Mutex::new(Vec::new()));
8037 let mut lease = store.select(sel("claude")).unwrap();
8038 let (seen, probe) = (observed.clone(), dropped.clone());
8039 let (cancel_tx, cancel) = tokio::sync::watch::channel(false);
8040 lease.remote = Some(crate::remote::RemoteEnd {
8041 end: Some(Box::new(move || {
8042 seen.lock().unwrap().push(probe.load(Ordering::SeqCst));
8043 None
8044 })),
8045 releases: Default::default(),
8046 cancel: Some(cancel),
8047 private: Default::default(),
8048 });
8049 let guard = Probe(dropped.clone());
8050 let inner = futures_util::stream::iter(vec![Ok(Bytes::from_static(b"a"))])
8051 .chain(futures_util::stream::pending())
8052 .map(move |item| {
8053 let _ = &guard;
8054 item
8055 })
8056 .boxed();
8057 let mut stream = Completing::new(inner, lease);
8058 assert!(stream.next().await.unwrap().is_ok());
8059 cancel_tx.send(true).unwrap();
8061 for _ in 0..100 {
8062 if !observed.lock().unwrap().is_empty() {
8063 break;
8064 }
8065 tokio::time::sleep(Duration::from_millis(10)).await;
8066 }
8067 assert_eq!(
8068 *observed.lock().unwrap(),
8069 vec![true],
8070 "body dropped first, then one end, without a poll"
8071 );
8072 let error = stream.next().await.unwrap().unwrap_err();
8073 assert_eq!(String::from_utf8_lossy(&error.body), "context canceled");
8074 assert!(stream.next().await.is_none());
8075 drop(stream);
8076 assert_eq!(observed.lock().unwrap().len(), 1);
8077 }
8078}