Skip to main content

cpa_server/
runtime.rs

1//! Axum-independent runtime: config snapshot, credential store, selection, credential
2//! preparation and attempt outcomes.
3//!
4//! Every attempt is a [`Lease`]: an owned guard that reports exactly one [`Outcome`],
5//! either explicitly or as `Cancelled` when dropped (client gone mid-execute, stream
6//! dropped). Streams wrap their lease in [`Completing`].
7
8use std::collections::HashMap;
9use std::io::Write;
10use std::panic::AssertUnwindSafe;
11use std::path::Path;
12use std::pin::Pin;
13use std::sync::atomic::{AtomicU64, Ordering};
14use std::sync::{Arc, Mutex, PoisonError, RwLock};
15use std::task::{Context, Poll, ready};
16use std::time::{Duration, Instant};
17
18use bytes::Bytes;
19use cpa_common::gostr::GoStr;
20use cpa_core::config::Config;
21use cpa_core::credential::{Credential, MetadataPatch, Source};
22use cpa_core::exec::{ExecError, ExecStream, FailureScope};
23use cpa_exec::{Executors, Readiness};
24use futures_util::{FutureExt, Stream, StreamExt};
25use serde_json::{Map, Value};
26
27use crate::refresh::RefreshState;
28use crate::registry::Registry;
29use crate::scheduler::{Policy, Scheduler};
30
31/// The config snapshot and credential epoch a registry was built from.
32type RegistryCache = (Arc<Config>, (u64, u64, u64), Arc<Registry>);
33
34pub struct Runtime {
35    config: RwLock<Arc<Config>>,
36    store: Arc<CredentialStore>,
37    pub executors: Executors,
38    refresh_task: Mutex<Option<tokio::task::AbortHandle>>,
39    refresh_state: Mutex<RefreshState>,
40    /// The registry derived from the current config and credential generation.
41    registry: Mutex<Option<RegistryCache>>,
42    oauth_sink: RwLock<Option<OAuthCallbackSink>>,
43    /// Go `modelPoolOffsets`: rotation cursors for OpenAI-compatible alias pools.
44    pool_offsets: Mutex<HashMap<String, usize>>,
45    /// Usage records for `GET /observability/usage/queue` (management configures it).
46    usage: Arc<crate::usage::UsageQueue>,
47    /// `--local-model`: embedded model catalogs only, no remote catalog refresh.
48    local_model: std::sync::atomic::AtomicBool,
49    /// The native plugin host, synced with each published config (see [`crate::plugins`]).
50    plugins: crate::plugins::PluginRuntime,
51    /// The remote dispatcher that replaces local selection (Go Home mode).
52    remote: RwLock<Option<Arc<dyn crate::remote::RemoteDispatch>>>,
53    /// Set only by [`crate::testing::runtime`]: executor calls get credentials that
54    /// cannot leave the machine.
55    pub(crate) deny_external: std::sync::atomic::AtomicBool,
56    /// Marked on every config publish, so background loops sleep until it changes.
57    config_changed: tokio::sync::watch::Sender<()>,
58    /// Marked whenever a background refresh job ends, so the refresh loop looks again
59    /// (a job may end after the loop stopped waiting for it).
60    refresh_done: tokio::sync::watch::Sender<()>,
61    /// Codex refresh transactions in flight ([`Runtime::shutdown_codex_refreshes`]).
62    codex_refreshes: CodexRefreshes,
63    /// The model catalog updaters' policy, once started (`model_updater::start`).
64    catalogs: Mutex<Option<crate::model_updater::Session>>,
65    /// Serializes config commits through their catalog reconfigure.
66    config_commit: Mutex<()>,
67}
68
69/// The Codex refresh transactions under a credential file's lock: shutdown stops the
70/// ones still waiting for the lock and waits for the ones that hold it, whose grant may
71/// have spent a refresh token.
72#[derive(Clone, Default)]
73struct CodexRefreshes {
74    shutdown: tokio_util::sync::CancellationToken,
75    tasks: tokio_util::task::TaskTracker,
76}
77
78/// An OAuth provider redirect received on the main listener.
79#[derive(Debug, Clone)]
80pub struct OAuthCallback {
81    /// `anthropic`, `codex`, `antigravity` or `devin`.
82    pub provider: &'static str,
83    pub state: String,
84    pub code: String,
85    pub error: String,
86}
87
88/// What became of a provider callback.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub enum OAuthDelivery {
91    Delivered,
92    /// No login with that state is pending, or the callback could not be recorded.
93    NotPending,
94    /// The state belongs to a Sign in with ChatGPT binding, which takes its callback
95    /// only through the v8 Management API (Go `ErrSIWCCallbackRequiresV8`).
96    RequiresV8,
97}
98
99/// Hands a callback to a pending management login (Go
100/// `WriteOAuthCallbackFileForPendingSession`).
101pub type OAuthCallbackSink = Arc<dyn Fn(&OAuthCallback) -> OAuthDelivery + Send + Sync>;
102
103impl Runtime {
104    /// The scheduler policy is derived from the config (`routing` plus the OAuth
105    /// provider rules), as on every publish.
106    pub fn new(config: Config, credentials: Vec<Credential>, executors: Executors) -> Self {
107        let mut policy = crate::management::policy(&config);
108        policy.compat_disable_cooling = crate::scheduler::compat_cooling(&config);
109        let cooldown_dir = cooldown_dir(&config, &policy);
110        let (enabled, strict) = signature_cache_config(&config);
111        cpa_translate::set_antigravity_signature_cache_config(enabled, strict);
112        let rt = Self {
113            config: RwLock::new(Arc::new(config)),
114            store: CredentialStore::new(credentials),
115            executors,
116            refresh_task: Mutex::default(),
117            refresh_state: Mutex::default(),
118            registry: Mutex::default(),
119            oauth_sink: RwLock::default(),
120            pool_offsets: Mutex::default(),
121            usage: Arc::default(),
122            local_model: Default::default(),
123            plugins: Default::default(),
124            remote: RwLock::default(),
125            deny_external: Default::default(),
126            config_changed: tokio::sync::watch::Sender::new(()),
127            refresh_done: tokio::sync::watch::Sender::new(()),
128            codex_refreshes: CodexRefreshes::default(),
129            catalogs: Mutex::default(),
130            config_commit: Mutex::default(),
131        };
132        rt.publish_policy(policy);
133        rt.store.configure_cooldown_store(cooldown_dir);
134        // Failed attempts publish Go's error events on the RESP `errors` channel.
135        let _ = rt.store.error_events.set(rt.usage.clone());
136        let _ = rt
137            .store
138            .session_closer
139            .set(Box::new(rt.executors.credential_session_closer()));
140        rt
141    }
142
143    /// The usage queue: the request path calls `enqueue` with one serialized Go
144    /// `queuedUsageDetail` per upstream request when `accepts()` (additive API).
145    pub fn usage_queue(&self) -> &crate::usage::UsageQueue {
146        &self.usage
147    }
148
149    /// Records `--local-model` (Go `modelCatalogUpdaterPlan`'s `localModel`): remote
150    /// model catalog updaters must not start when set (additive API).
151    pub fn set_local_model(&self, on: bool) {
152        self.local_model.store(on, std::sync::atomic::Ordering::Relaxed);
153    }
154
155    /// Persists cooldowns to `backend` instead of `.cds` files (PGSTORE's cooldown
156    /// table; additive API). Call before serving.
157    pub fn set_cooldown_backend(&self, backend: Arc<dyn crate::cooldown_store::Backend>) {
158        self.store.set_cooldown_backend(backend);
159    }
160
161    /// Whether `--local-model` was given.
162    pub fn local_model(&self) -> bool {
163        self.local_model.load(std::sync::atomic::Ordering::Relaxed)
164    }
165
166    /// Serves with `host`, a plugin host that already runs (Go hands the host that
167    /// registered plugin command-line flags to the service; additive API). Call before
168    /// [`crate::plugins::start`].
169    pub fn with_plugin_host(mut self, host: cpa_plugin::Host) -> Self {
170        self.plugins = crate::plugins::PluginRuntime::with_host(host);
171        self
172    }
173
174    /// The plugin host (additive API). [`crate::plugins::start`] syncs it with the
175    /// config.
176    pub fn plugins(&self) -> &cpa_plugin::Host {
177        self.plugins.host()
178    }
179
180    pub(crate) fn plugin_runtime(&self) -> &crate::plugins::PluginRuntime {
181        &self.plugins
182    }
183
184    /// The config snapshot to use for one whole request.
185    pub fn config(&self) -> Arc<Config> {
186        self.config.read().unwrap_or_else(PoisonError::into_inner).clone()
187    }
188
189    /// Starts the model catalog updaters for this runtime's lifetime under the commit
190    /// mutex, so no commit is configured before or skipped by the startup configure.
191    pub(crate) fn start_catalogs(&self, session: crate::model_updater::Session) {
192        let commit = self.config_commit.lock().unwrap_or_else(PoisonError::into_inner);
193        session.configure(&self.config(), true);
194        let previous = self
195            .catalogs
196            .lock()
197            .unwrap_or_else(PoisonError::into_inner)
198            .replace(session);
199        drop(commit);
200        drop(previous);
201    }
202
203    /// Whether [`Runtime::publish_config_and_policy`] would commit `config`: Go
204    /// `commitConfigUpdate` rejects the whole update over invalid catalog sources.
205    /// Callers that apply other state before publishing check this first.
206    pub(crate) fn accepts_config(config: &Config) -> bool {
207        match cpa_core::config::validate_catalog_sources(config.document.get("models")) {
208            Ok(()) => true,
209            Err(error) => {
210                tracing::warn!("rejected invalid model catalog sources: {error}");
211                false
212            }
213        }
214    }
215
216    /// Resolves after the next config publish.
217    pub fn subscribe_config(&self) -> tokio::sync::watch::Receiver<()> {
218        self.config_changed.subscribe()
219    }
220
221    /// Replaces the config and the scheduler policy derived from it. Requests already
222    /// running keep their snapshot. False when the config was rejected and nothing
223    /// changed ([`Runtime::accepts_config`]).
224    pub fn publish_config(&self, config: Config) -> bool {
225        let policy = crate::management::policy(&config);
226        self.publish_config_and_policy(config, policy)
227    }
228
229    /// Integration should use this when publishing parsed routing settings too. False
230    /// when the config was rejected and nothing changed.
231    pub fn publish_config_and_policy(&self, config: Config, mut policy: Policy) -> bool {
232        if !Self::accepts_config(&config) {
233            return false;
234        }
235        policy.compat_disable_cooling = crate::scheduler::compat_cooling(&config);
236        let (enabled, strict) = signature_cache_config(&config);
237        cpa_translate::set_antigravity_signature_cache_config(enabled, strict);
238        let dir = cooldown_dir(&config, &policy);
239        let ws_auth = crate::relay::ws_auth(&config);
240        let config = Arc::new(config);
241        // Commits are serialized, so the catalogs are configured in commit order; the
242        // config lock itself is released first, as Go releases cfgMu before
243        // `UpdateModelCatalogSources`, so readers never wait on a catalog publish.
244        let commit = self.config_commit.lock().unwrap_or_else(PoisonError::into_inner);
245        let previous = {
246            // The plugin worker is told under the same lock, so concurrent publishes
247            // reach it in the order they replaced the config.
248            let mut current = self.config.write().unwrap_or_else(PoisonError::into_inner);
249            let previous = std::mem::replace(&mut *current, config.clone());
250            self.plugins.config_published(config.clone());
251            previous
252        };
253        // Go `UpdateModelCatalogSources`; the reads stay async.
254        if let Some(catalogs) = self.catalogs.lock().unwrap_or_else(PoisonError::into_inner).as_ref() {
255            catalogs.configure(&config, false);
256        }
257        drop(commit);
258        // Go's websocket auth change handler: turning ws-auth on ends the relay sessions
259        // that connected without a key.
260        if ws_auth && !crate::relay::ws_auth(&previous) {
261            self.executors.google.aistudio.relay.stop();
262        }
263        self.publish_policy(policy);
264        self.store.configure_cooldown_store(dir);
265        self.config_changed.send_replace(());
266        true
267    }
268
269    /// A coherent config/policy pair for the entire route attempt loop.
270    pub fn request_snapshot(&self) -> (Arc<Config>, Arc<Policy>) {
271        let config = self.config.read().unwrap_or_else(PoisonError::into_inner);
272        (config.clone(), self.policy())
273    }
274
275    pub fn store(&self) -> &Arc<CredentialStore> {
276        &self.store
277    }
278
279    /// Go `nextModelPoolOffset`: the start index for the next pass over a pool.
280    pub fn next_pool_offset(&self, key: &str, size: usize) -> usize {
281        if size <= 1 || key.trim().is_empty() {
282            return 0;
283        }
284        let key = key.trim();
285        let mut offsets = self.pool_offsets.lock().unwrap_or_else(PoisonError::into_inner);
286        // ponytail: Go never prunes, so keys of removed credentials would accumulate
287        // forever. Past 4096 keys the map is cleared, which restarts rotation for every
288        // pool from offset 0. Upgrade path: drop a pool's key when its credential set
289        // changes in reconcile.
290        if offsets.len() >= 4096 && !offsets.contains_key(key) {
291            offsets.clear();
292        }
293        let slot = offsets.entry(key.to_owned()).or_default();
294        let offset = if *slot >= 2_147_483_640 { 0 } else { *slot };
295        *slot = offset + 1;
296        offset % size
297    }
298
299    /// One credential's projected state for one registered model (Go
300    /// `clientModelProjectionForAuth`), for listings and `auto`.
301    pub fn suspension(&self, credential_id: &str, model: &str) -> crate::registry::Suspension {
302        use crate::registry::Suspension;
303        let Some(credential) = self.store.get(credential_id) else {
304            return Suspension::None;
305        };
306        let aliases = crate::registry::global_aliases(&self.config());
307        let key = crate::registry::selection_model(&aliases, &credential, model);
308        self.store.suspension(&credential, &key)
309    }
310
311    /// Wires OAuth callback delivery (management owns pending login sessions).
312    pub fn set_oauth_callback_sink(&self, sink: Option<OAuthCallbackSink>) {
313        *self.oauth_sink.write().unwrap_or_else(PoisonError::into_inner) = sink;
314    }
315
316    pub fn deliver_oauth_callback(&self, callback: &OAuthCallback) -> OAuthDelivery {
317        let sink = self.oauth_sink.read().unwrap_or_else(PoisonError::into_inner).clone();
318        sink.map_or(OAuthDelivery::NotPending, |sink| sink(callback))
319    }
320
321    /// The credential an executor call receives: the stored one, or in a test runtime
322    /// ([`crate::testing`]) a copy that cannot leave the machine.
323    pub(crate) fn for_executor(&self, credential: &Arc<Credential>) -> Arc<Credential> {
324        if self.deny_external.load(std::sync::atomic::Ordering::Relaxed) {
325            crate::testing::guarded(credential, &self.config())
326        } else {
327            credential.clone()
328        }
329    }
330
331    /// The model registry for the current config and credential set. Rebuilt only when
332    /// either changed, so it is always derived, never separately maintained.
333    pub fn registry(&self) -> Arc<Registry> {
334        let config = self.config();
335        // A refreshed static catalog re-registers every credential (Go's model refresh
336        // callback), so the catalog generation is part of the key.
337        // Capability probes (Antigravity web search) refine registered models later.
338        let generation = (
339            self.store.epoch(),
340            cpa_core::registry::catalog_generation(),
341            cpa_core::registry::dynamic::capabilities_generation(),
342        );
343        let mut cache = self.registry.lock().unwrap_or_else(PoisonError::into_inner);
344        if let Some((cfg, g, registry)) = cache.as_ref()
345            && Arc::ptr_eq(cfg, &config)
346            && *g == generation
347        {
348            return registry.clone();
349        }
350        let snapshot = self.store.snapshot();
351        let registry = Arc::new(Registry::build(&config, &snapshot));
352        let credentials: Vec<_> = snapshot.iter().map(|c| self.for_executor(c)).collect();
353        self.executors.models_registered(&credentials, &config);
354        *cache = Some((config, generation, registry.clone()));
355        registry
356    }
357
358    pub fn policy(&self) -> Arc<Policy> {
359        self.store.policy.read().unwrap_or_else(PoisonError::into_inner).clone()
360    }
361
362    pub fn publish_policy(&self, policy: Policy) {
363        {
364            let mut current = self.store.policy.write().unwrap_or_else(PoisonError::into_inner);
365            self.store
366                .scheduler
367                .lock()
368                .unwrap_or_else(PoisonError::into_inner)
369                .configure(&current, &policy);
370            *current = Arc::new(policy);
371        }
372        self.store.clear_disabled_cooldowns();
373    }
374
375    /// Selects a credential that registered the route model. Waits for preparation only
376    /// when the executor reports [`Readiness::PrepareNow`] (single-flighted per
377    /// credential); token refreshes stay with the background loop. The returned lease
378    /// holds the committed snapshot.
379    pub async fn acquire(
380        &self,
381        selection: Selection,
382        cfg: &Config,
383        policy: Arc<Policy>,
384        registry: &Registry,
385    ) -> Result<Lease, AcquireError> {
386        let aliases = crate::registry::global_aliases(cfg);
387        let scope = selection.clone();
388        let admit = admission(registry, &aliases, &scope, &self.executors, self.plugin_runtime());
389        // `soonest-reset` ranks by the quota windows the executors last observed.
390        let now = std::time::SystemTime::now();
391        let ranks = |c: &Credential| {
392            let provider = c.provider.trim().to_ascii_lowercase();
393            let snapshot = match provider.as_str() {
394                "claude" => self.executors.claude.quota().snapshot(&c.id),
395                "codex" => self.executors.codex.quota().snapshot(&c.id),
396                _ => None,
397            };
398            snapshot.map_or_else(crate::scheduler::Windows::default, |s| {
399                crate::scheduler::Windows::observed(&provider, &s.signals, s.observed_at, now)
400            })
401        };
402        let request = crate::plugins::scheduler_request(self, &selection, &policy, &admit)?;
403        let pick = crate::plugins::scheduler_pick(self, request).await?;
404        let mut lease = self.store.select_picked(selection, policy, &admit, &ranks, pick)?;
405        if self.executors.readiness(&lease.credential, cfg) != Readiness::PrepareNow {
406            return Ok(lease);
407        }
408        let id = lease.credential.id.clone();
409        match self.prepare_credential(&id, cfg, None).await {
410            Ok(current) => {
411                lease.credential = current;
412                // Go stamps the dispatch after preparing the credential.
413                lease.redispatch();
414            }
415            Err(error) => {
416                lease.complete(Outcome::Failure(error.clone()));
417                return Err(AcquireError::Prepare { id, error });
418            }
419        }
420        Ok(lease)
421    }
422
423    /// A lease on credential `id` outside selection, prepared like [`Self::acquire`]
424    /// (Go's credits fallback, `findAllAntigravityCreditsCandidateAuths` then
425    /// `prepareRequestAuth`). `None` when the credential is gone or preparation fails;
426    /// Go records no result for either.
427    pub(crate) async fn lease_credential(
428        &self,
429        id: &str,
430        selection: Selection,
431        cfg: &Config,
432        prepare: impl FnOnce(&Credential) -> Credential,
433    ) -> Option<Lease> {
434        let mut credential = self.store.get(id).filter(|c| !self.store.tokens_rejected(c))?;
435        if self.executors.readiness(&credential, cfg) == Readiness::PrepareNow {
436            credential = self.prepare_credential(id, cfg, None).await.ok()?;
437        }
438        // Outside selection: the lease answers no bounded window or probe reservation.
439        let (picked, seq) = {
440            let mut scheduler = self.store.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
441            (scheduler.outside_selection_generation(), scheduler.dispatch())
442        };
443        Some(Lease {
444            store: self.store.clone(),
445            credential: Arc::new(prepare(&credential)),
446            execution_model: selection.model.clone(),
447            selection,
448            attempt: self.store.attempts.fetch_add(1, Ordering::Relaxed),
449            policy: self.policy(),
450            reported: false,
451            picked,
452            seq,
453            remote: None,
454            lcp: None,
455        })
456    }
457
458    /// The installed remote dispatcher (Go Home mode), if any.
459    pub fn remote_dispatch(&self) -> Option<Arc<dyn crate::remote::RemoteDispatch>> {
460        self.remote.read().unwrap_or_else(PoisonError::into_inner).clone()
461    }
462
463    /// Routes every request through `dispatch` (Go Home mode) or back to the local
464    /// scheduler with `None` (additive API).
465    pub fn set_remote_dispatch(&self, dispatch: Option<Arc<dyn crate::remote::RemoteDispatch>>) {
466        *self.remote.write().unwrap_or_else(PoisonError::into_inner) = dispatch;
467    }
468
469    /// One remote pick (Go `pickHomeDispatchSelection`): the lease ends through the
470    /// dispatcher, and its release joins `releases`. Also returns Home's request-retry
471    /// limit and the client key Home authenticated (empty when it sent none).
472    pub(crate) async fn acquire_remote(
473        &self,
474        dispatch: &dyn crate::remote::RemoteDispatch,
475        selection: Selection,
476        request: crate::remote::RemoteRequest,
477        releases: &crate::remote::PendingReleases,
478    ) -> Result<(Lease, Option<i64>, String), crate::remote::RemoteError> {
479        let grant = dispatch.dispatch(request).await?;
480        let lease = Lease {
481            store: self.store.clone(),
482            credential: Arc::new(grant.credential),
483            execution_model: selection.model.clone(),
484            selection,
485            attempt: self.store.attempts.fetch_add(1, Ordering::Relaxed),
486            policy: self.policy(),
487            reported: false,
488            picked: 0,
489            seq: 0,
490            remote: Some(crate::remote::RemoteEnd {
491                end: Some(grant.end),
492                releases: releases.clone(),
493                cancel: grant.cancel,
494                private: grant.private,
495            }),
496            // ponytail: Go's Home dispatch keys its own session aliases on LCP matches
497            // (home_session_alias.go); remote picks here carry no LCP binding.
498            lcp: None,
499        };
500        Ok((lease, grant.request_retry, grant.user_api_key))
501    }
502
503    /// Go `prepareHomeRequestAuth`: a dispatched credential that requests must wait for
504    /// (a Claude identity, a Meta mint) is prepared on this attempt's copy only,
505    /// serialized per credential ID. Home owns the stored credential, so nothing is
506    /// committed locally.
507    pub(crate) async fn prepare_remote(&self, lease: &mut Lease, cfg: &Config) -> Result<(), ExecError> {
508        if self.executors.readiness(&lease.credential, cfg) != Readiness::PrepareNow {
509            return Ok(());
510        }
511        let lock = self.store.prepare_lock(&lease.credential.id);
512        let _guard = lock.lock().await;
513        let patch = self
514            .executors
515            .prepare(&self.for_executor(&lease.credential), cfg)
516            .await?;
517        let mut prepared = (*lease.credential).clone();
518        patch.apply(&mut prepared.metadata);
519        lease.credential = Arc::new(prepared);
520        Ok(())
521    }
522
523    /// At shutdown: Codex refreshes still waiting for a credential file's lock (another
524    /// process holds it) give up, and those that hold it finish, so new tokens a grant
525    /// returned are written before the process exits. Later refreshes fail.
526    pub async fn shutdown_codex_refreshes(&self) {
527        self.codex_refreshes.shutdown.cancel();
528        self.codex_refreshes.tasks.close();
529        self.codex_refreshes.tasks.wait().await;
530    }
531
532    /// The credential lock of the Codex file at `path`, for a write that is not a refresh
533    /// (a management save), waited for as a refresh waits: shutdown stops the wait.
534    pub(crate) async fn lock_codex_file(&self, path: &Path) -> Result<std::fs::File, String> {
535        wait_codex_lock(path, &self.codex_refreshes.shutdown).await
536    }
537
538    /// Prepares and commits one credential. `failed` is the revision whose token an
539    /// upstream rejected: preparation then runs even when not due, unless another task
540    /// already replaced that revision (Go `refreshAuthForRequest`).
541    async fn prepare_credential(
542        &self,
543        id: &str,
544        cfg: &Config,
545        failed: Option<u64>,
546    ) -> Result<Arc<Credential>, ExecError> {
547        self.prepare_credential_with(id, cfg, failed, false, None).await
548    }
549
550    /// [`Self::prepare_credential`]; `management` marks a management refresh, which
551    /// refreshes every Codex token partition (Go `IsForceRefreshContext`). `refused` is
552    /// the credential as an upstream refused it with a 401: a refresh that then fails with
553    /// invalid_grant records it ([`CredentialStore::reject_refused`]) before the
554    /// preparation lock is released.
555    async fn prepare_credential_with(
556        &self,
557        id: &str,
558        cfg: &Config,
559        failed: Option<u64>,
560        management: bool,
561        refused: Option<&Credential>,
562    ) -> Result<Arc<Credential>, ExecError> {
563        test_hook("prepare", id);
564        // Owned: the commit below moves it into its blocking job, which a cancelled caller
565        // cannot stop, so the lock covers the read through the commit either way.
566        let guard = self.store.prepare_lock(id).lock_owned().await;
567        // Another request may have prepared it while we waited. The registration it is
568        // current under is what a plugin refresh records its outcome for.
569        let (current, registered) = self
570            .store
571            .get_registered(id)
572            .filter(|(c, _)| !c.disabled)
573            .ok_or_else(|| {
574                ExecError::local(
575                    409,
576                    FailureScope::Request,
577                    "credential removed or disabled during preparation",
578                )
579            })?;
580        // Go `refreshAuthForRequestAtEpoch` under its refresh lock: tokens refused for good
581        // are not refreshed again, except by a management refresh.
582        if !management && self.store.tokens_rejected(&current) {
583            return Err(ExecError::local(
584                401,
585                FailureScope::Request,
586                "credential tokens were refused; sign in again",
587            ));
588        }
589        match failed {
590            Some(revision) if current.revision != revision => return Ok(current),
591            Some(_) => {}
592            None if self.readiness(&current, cfg) == Readiness::Ready => return Ok(current),
593            None => {}
594        }
595        let refused = refused.map(|c| Arc::new(c.clone()));
596        let view = self.for_executor(&current);
597        let plugin = crate::plugins::refresh::route(self, &current, cfg);
598        // Go `CodexExecutor.Refresh` on a Codex credential file: one transaction, under
599        // the file's credential lock, owned apart from the caller. A plugin-virtual one
600        // refreshes in memory and leaves the plugin's file alone (DIFFERENCES-FROM-GO.md).
601        if let Source::File(path) = &current.source
602            && plugin.is_none()
603            && current.provider == "codex"
604            && !marked(&current, "plugin_virtual")
605            && path.exists()
606        {
607            let mode = match failed {
608                Some(_) if management => cpa_exec::codex_refresh::RefreshMode::Management,
609                Some(_) => cpa_exec::codex_refresh::RefreshMode::Rejected,
610                None => cpa_exec::codex_refresh::RefreshMode::Due,
611            };
612            let refresher = self.executors.codex.refresher(&view, cfg);
613            let file = path.clone();
614            return refresh_codex_file(
615                &self.codex_refreshes,
616                self.store.clone(),
617                refresher,
618                guard,
619                current,
620                registered,
621                view,
622                file,
623                mode,
624                refused,
625            )
626            .await;
627        }
628        // A plugin answer also publishes the committed document's proxy URL (the merged
629        // one, or a concurrent edit's when that wins the merge) as the credential's.
630        let prepared = match (&plugin, failed) {
631            (Some(route), _) => crate::plugins::refresh::refresh(self, route, &view, &current, registered.registration)
632                .await
633                .map(|r| (r.patch, r.answered)),
634            (None, Some(_)) if management => self.executors.prepare_management(&view, cfg).await.map(|p| (p, false)),
635            (None, Some(_)) => self.executors.prepare_forced(&view, cfg).await.map(|p| (p, false)),
636            (None, None) => self.executors.prepare(&view, cfg).await.map(|p| (p, false)),
637        };
638        // Still under the preparation lock, so a refresh queued behind this one sees it.
639        let (patch, publish_proxy) = prepared.inspect_err(|e| {
640            if let Some(refused) = &refused {
641                self.store.reject_refused(refused, registered.registration, e);
642            }
643        })?;
644        let publish_proxy = move |c: &mut Credential| {
645            if publish_proxy {
646                publish_metadata_proxy(c);
647            }
648        };
649        let store = self.store.clone();
650        let fence = RefreshFence {
651            registered,
652            plugin: plugin.is_some(),
653        };
654        let committed = tokio::task::spawn_blocking({
655            let id = id.to_owned();
656            move || {
657                let _guard = guard;
658                commit_refresh(&store, &id, &current, fence, &patch, publish_proxy)
659            }
660        })
661        .await
662        .map_err(|e| PatchError::Io(e.to_string()))
663        .and_then(|r| r)
664        .map_err(|e| {
665            // The plugin refreshed but nothing was saved: back off as for a failed refresh.
666            if plugin.is_some() {
667                crate::plugins::refresh::failed(&self.store, id, registered.registration, chrono::Utc::now());
668            }
669            ExecError::local(
670                500,
671                FailureScope::Request,
672                format!("committing prepared credential: {e:?}"),
673            )
674        })?;
675        Ok(committed)
676    }
677
678    /// [`Executors::readiness`], plus the schedule of credentials a plugin refreshes
679    /// ([`crate::plugins::refresh`]), which never holds a request. Built-in providers
680    /// read their executor alone.
681    fn readiness(&self, c: &Credential, cfg: &Config) -> Readiness {
682        match self.executors.readiness(c, cfg) {
683            Readiness::Ready if self.plugin_refresh_due(c, cfg, chrono::Utc::now()) => Readiness::RefreshSoon,
684            readiness => readiness,
685        }
686    }
687
688    fn plugin_refresh_due(&self, c: &Credential, cfg: &Config, now: chrono::DateTime<chrono::Utc>) -> bool {
689        crate::plugins::refresh::due_at(self, c, cfg).is_some_and(|at| at <= now)
690    }
691
692    /// Go `tryRefreshAfterUnauthorized`: after an upstream 401 on a credential whose
693    /// executor reports a refresh credential, prepare it once (due or not) and return the
694    /// committed replacement to retry with.
695    // ponytail: refresh goes through `Executors::prepare_forced`: Codex refreshes when not
696    // due, and Claude's prepare already refreshed on demand before 50d5fe18. A provider
697    // whose prepare refreshes only inside its lead cannot recover a revoked token here.
698    pub async fn refresh_after_unauthorized(&self, credential: &Credential, cfg: &Config) -> Option<Arc<Credential>> {
699        if !self.executors.has_refresh_credential(credential) {
700            return None;
701        }
702        let refreshed = match self
703            .prepare_credential_with(&credential.id, cfg, Some(credential.revision), false, Some(credential))
704            .await
705        {
706            Ok(refreshed) => refreshed,
707            Err(error) => {
708                // Go `refreshAuthForRequestAtEpoch`: neither token can recover without a
709                // new sign-in when the refresh token is refused while the access token
710                // the upstream refused is still current. The credential is neither
711                // selected nor refreshed again until its tokens change. Preparation
712                // recorded that, or refused because another 401 had.
713                if self.store.tokens_rejected(credential) {
714                    if !crate::classify::is_invalid_grant(error.status, &String::from_utf8_lossy(&error.body)) {
715                        return None;
716                    }
717                    tracing::warn!(
718                        "credential refresh failed for {} ({}) after an upstream 401: status {}: {}; it is not used again until it is signed in again or its file changes",
719                        credential.provider,
720                        credential.id,
721                        error.status,
722                        refresh_error_text(&error)
723                    );
724                    return None;
725                }
726                // The background loop's schedule is the next attempt, unless another 401
727                // comes first.
728                let next = self
729                    .refresh_state
730                    .lock()
731                    .unwrap_or_else(PoisonError::into_inner)
732                    .retry_at(credential)
733                    .map(|at| at.saturating_duration_since(Instant::now()));
734                warn_refresh_failed(credential, " after an upstream 401", &error, next);
735                return None;
736            }
737        };
738        (refreshed.revision != credential.revision).then(|| {
739            tracing::info!(
740                "credential refreshed for {} ({}) after an upstream 401",
741                credential.provider,
742                credential.id
743            );
744            refreshed
745        })
746    }
747
748    /// One replaceable refresh loop. Uses the same preparation lock and atomic
749    /// revision-checked commit as request acquisition; executors never persist files.
750    /// ponytail: needs_prepare controls eligibility. Unauthorized lifecycle gating
751    /// and end-to-end token rotation await executor/lifecycle integration (M4-0027).
752    pub fn start_auto_refresh(self: &Arc<Self>) {
753        let mut task = self.refresh_task.lock().unwrap_or_else(PoisonError::into_inner);
754        if let Some(previous) = task.take() {
755            previous.abort();
756        }
757        let weak = Arc::downgrade(self);
758        // Only shutdown (the runtime dropped) or an abort ends the loop: a panic anywhere
759        // in it is logged and the loop starts again, with fresh subscriptions.
760        let handle = tokio::spawn(async move {
761            loop {
762                match AssertUnwindSafe(refresh_loop(weak.clone())).catch_unwind().await {
763                    Ok(()) => break,
764                    Err(panic) => {
765                        tracing::error!(
766                            "credential auto-refresh loop panicked ({}); restarting it in {}s",
767                            panic_text(&*panic),
768                            REFRESH_LOOP_RESTART.as_secs()
769                        );
770                        tokio::time::sleep(REFRESH_LOOP_RESTART).await;
771                    }
772                }
773            }
774        });
775        *task = Some(handle.abort_handle());
776    }
777
778    pub fn stop_auto_refresh(&self) {
779        if let Some(task) = self.refresh_task.lock().unwrap_or_else(PoisonError::into_inner).take() {
780            task.abort();
781        }
782    }
783}
784
785/// The body of [`Runtime::start_auto_refresh`]'s task. Returns when the runtime is gone.
786async fn refresh_loop(weak: std::sync::Weak<Runtime>) {
787    // Wakes at the next deadline, or at the latest after `HORIZON` of awake time while
788    // a credential can ever come due, to catch wall-clock jumps such as a resumed
789    // laptop (monotonic time stops during suspend). Credentials that never need
790    // preparation (API keys, Gemini CLI tokens) set no timer.
791    const HORIZON: Duration = Duration::from_secs(600);
792    // Store changes (each committed refresh is one) are coalesced over this delay.
793    const COALESCE: Duration = Duration::from_secs(1);
794    let Some(rt) = weak.upgrade() else {
795        return;
796    };
797    let mut store_changes = rt.store.subscribe();
798    let mut config_changes = rt.subscribe_config();
799    let mut unauthorized = rt.store.subscribe_unauthorized();
800    let mut jobs_ended = rt.refresh_done.subscribe();
801    // The worker bound (`oauth.auth-auto-refresh-workers`), taken inside each job.
802    let mut workers = (0, Arc::new(tokio::sync::Semaphore::new(0)));
803    drop(rt);
804    loop {
805        let Some(rt) = weak.upgrade() else {
806            break;
807        };
808        // Everything published so far is in this scan, including the commits of
809        // the previous batch.
810        store_changes.borrow_and_update();
811        config_changes.borrow_and_update();
812        unauthorized.borrow_and_update();
813        jobs_ended.borrow_and_update();
814        let cfg = rt.config();
815        // A soft bound while the setting changes: jobs holding the old semaphore's
816        // permits run beside the new one's.
817        if workers.0 != refresh_workers(&cfg) {
818            workers = (
819                refresh_workers(&cfg),
820                Arc::new(tokio::sync::Semaphore::new(refresh_workers(&cfg))),
821            );
822        }
823        // Each reservation is handed to its spawned job at once, so whatever ends this
824        // loop (an abort, a panic later in the scan), the job ends the reservation.
825        let spawn = |c: &Arc<Credential>, rejected: bool| {
826            let job = run_refresh_job(rt.clone(), cfg.clone(), workers.1.clone(), c.clone(), rejected);
827            tokio::spawn(job)
828        };
829        // Before the snapshot, so a 401 on a revision newer than it stays.
830        let rejected = rt.store.unauthorized();
831        let snapshot = rt.store.snapshot();
832        let (now, wall) = (Instant::now(), chrono::Utc::now());
833        let horizon = chrono::Duration::from_std(HORIZON).unwrap_or_default();
834        let mut wake: Option<Instant> = None;
835        let mut soonest = |at: Instant| wake = Some(wake.map_or(at, |w| w.min(at)));
836        let jobs: Vec<_> = {
837            let mut state = rt.refresh_state.lock().unwrap_or_else(PoisonError::into_inner);
838            state.reconcile(&snapshot);
839            let mut jobs = Vec::new();
840            rt.store.forget_stale_unauthorized(&rejected, &snapshot);
841            // Far enough ahead that any expiry has passed, near enough for chrono.
842            let eventually = wall + chrono::Duration::days(36_500);
843            // Go `shouldRefresh`: tokens refused for good wait for new ones.
844            let refused = rt.store.rejected_ids(&snapshot);
845            for c in snapshot.iter().filter(|c| !refused.contains(&c.id)) {
846                if !refresh_candidate(c) {
847                    // Go refreshes a disabled credential; here its due refresh waits until
848                    // it is enabled, logged once per revision, when it comes due.
849                    if c.disabled && refreshable(c) && !state.disabled_noted(c) {
850                        let plugin_due = crate::plugins::refresh::due_at(&rt, c, &cfg);
851                        if rt.executors.needs_prepare_at(c, &cfg, eventually) || plugin_due.is_some() {
852                            soonest(now + HORIZON);
853                        }
854                        let due = rt
855                            .executors
856                            .prepare_due(c, &cfg, wall, horizon)
857                            .or_else(|| plugin_due.filter(|at| *at <= wall + horizon).map(|at| at.max(wall)));
858                        match due {
859                            Some(at) if at > wall => soonest(now + (at - wall).to_std().unwrap_or_default()),
860                            Some(_) => {
861                                state.note_disabled(c);
862                                tracing::info!(
863                                    "credential refresh for {} ({}) is due but the credential is disabled; it is refreshed once it is enabled",
864                                    c.provider,
865                                    c.id
866                                );
867                            }
868                            None => {}
869                        }
870                    }
871                    continue;
872                }
873                test_hook("scan", &c.id);
874                // Go `scheduleUnauthorizedRefresh`: a rejected token refreshes now,
875                // or after `refreshFailureBackoff` when it was refreshed just before.
876                // Its deadline replaces the ordinary one (Go `NextRefreshAfter`).
877                if let Some((revision, noted)) = rejected.pending.get(&c.id)
878                    && *revision == c.revision
879                    && rt.executors.has_refresh_credential(c)
880                {
881                    let at = rejected
882                        .refreshed
883                        .get(&c.id)
884                        .filter(|(r, t)| {
885                            *r <= *revision && noted.saturating_duration_since(*t) < REFRESH_FAILURE_BACKOFF
886                        })
887                        .map_or(*noted, |_| *noted + REFRESH_FAILURE_BACKOFF);
888                    if at > now {
889                        soonest(at);
890                    } else if state.reserve(c, now) {
891                        jobs.push(spawn(c, true));
892                    } else {
893                        warn_refresh_held(&mut state, c);
894                        if let Some(at) = state.retry_at(c).filter(|at| *at > now) {
895                            soonest(at);
896                        }
897                    }
898                    continue;
899                }
900                let plugin_due = crate::plugins::refresh::due_at(&rt, c, &cfg);
901                if rt.executors.needs_prepare_at(c, &cfg, eventually) || plugin_due.is_some() {
902                    soonest(now + HORIZON);
903                }
904                let due = rt
905                    .executors
906                    .prepare_due(c, &cfg, wall, horizon)
907                    .or_else(|| plugin_due.filter(|at| *at <= wall + horizon).map(|at| at.max(wall)));
908                match due {
909                    Some(at) if at > wall => soonest(now + (at - wall).to_std().unwrap_or_default()),
910                    Some(_) if state.reserve(c, now) => jobs.push(spawn(c, false)),
911                    // A job still running, or backing off: look again when that ends.
912                    Some(_) => {
913                        warn_refresh_held(&mut state, c);
914                        if let Some(at) = state.retry_at(c).filter(|at| *at > now) {
915                            soonest(at);
916                        }
917                    }
918                    None => {}
919                }
920            }
921            jobs
922        };
923        let busy = !jobs.is_empty();
924        // The loop waits for the batch, at most `REFRESH_JOB_BOUND` per job, then looks
925        // again. A job it stops waiting for runs on, records its own outcome and wakes the
926        // loop when it ends.
927        futures_util::future::join_all(jobs.into_iter().map(|job| async move {
928            let _ = tokio::time::timeout(REFRESH_JOB_BOUND, job).await;
929        }))
930        .await;
931        drop(rt);
932        // After a batch, look again at once: finished jobs set their backoff,
933        // and the next scan turns it into a deadline.
934        if busy {
935            continue;
936        }
937        let deadline = async {
938            match wake {
939                Some(at) => tokio::time::sleep_until(at.into()).await,
940                None => std::future::pending().await,
941            }
942        };
943        tokio::select! {
944            () = deadline => {}
945            changed = store_changes.changed() => {
946                if changed.is_err() {
947                    break;
948                }
949                tokio::time::sleep(COALESCE).await;
950            }
951            changed = config_changes.changed() => {
952                if changed.is_err() {
953                    break;
954                }
955                tokio::time::sleep(COALESCE).await;
956            }
957            changed = unauthorized.changed() => {
958                if changed.is_err() {
959                    break;
960                }
961            }
962            changed = jobs_ended.changed() => {
963                if changed.is_err() {
964                    break;
965                }
966                tokio::time::sleep(COALESCE).await;
967            }
968        }
969    }
970}
971
972/// One background refresh job of [`refresh_loop`], which reserved `credential`: prepares
973/// it, then records the outcome in the refresh state, which ends the reservation, and
974/// logs it. The reservation ends exactly once on every path: a panic while preparing is
975/// a failure, and [`Reservation`]'s drop covers the rest. The exception is a job dropped
976/// before its first poll at runtime teardown, whose reservation goes with the runtime.
977// ponytail: a job the loop stopped waiting for keeps running, holding the runtime, and
978// neither `stop_auto_refresh` nor the runtime's drop aborts it (its exchange may have
979// spent the refresh token already). At most one such job per credential outlives the
980// loop, as the reservation holds off the next.
981async fn run_refresh_job(
982    rt: Arc<Runtime>,
983    cfg: Arc<Config>,
984    workers: Arc<tokio::sync::Semaphore>,
985    credential: Arc<Credential>,
986    rejected: bool,
987) {
988    let mut reservation = Reservation {
989        rt: rt.clone(),
990        credential: credential.clone(),
991        ended: false,
992    };
993    let worker = workers.acquire_owned().await;
994    let failed = rejected.then_some(credential.revision);
995    // A 401 refresh records a refused token as the inline one does.
996    let refused = rejected.then_some(&*credential);
997    let prepare =
998        AssertUnwindSafe(rt.prepare_credential_with(&credential.id, &cfg, failed, false, refused)).catch_unwind();
999    tokio::pin!(prepare);
1000    // Past the bound, counted from when the job got its worker, the worker goes to the
1001    // next job, and this one awaits the same preparation, never cancelled, still
1002    // holding its reservation.
1003    let prepared = match tokio::time::timeout(REFRESH_JOB_BOUND, &mut prepare).await {
1004        Ok(prepared) => prepared,
1005        Err(_) => {
1006            tracing::warn!(
1007                "credential refresh for {} ({}) still running after {}s; no other refresh of it starts until it ends",
1008                credential.provider,
1009                credential.id,
1010                REFRESH_JOB_BOUND.as_secs()
1011            );
1012            drop(worker);
1013            prepare.await
1014        }
1015    };
1016    // A panic records no plugin deadline: it backs off here, plugin route or not.
1017    let (result, panicked) = match prepared {
1018        Ok(result) => (result, false),
1019        Err(panic) => {
1020            let text = format!("refresh panicked: {}", panic_text(&*panic));
1021            (Err(ExecError::local(500, FailureScope::Request, text)), true)
1022        }
1023    };
1024    if rejected && result.is_ok() {
1025        rt.store.forget_unauthorized(&credential.id, credential.revision);
1026    }
1027    let trigger = if rejected { " after an upstream 401" } else { "" };
1028    let current = rt.store.get(&credential.id);
1029    // A concurrent removal/re-import or management edit must not transfer the old
1030    // refresh failure/backoff to the replacement, which is tried when it is due.
1031    let Some(current) = current.filter(|c| match &result {
1032        Ok(prepared) => prepared.revision == c.revision,
1033        Err(_) => c.revision == credential.revision,
1034    }) else {
1035        reservation.end(|state| state.release(&credential.id));
1036        if let Err(error) = &result {
1037            warn_refresh_failed(&credential, trigger, error, None);
1038        }
1039        return;
1040    };
1041    // A plugin refresh keeps its own deadlines (Go `NextRefreshAfter`,
1042    // `crate::plugins::refresh`): release the reservation only.
1043    let plugin = !panicked && crate::plugins::refresh::route(&rt, &current, &cfg).is_some();
1044    let ineffective = !plugin && result.is_ok() && rt.readiness(&current, &cfg) != Readiness::Ready;
1045    let now = Instant::now();
1046    let retry = reservation.end(|state| {
1047        state.finish(&current, result.as_ref().err().filter(|_| !plugin), ineffective, now);
1048        state.retry_at(&current)
1049    });
1050    match &result {
1051        Err(error) => {
1052            let next = if plugin {
1053                crate::plugins::refresh::due_at(&rt, &current, &cfg)
1054                    .map(|at| (at - chrono::Utc::now()).to_std().unwrap_or_default())
1055            } else {
1056                retry.map(|at| at.saturating_duration_since(now))
1057            };
1058            warn_refresh_failed(&current, trigger, error, next);
1059        }
1060        // Go retries an ineffective refresh after 30 s and logs nothing.
1061        Ok(_) if ineffective => {}
1062        Ok(_) if current.revision != credential.revision => {
1063            tracing::info!(
1064                "credential refreshed for {} ({}){trigger}",
1065                current.provider,
1066                current.id
1067            );
1068        }
1069        Ok(_) => {}
1070    }
1071}
1072
1073/// A refresh job's reservation of its credential. [`Self::end`] records the outcome and
1074/// wakes the refresh loop; a job that ends any other way (a panic outside preparation,
1075/// cancellation at shutdown) records a failure when this drops.
1076struct Reservation {
1077    rt: Arc<Runtime>,
1078    credential: Arc<Credential>,
1079    ended: bool,
1080}
1081
1082impl Reservation {
1083    fn end<T>(&mut self, record: impl FnOnce(&mut RefreshState) -> T) -> T {
1084        let recorded = record(&mut self.rt.refresh_state.lock().unwrap_or_else(PoisonError::into_inner));
1085        self.ended = true;
1086        self.rt.refresh_done.send_replace(());
1087        recorded
1088    }
1089}
1090
1091impl Drop for Reservation {
1092    fn drop(&mut self) {
1093        if !self.ended {
1094            let error = ExecError::local(500, FailureScope::Request, "refresh job ended without a result");
1095            let credential = self.credential.clone();
1096            self.end(|state| state.finish(&credential, Some(&error), false, Instant::now()));
1097            warn_refresh_failed(&credential, "", &error, None);
1098        }
1099    }
1100}
1101
1102impl Drop for Runtime {
1103    fn drop(&mut self) {
1104        if let Some(task) = self
1105            .refresh_task
1106            .get_mut()
1107            .unwrap_or_else(PoisonError::into_inner)
1108            .take()
1109        {
1110            task.abort();
1111        }
1112    }
1113}
1114
1115/// Where cooldowns persist: `auth-dir` while `save-cooldown-status` is on.
1116fn cooldown_dir(cfg: &Config, policy: &Policy) -> Option<std::path::PathBuf> {
1117    policy.save_cooldown_status.then(|| cfg.auth_dir.clone())
1118}
1119
1120/// How long the refresh loop waits for one refresh job, the wait for the credential's
1121/// preparation lock included, before it moves on. The job runs on and records its own
1122/// outcome; until then no other job for the credential starts.
1123const REFRESH_JOB_BOUND: Duration = Duration::from_secs(if cfg!(test) { 2 } else { 300 });
1124
1125/// The pause before a panicked refresh loop starts again.
1126const REFRESH_LOOP_RESTART: Duration = Duration::from_secs(if cfg!(test) { 0 } else { 5 });
1127
1128/// One warning per failed refresh (Go logs most of these at Debug; see
1129/// DIFFERENCES-FROM-GO.md): provider, credential, status, the error and when the next
1130/// attempt is. `trigger` names what started it. A built-in executor's error text is
1131/// sanitized as client-facing upstream errors are (Go `SanitizeUpstreamErrorSummary`);
1132/// of a plugin's failure only its error code is logged, never its message, and token
1133/// endpoint bodies never reach the error text.
1134fn warn_refresh_failed(c: &Credential, trigger: &str, error: &ExecError, next: Option<Duration>) {
1135    let text = refresh_error_text(error);
1136    let next = match next {
1137        Some(after) if !after.is_zero() => format!("next attempt in {}s", ceil_secs(after)),
1138        _ => "next attempt when it is next due".to_owned(),
1139    };
1140    tracing::warn!(
1141        "credential refresh failed for {} ({}){trigger}: status {}: {text}; {next}",
1142        c.provider,
1143        c.id,
1144        error.status
1145    );
1146}
1147
1148/// What [`warn_refresh_failed`] logs of a refresh error.
1149fn refresh_error_text(error: &ExecError) -> String {
1150    match error.headers.get(crate::plugins::refresh::PLUGIN_ERROR) {
1151        Some(code) => match code.to_str().unwrap_or_default() {
1152            "" => "plugin refresh failed".to_owned(),
1153            code => format!("plugin refresh failed: {code}"),
1154        },
1155        None => crate::sanitize::summary(&String::from_utf8_lossy(&error.body)),
1156    }
1157}
1158
1159/// A due refresh held back by its previous job, which is still running: logged once
1160/// per job. A finished job's backoff is not logged here: a failure announced it, and Go
1161/// is silent about an ineffective refresh.
1162fn warn_refresh_held(state: &mut RefreshState, c: &Credential) {
1163    if state.note_hold(c) {
1164        tracing::warn!(
1165            "credential refresh for {} ({}) is due but its previous refresh is still running; it is tried again once that ends",
1166            c.provider,
1167            c.id
1168        );
1169    }
1170}
1171
1172fn ceil_secs(d: Duration) -> u64 {
1173    d.as_secs() + u64::from(d.subsec_nanos() > 0)
1174}
1175
1176/// A panic payload's message.
1177fn panic_text(panic: &(dyn std::any::Any + Send)) -> &str {
1178    panic
1179        .downcast_ref::<&str>()
1180        .copied()
1181        .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
1182        .unwrap_or("non-string payload")
1183}
1184
1185/// Go `refreshFailureBackoff`: an unauthorized refresh waits this long after a refresh
1186/// that just rotated the tokens.
1187const REFRESH_FAILURE_BACKOFF: Duration = Duration::from_secs(300);
1188
1189/// `oauth.auth-auto-refresh-workers`: non-positive means Go's default of 16.
1190fn refresh_workers(cfg: &Config) -> usize {
1191    cfg.document
1192        .get("oauth")
1193        .and_then(|o| o.get("auth-auto-refresh-workers"))
1194        .and_then(serde_yaml_ng::Value::as_i64)
1195        .filter(|n| *n > 0)
1196        .map_or(16, |n| n as usize)
1197}
1198
1199/// How a selection picks among its ready candidates.
1200pub(crate) enum Pick {
1201    /// The configured selector (strategy and session affinity).
1202    Configured,
1203    /// A plugin scheduler's delegation to a built-in strategy.
1204    Builtin(crate::scheduler::Strategy),
1205    /// A plugin scheduler's chosen credential.
1206    Auth(String),
1207}
1208
1209/// A selection's eligible credentials with their execution models, and those of them
1210/// that are ready now with their provider keys; when none is ready, the error Go's
1211/// availability check reports. `configured` adds the configured selector's own filter
1212/// (weighted round-robin skips non-positive weights); a plugin scheduler and its
1213/// built-in delegates see every available credential.
1214#[allow(clippy::type_complexity)]
1215fn ready<'a>(
1216    creds: &'a [Arc<Credential>],
1217    scheduler: &crate::scheduler::Scheduler,
1218    selection: &Selection,
1219    policy: &Policy,
1220    admit: &Admit<'_>,
1221    now: Instant,
1222    configured: bool,
1223) -> Result<(Vec<(&'a Arc<Credential>, String)>, Vec<(&'a Arc<Credential>, String)>), AcquireError> {
1224    let eligible: Vec<_> = creds
1225        .iter()
1226        .filter(|c| !c.disabled && !selection.exclude.contains(&c.id))
1227        .filter(|c| policy.retry_limit(c) >= selection.retry_round && (!configured || scheduler.admits(c, policy)))
1228        .filter_map(|c| admit(c).map(|m| (c, m)))
1229        .collect();
1230    let candidates: Vec<_> = eligible
1231        .iter()
1232        .filter(|(c, m)| scheduler.wait(c, m, now).is_none() && !scheduler.tokens_rejected(c))
1233        .map(|(c, _)| (*c, crate::registry::provider_key(c)))
1234        .collect();
1235    if candidates.is_empty() {
1236        let cause = eligible
1237            .iter()
1238            .filter_map(|(c, m)| scheduler.last_error(c, m))
1239            .max_by_key(|s| s.deadline)
1240            .map(|s| s.error.clone());
1241        let wait = eligible.iter().filter_map(|(c, m)| scheduler.wait(c, m, now)).min();
1242        if !eligible.is_empty() && eligible.iter().all(|(c, m)| scheduler.quota_cooling(c, m, now)) {
1243            return Err(AcquireError::Cooldown {
1244                wait: wait.unwrap_or_default(),
1245                cause,
1246            });
1247        }
1248        return Err(AcquireError::Unavailable {
1249            retry_after: if eligible.is_empty() { None } else { wait },
1250            cause,
1251        });
1252    }
1253    Ok((eligible, candidates))
1254}
1255
1256#[derive(Debug)]
1257pub enum AcquireError {
1258    /// No candidate is ready. `retry_after` is set when candidates exist but are all
1259    /// blocked (Go `auth_unavailable`); unset means none is eligible (`auth_not_found`).
1260    Unavailable {
1261        retry_after: Option<Duration>,
1262        cause: Option<String>,
1263    },
1264    /// Every eligible candidate is quota-cooling for this model.
1265    Cooldown {
1266        wait: Duration,
1267        cause: Option<String>,
1268    },
1269    Prepare {
1270        id: String,
1271        error: ExecError,
1272    },
1273    /// A plugin scheduler rejected the selection (Go `pickViaPluginScheduler`).
1274    Rejected {
1275        code: String,
1276        message: String,
1277    },
1278    /// A plugin scheduler's call failed; Go returns its error as the selection error.
1279    Scheduler(ExecError),
1280}
1281
1282/// Route-model state is separate from the lease's resolved execution model.
1283#[derive(Debug, Clone, Default)]
1284pub struct Selection {
1285    /// Provider keys that may serve the model, in preference order.
1286    pub providers: Vec<String>,
1287    /// Single-provider shorthand, used when `providers` is empty.
1288    pub provider: String,
1289    /// The route model (requested model after `auto` resolution, suffix kept).
1290    pub model: String,
1291    /// The session this request binds to (Go `extractSessionIDs` primary).
1292    pub session: Option<String>,
1293    /// The session's parent, or the conversation alias of a prompt-cache-key session
1294    /// (Go fallback ID).
1295    pub session_parent: Option<String>,
1296    /// The session forked from `session_parent` (inherits its binding).
1297    pub session_fork: bool,
1298    /// A request without an explicit session, for the LCP matcher (Go `pickLCP`). Set
1299    /// only with session affinity on; it then decides the binding instead of `session`.
1300    pub lcp: Option<Arc<crate::lcp::Request>>,
1301    /// Credential IDs already tried in this request.
1302    pub exclude: Vec<String>,
1303    pub retry_round: usize,
1304    /// What a plugin scheduler learns about the request (Go `SchedulerPickRequest`'s
1305    /// `Stream` and `Options`); without it the scheduler sees no headers or metadata.
1306    pub scheduler: Option<Arc<crate::plugins::SchedulerContext>>,
1307}
1308
1309impl Selection {
1310    pub fn new(provider: &str, model: &str) -> Self {
1311        Self {
1312            providers: vec![provider.to_owned()],
1313            model: model.to_owned(),
1314            ..Self::default()
1315        }
1316    }
1317
1318    /// The provider keys this selection may use.
1319    pub fn provider_keys(&self) -> Vec<String> {
1320        if self.providers.is_empty() && !self.provider.is_empty() {
1321            vec![self.provider.clone()]
1322        } else {
1323            self.providers.clone()
1324        }
1325    }
1326}
1327
1328/// Whether `next` would change nothing about the stored `current` (reconcile keeps
1329/// such a credential and its revision).
1330fn unchanged(current: &Credential, next: &Credential) -> bool {
1331    current.id == next.id
1332        && current.source == next.source
1333        && current.metadata == next.metadata
1334        && current.attributes == next.attributes
1335        && current.provider == next.provider
1336        && current.disabled == next.disabled
1337        && current.label == next.label
1338}
1339
1340/// Admission without a registry: provider match, prefix, config aliases and the
1341/// credential's exclusions (attributes first, then file metadata).
1342pub fn standalone_admission<'a>(
1343    selection: &'a Selection,
1344    policy: &'a Policy,
1345) -> impl Fn(&Credential) -> Option<String> + 'a {
1346    let providers = selection.provider_keys();
1347    move |c| {
1348        if !providers.contains(&crate::registry::provider_key(c)) {
1349            return None;
1350        }
1351        let route = selection.model.trim();
1352        let prefix = crate::registry::credential_prefix(c);
1353        let base = crate::scheduler::canonical_model(route);
1354        if !prefix.is_empty()
1355            && !base.starts_with(&format!("{prefix}/"))
1356            && (policy.force_model_prefix || base.contains('/'))
1357        {
1358            return None;
1359        }
1360        let (models, _) = crate::registry::execution_models(&HashMap::new(), c, route);
1361        let model = models.into_iter().next().unwrap_or_default();
1362        // Go lowercases model IDs and patterns with `strings.ToLower`.
1363        let key = crate::scheduler::canonical_model(&model).go_lower();
1364        let excluded: Vec<String> = match c.attributes.get("excluded_models").filter(|v| !v.trim().is_empty()) {
1365            Some(list) => list.split(',').map(|p| p.trim().go_lower()).collect(),
1366            None => c
1367                .metadata
1368                .get("excluded_models")
1369                .and_then(serde_json::Value::as_array)
1370                .map(|l| {
1371                    l.iter()
1372                        .filter_map(|v| v.as_str())
1373                        .map(|p| p.trim().go_lower())
1374                        .collect()
1375                })
1376                .unwrap_or_default(),
1377        };
1378        if excluded.iter().any(|p| crate::registry::wildcard(p, &key)) {
1379            return None;
1380        }
1381        Some(model)
1382    }
1383}
1384
1385#[derive(Debug, Clone)]
1386pub enum Outcome {
1387    Success,
1388    Failure(ExecError),
1389    /// A failure that must not change availability (compact request faults, a
1390    /// missing count_tokens endpoint).
1391    Neutral(ExecError),
1392    /// The client went away before the response finished.
1393    Cancelled,
1394}
1395
1396/// Decides which credentials may serve a selection and under which model key their
1397/// cooldowns live. `None` rejects the credential.
1398pub type Admit<'a> = dyn Fn(&Credential) -> Option<String> + 'a;
1399
1400/// Registry admission (Go `authSupportsRouteModel`) plus a registered executor, built-in
1401/// or plugin.
1402pub fn admission<'a>(
1403    registry: &'a Registry,
1404    aliases: &'a HashMap<String, Vec<crate::registry::OAuthAlias>>,
1405    selection: &'a Selection,
1406    executors: &'a Executors,
1407    plugins: &'a crate::plugins::PluginRuntime,
1408) -> impl Fn(&Credential) -> Option<String> + 'a {
1409    move |c| {
1410        let provider = crate::registry::provider_key(c);
1411        if !selection.provider_keys().contains(&provider)
1412            || !(executors.supports(&c.provider) || plugins.has_executor(&c.provider))
1413        {
1414            return None;
1415        }
1416        let key = crate::registry::selection_model(aliases, c, &selection.model);
1417        let route = crate::scheduler::canonical_model(&selection.model);
1418        if route.is_empty() {
1419            return Some(key);
1420        }
1421        let selection_key = crate::scheduler::canonical_model(&key);
1422        (registry.client_supports(&c.id, route)
1423            || (selection_key != route && registry.client_supports(&c.id, selection_key)))
1424        .then_some(key)
1425    }
1426}
1427
1428/// One attempt with one credential. Not cloneable: it reports exactly one outcome.
1429pub struct Lease {
1430    store: Arc<CredentialStore>,
1431    pub credential: Arc<Credential>,
1432    pub selection: Selection,
1433    /// The model key the outcome is recorded under (Go `stateModelForExecution`).
1434    pub execution_model: String,
1435    pub attempt: u64,
1436    policy: Arc<Policy>,
1437    reported: bool,
1438    /// The scheduler generation at the pick (`Scheduler::reserve_probe`): bounded
1439    /// windows opened or probes reserved after it are not answered by this attempt.
1440    picked: u32,
1441    /// The upstream dispatch stamp (`Scheduler::dispatch`); 0 for remote leases.
1442    seq: u64,
1443    /// A credential from the remote dispatcher: its lease ends there, not in the local
1444    /// scheduler.
1445    remote: Option<crate::remote::RemoteEnd>,
1446    /// The LCP binding of the pick: the attempt's canonical session and lineage.
1447    pub lcp: Option<crate::lcp::Match>,
1448}
1449
1450impl Lease {
1451    /// Whether the credential came from the remote dispatcher (Go Home mode).
1452    pub fn is_remote(&self) -> bool {
1453        self.remote.is_some()
1454    }
1455
1456    /// What a Home pick keeps outside the credential's attributes.
1457    pub(crate) fn home_private(&self) -> Option<&crate::remote::Private> {
1458        self.remote.as_ref().map(|remote| &remote.private)
1459    }
1460
1461    /// Resolves when the dispatcher cancels this lease's execution.
1462    pub(crate) fn remote_cancelled(&self) -> Option<futures_util::future::BoxFuture<'static, ()>> {
1463        let signal = self.remote.as_ref()?.cancel.clone()?;
1464        Some(crate::remote::cancelled(signal))
1465    }
1466
1467    /// Whether the dispatcher cancelled this lease's execution (Go: its selection is
1468    /// no longer `Active`).
1469    pub(crate) fn remote_cancel_requested(&self) -> bool {
1470        self.remote
1471            .as_ref()
1472            .and_then(|remote| remote.cancel.as_ref())
1473            .is_some_and(|signal| *signal.borrow())
1474    }
1475
1476    /// A new upstream dispatch on this lease (a refresh-and-retry): a cooldown recorded
1477    /// before it no longer predates this attempt (Go `nextExecutionSeq` per dispatch).
1478    pub(crate) fn redispatch(&mut self) {
1479        if self.remote.is_none() {
1480            self.seq = self
1481                .store
1482                .scheduler
1483                .lock()
1484                .unwrap_or_else(PoisonError::into_inner)
1485                .dispatch();
1486        }
1487    }
1488
1489    pub fn complete(mut self, outcome: Outcome) {
1490        self.report(outcome);
1491    }
1492
1493    /// Joins a remote lease's release to `releases`: a Home pick a session kept ends or
1494    /// runs again during a later request, which awaits the release before its next pick.
1495    pub(crate) fn release_into(&mut self, releases: &crate::remote::PendingReleases) {
1496        if let Some(remote) = &mut self.remote {
1497            remote.releases = releases.clone();
1498        }
1499    }
1500
1501    /// The upstream accepted this attempt (a stream's first chunk arrived): an ended
1502    /// bounded window it probes has its answer, so other requests may use the account
1503    /// while the stream runs (`Scheduler::accept_probe`). A no-op for remote leases.
1504    pub fn accepted(&self) {
1505        if self.remote.is_none() {
1506            self.store.accept_probe(self);
1507        }
1508    }
1509
1510    /// Records an intermediate outcome for one model of a pooled alias without ending
1511    /// the lease.
1512    pub fn note(&self, model: &str, outcome: &Outcome) {
1513        if self.remote.is_none() {
1514            self.store.record_model(self, model, outcome);
1515        }
1516    }
1517
1518    fn report(&mut self, outcome: Outcome) {
1519        if std::mem::replace(&mut self.reported, true) {
1520            return;
1521        }
1522        match &mut self.remote {
1523            // Go `reportHomeResult` leaves local cooldowns alone; the scope ends with
1524            // its release.
1525            Some(remote) => remote.finish(),
1526            None => self.store.record(self, &outcome),
1527        }
1528    }
1529}
1530
1531impl Drop for Lease {
1532    fn drop(&mut self) {
1533        self.report(Outcome::Cancelled);
1534    }
1535}
1536
1537impl std::fmt::Debug for Lease {
1538    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1539        f.debug_struct("Lease")
1540            .field("credential", &self.credential.id)
1541            .field("attempt", &self.attempt)
1542            .finish()
1543    }
1544}
1545
1546#[derive(Debug, PartialEq, Eq)]
1547pub enum PatchError {
1548    NotFound,
1549    /// The credential changed since the caller read it; re-read and retry.
1550    Stale {
1551        current: u64,
1552    },
1553    /// `type` decides the executor and cannot change in place.
1554    TypeIsImmutable,
1555    /// Config-sourced credentials persist through the config document, not here.
1556    ConfigBacked,
1557    Io(String),
1558    /// Another holder has the credential lock of this Codex file. The writer releases
1559    /// what it holds, waits for the lock ([`Runtime::lock_codex_file`]) and retries.
1560    LockBusy(std::path::PathBuf),
1561}
1562
1563/// What a token refresh commits under ([`CredentialStore::apply_refresh`]).
1564#[derive(Debug, Clone, Copy)]
1565pub(crate) struct RefreshFence {
1566    /// The registration and credential version the refresh started under
1567    /// ([`CredentialStore::get_registered`]).
1568    pub registered: crate::plugins::refresh::Registered,
1569    /// A plugin refresh, whose schedule the commit records.
1570    pub plugin: bool,
1571}
1572
1573#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1574pub struct AttemptStats {
1575    pub success: u64,
1576    pub failure: u64,
1577    pub cancelled: u64,
1578}
1579
1580struct Inner {
1581    creds: Vec<Arc<Credential>>,
1582    /// Last revision handed out. Revisions are never reused.
1583    generation: u64,
1584    /// Bumped on every reconcile and patch, including pure removals.
1585    epoch: u64,
1586}
1587
1588/// Pending background refreshes after a 401, and the last token rotations.
1589#[derive(Debug, Clone, Default)]
1590pub(crate) struct Rejections {
1591    /// The rejected revision of each credential and when the 401 was recorded.
1592    pub pending: HashMap<String, (u64, Instant)>,
1593    /// The revision each credential's last token rotation committed, and when.
1594    pub refreshed: HashMap<String, (u64, Instant)>,
1595}
1596
1597pub struct CredentialStore {
1598    inner: RwLock<Inner>,
1599    policy: RwLock<Arc<Policy>>,
1600    scheduler: Mutex<Scheduler>,
1601    attempts: AtomicU64,
1602    stats: [AtomicU64; 3],
1603    prepare_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
1604    /// Go 8a01bc02: a credential file is written with `inner` unlocked. Each
1605    /// credential's lock serializes its own writes, and the reload gate (shared by
1606    /// writes, exclusive for a set replacement) keeps a reload from landing between a
1607    /// write and its publication.
1608    write_locks: Mutex<HashMap<String, Arc<Mutex<()>>>>,
1609    reload: RwLock<()>,
1610    /// `auth-dir` while `save-cooldown-status` is on (Go `cooldownStore`).
1611    cooldown_dir: RwLock<Option<std::path::PathBuf>>,
1612    /// Replaces the `.cds` files while set (Go's token-store cooldown provider).
1613    cooldown_backend: RwLock<Option<Arc<dyn crate::cooldown_store::Backend>>>,
1614    /// Serializes cooldown snapshots with their writes, so the last write is the newest.
1615    cooldown_write: Mutex<()>,
1616    activity: Mutex<HashMap<String, CredentialActivity>>,
1617    /// Credential revisions whose file already holds Go's persisted form.
1618    persisted: Mutex<HashMap<String, u64>>,
1619    /// What each file credential's metadata was built from, by ID ([`Provenance`]).
1620    provenance: Mutex<HashMap<String, Provenance>>,
1621    /// Where failed attempts publish Go's error events (set by the runtime).
1622    error_events: std::sync::OnceLock<Arc<crate::usage::UsageQueue>>,
1623    /// Marked on every epoch bump, so background loops sleep until the set changes.
1624    changed: tokio::sync::watch::Sender<()>,
1625    /// Codex credentials an upstream answered 401, and the last token rotations, for the
1626    /// background refresh (Go `RejectedAccessToken`, `scheduleUnauthorizedRefresh` and
1627    /// `LastRefreshedAt`).
1628    unauthorized: Mutex<Rejections>,
1629    unauthorized_noted: tokio::sync::watch::Sender<()>,
1630    /// Ends the upstream WebSocket sessions a removed credential serves (set by the
1631    /// runtime to [`cpa_exec::Executors::credential_session_closer`]).
1632    session_closer: std::sync::OnceLock<SessionCloser>,
1633    /// Go `LastRefreshedAt` and `NextRefreshAfter` of the credentials a plugin
1634    /// refreshes, registered with every change to the set
1635    /// ([`crate::plugins::refresh::register`]).
1636    refresh_schedules: crate::plugins::refresh::Schedules,
1637}
1638
1639/// Ends the upstream WebSocket sessions of one removed credential.
1640type SessionCloser = Box<dyn Fn(&str) + Send + Sync>;
1641
1642/// The ids in `before` that `after` no longer has.
1643fn removed_ids(before: &[Arc<Credential>], after: &[Arc<Credential>]) -> Vec<String> {
1644    let kept: std::collections::HashSet<&str> = after.iter().map(|c| c.id.as_str()).collect();
1645    before
1646        .iter()
1647        .filter(|c| !kept.contains(c.id.as_str()))
1648        .map(|c| c.id.clone())
1649        .collect()
1650}
1651
1652/// Whether `key` is set to `true` on `credential` (Go `IsPluginVirtualAuth`,
1653/// `isRuntimeOnlyAuth`).
1654fn marked(credential: &Credential, key: &str) -> bool {
1655    credential
1656        .attributes
1657        .get(key)
1658        .is_some_and(|v| v.trim().eq_ignore_ascii_case("true"))
1659}
1660
1661/// What a file credential's metadata was built from at one revision. Go holds a loaded
1662/// file's numbers as float64 read from its text, and a saved record's numbers as that
1663/// save held them; a management save reproduces them only from this, never from the
1664/// file as it is now, which may have changed since without a reload.
1665#[derive(Clone)]
1666pub struct Provenance {
1667    pub revision: u64,
1668    /// SHA-256 of the file bytes the revision was loaded from or committed as.
1669    pub sha256: [u8; 32],
1670    /// The document a management save committed (Go's metadata after that save); `None`
1671    /// for a load, where Go's metadata is the float64 reading of those bytes.
1672    pub doc: Option<Arc<cpa_common::json::GoValue>>,
1673    /// `doc` is the text of the bytes the record was loaded from (numbers as written,
1674    /// read as float64 by a save) with a refresh's values in place, not Go's metadata.
1675    pub loaded: bool,
1676}
1677
1678/// How [`CredentialStore::commit_file_backed`] commits a management save.
1679pub struct Commit {
1680    /// The bytes the file holds after the save (and the re-synthesized `next` read).
1681    pub bytes: Vec<u8>,
1682    /// `false` when the file already holds an equal document: nothing is written, but
1683    /// the credential is published from `bytes` all the same (Go's store skips the
1684    /// write, its manager still publishes the auth).
1685    pub write: bool,
1686    /// What the file must still hold under the write lock, else `Stale`: `Some(None)`
1687    /// for no file, `Some(Some(hash))` for those bytes; `None` skips the check.
1688    pub disk: Option<Option<[u8; 32]>>,
1689    /// Go's metadata after the save, recorded as the new revision's [`Provenance`].
1690    pub doc: Option<Arc<cpa_common::json::GoValue>>,
1691}
1692
1693/// A value a refresh set, as Go holds it (integers exact, other numbers float64), or
1694/// with `literal` its number text as written.
1695fn go_typed(v: &Value, literal: bool) -> cpa_common::json::GoValue {
1696    use cpa_common::json::GoValue;
1697    match v {
1698        Value::Null => GoValue::Null,
1699        Value::Bool(b) => GoValue::Bool(*b),
1700        Value::Number(n) if literal || n.is_i64() || n.is_u64() => GoValue::Number(n.to_string()),
1701        Value::Number(n) => {
1702            let text = n.to_string();
1703            GoValue::Number(
1704                cpa_common::json::json_float(cpa_common::json::parse_float(text.as_bytes())).unwrap_or(text),
1705            )
1706        }
1707        Value::String(s) => GoValue::String(s.clone()),
1708        Value::Array(items) => GoValue::Array(items.iter().map(|v| go_typed(v, literal)).collect()),
1709        Value::Object(map) => GoValue::Object(map.iter().map(|(k, v)| (k.clone(), go_typed(v, literal))).collect()),
1710    }
1711}
1712
1713fn sha256(bytes: &[u8]) -> [u8; 32] {
1714    use sha2::{Digest, Sha256};
1715    Sha256::digest(bytes).into()
1716}
1717
1718impl CredentialStore {
1719    pub fn new(credentials: Vec<Credential>) -> Arc<Self> {
1720        let mut inner = Inner {
1721            creds: Vec::new(),
1722            generation: 0,
1723            epoch: 0,
1724        };
1725        for mut cred in credentials {
1726            inner.generation += 1;
1727            cred.revision = inner.generation;
1728            inner.creds.push(Arc::new(cred));
1729        }
1730        let refresh_schedules = crate::plugins::refresh::Schedules::default();
1731        crate::plugins::refresh::register(&refresh_schedules, &[], &inner.creds, &[]);
1732        crate::plugins::refresh::begin_bootstrap(&refresh_schedules);
1733        Arc::new(Self {
1734            inner: RwLock::new(inner),
1735            policy: RwLock::new(Arc::new(Policy::default())),
1736            scheduler: Mutex::default(),
1737            attempts: AtomicU64::new(0),
1738            stats: Default::default(),
1739            prepare_locks: Mutex::default(),
1740            write_locks: Mutex::default(),
1741            reload: RwLock::default(),
1742            cooldown_dir: RwLock::default(),
1743            cooldown_backend: RwLock::default(),
1744            cooldown_write: Mutex::default(),
1745            activity: Mutex::default(),
1746            persisted: Mutex::default(),
1747            provenance: Mutex::default(),
1748            error_events: std::sync::OnceLock::new(),
1749            changed: tokio::sync::watch::Sender::new(()),
1750            unauthorized: Mutex::default(),
1751            unauthorized_noted: tokio::sync::watch::Sender::new(()),
1752            session_closer: std::sync::OnceLock::new(),
1753            refresh_schedules,
1754        })
1755    }
1756
1757    /// Credential `id` and what a refresh of it starts under: its registration and
1758    /// credential version ([`crate::plugins::refresh::Registered`]), read together.
1759    pub(crate) fn get_registered(&self, id: &str) -> Option<(Arc<Credential>, crate::plugins::refresh::Registered)> {
1760        let inner = self.read();
1761        let credential = inner.creds.iter().find(|c| c.id == id)?.clone();
1762        Some((credential, self.registered(id)))
1763    }
1764
1765    /// What a refresh of `id` starts under; zeros for none.
1766    fn registered(&self, id: &str) -> crate::plugins::refresh::Registered {
1767        crate::plugins::refresh::registered(&self.refresh_schedules, id).unwrap_or_default()
1768    }
1769
1770    /// Records a refresh outcome of credential `id` (Go reads `m.auths[id]` under
1771    /// `m.mu`): `record` sees the current credential and its schedule while it is under
1772    /// `registration` (`NotFound` otherwise) and, given `at`, still at that revision
1773    /// and credential version (`Stale` otherwise). No registration can land in
1774    /// between.
1775    pub(crate) fn record_refresh(
1776        &self,
1777        id: &str,
1778        registration: u64,
1779        at: Option<(u64, u64)>,
1780        record: impl FnOnce(&Credential, &mut crate::plugins::refresh::Schedule),
1781    ) -> Result<Arc<Credential>, PatchError> {
1782        let inner = self.read();
1783        let current = inner.creds.iter().find(|c| c.id == id).ok_or(PatchError::NotFound)?;
1784        let registered = self.registered(id);
1785        if registered.registration != registration {
1786            return Err(PatchError::NotFound);
1787        }
1788        if at.is_some_and(|at| at != (current.revision, registered.credential_version)) {
1789            return Err(PatchError::Stale {
1790                current: current.revision,
1791            });
1792        }
1793        crate::plugins::refresh::record(&self.refresh_schedules, current, record);
1794        Ok(current.clone())
1795    }
1796
1797    /// The plugin refresh schedules ([`crate::plugins::refresh::register`]).
1798    pub(crate) fn refresh_schedules(&self) -> &crate::plugins::refresh::Schedules {
1799        &self.refresh_schedules
1800    }
1801
1802    /// Ends the startup window in which the credentials loaded before plugins parse auth
1803    /// files are provisional ([`crate::plugins::refresh::end_bootstrap`]).
1804    pub fn end_bootstrap(&self) {
1805        if crate::plugins::refresh::end_bootstrap(&self.refresh_schedules) {
1806            // The refresh loop skips provisional schedules; it looks again.
1807            self.changed.send_replace(());
1808        }
1809    }
1810
1811    /// Registers the plugin refresh schedules of a change from `before` to `after`;
1812    /// called with the change, under the write lock.
1813    fn register(
1814        &self,
1815        before: &[Arc<Credential>],
1816        after: &[Arc<Credential>],
1817        saved: &[cpa_plugin::hostauth::SavedRefresh],
1818    ) {
1819        crate::plugins::refresh::register(&self.refresh_schedules, before, after, saved);
1820    }
1821
1822    /// Go `ApplyConfigWithCooldownStateStore` plus the restore that follows a config
1823    /// update: the current state goes to the old store before the swap, and a newly
1824    /// enabled store is restored from. `None` disables persistence.
1825    pub fn configure_cooldown_store(&self, dir: Option<std::path::PathBuf>) {
1826        let old = self.cooldown_dir.read().unwrap_or_else(PoisonError::into_inner).clone();
1827        if old == dir {
1828            return;
1829        }
1830        if old.is_some() {
1831            self.persist_cooldowns();
1832        }
1833        *self.cooldown_dir.write().unwrap_or_else(PoisonError::into_inner) = dir.clone();
1834        if let Some(dir) = dir {
1835            self.restore_cooldowns(&dir);
1836        }
1837    }
1838
1839    /// Go's builder taking the token store's `CooldownStateStore`: cooldowns go to
1840    /// `backend` instead of `.cds` files, restored from it now when persistence is on.
1841    pub fn set_cooldown_backend(&self, backend: Arc<dyn crate::cooldown_store::Backend>) {
1842        *self.cooldown_backend.write().unwrap_or_else(PoisonError::into_inner) = Some(backend);
1843        let dir = self.cooldown_dir.read().unwrap_or_else(PoisonError::into_inner).clone();
1844        if let Some(dir) = dir {
1845            self.restore_cooldowns(&dir);
1846        }
1847    }
1848
1849    fn cooldown_backend(&self) -> Option<Arc<dyn crate::cooldown_store::Backend>> {
1850        self.cooldown_backend
1851            .read()
1852            .unwrap_or_else(PoisonError::into_inner)
1853            .clone()
1854    }
1855
1856    /// Go `RestoreCooldownStates`: live records of live credentials whose cooling is
1857    /// enabled, then a rewrite that drops everything else.
1858    fn restore_cooldowns(&self, dir: &std::path::Path) {
1859        let loaded = match self.cooldown_backend() {
1860            Some(backend) => backend.load(),
1861            None => crate::cooldown_store::load(dir),
1862        };
1863        let records = match loaded {
1864            Ok(records) => records,
1865            Err(error) => {
1866                tracing::warn!(%error, "failed to restore cooldown state");
1867                return;
1868            }
1869        };
1870        if !records.is_empty() {
1871            let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
1872            let inner = self.read();
1873            let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
1874            let (now, wall) = (Instant::now(), std::time::SystemTime::now());
1875            for record in &records {
1876                let Some(c) = inner.creds.iter().find(|c| c.id == record.auth_id.trim()) else {
1877                    continue;
1878                };
1879                if c.disabled || policy.cooling_disabled(c) {
1880                    continue;
1881                }
1882                let has_models = records
1883                    .iter()
1884                    .any(|r| r.auth_id.trim() == c.id && !r.model.trim().is_empty());
1885                scheduler.restore(c, record, has_models, policy.max_trusted_cooldown, now, wall);
1886            }
1887        }
1888        self.persist_cooldowns();
1889    }
1890
1891    /// Go `persistCooldownStates`: rewrites the `.cds` files from the live cooldowns.
1892    pub fn persist_cooldowns(&self) {
1893        let Some(dir) = self.cooldown_dir.read().unwrap_or_else(PoisonError::into_inner).clone() else {
1894            return;
1895        };
1896        let _write = self.cooldown_write.lock().unwrap_or_else(PoisonError::into_inner);
1897        let wall = std::time::SystemTime::now();
1898        let mut records = {
1899            let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
1900            let inner = self.read();
1901            let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
1902            let now = Instant::now();
1903            inner
1904                .creds
1905                .iter()
1906                .filter(|c| !c.disabled && !policy.cooling_disabled(c))
1907                .flat_map(|c| scheduler.records(c, now, wall))
1908                .collect::<Vec<_>>()
1909        };
1910        records.sort_by(|a, b| (&a.provider, &a.auth_id, &a.model).cmp(&(&b.provider, &b.auth_id, &b.model)));
1911        let saved = match self.cooldown_backend() {
1912            Some(backend) => backend.save(records, wall),
1913            None => crate::cooldown_store::save(&dir, records, wall),
1914        };
1915        if let Err(error) = saved {
1916            tracing::warn!(%error, "failed to persist cooldown state");
1917        }
1918    }
1919
1920    /// Go `Manager.persist` after a result (`FileTokenStore.Save`, metadata branch): the
1921    /// file gets `"disabled"` and Go's `json.Marshal` form unless it already holds the
1922    /// same JSON. Checked once per revision, so a file is not read on every request.
1923    // ponytail: atomic replace instead of Go's in-place truncate, and a removed file is
1924    // not recreated (Go recreates one deleted between watcher reloads).
1925    fn persist_credential(&self, credential: &Arc<Credential>) {
1926        let Source::File(path) = &credential.source else {
1927            return;
1928        };
1929        // Go also skips runtime-only credentials and plugin-virtual ones.
1930        if credential.disabled || marked(credential, "runtime_only") || marked(credential, "plugin_virtual") {
1931            return;
1932        }
1933        {
1934            let persisted = self.persisted.lock().unwrap_or_else(PoisonError::into_inner);
1935            if persisted.get(&credential.id) == Some(&credential.revision) {
1936                return;
1937            }
1938        }
1939        // Written only while the credential is still at this revision: a refresh committed
1940        // after this request started must not be overwritten with the older token.
1941        let written = self.write_unlocked(
1942            &credential.id,
1943            credential.revision,
1944            None,
1945            false,
1946            |current| {
1947                let mut metadata = current.metadata.clone();
1948                metadata.insert("disabled".into(), Value::Bool(current.disabled));
1949                let go = |bytes: &[u8]| cpa_common::json::GoValue::parse_f64(bytes);
1950                // An unreadable file or metadata is skipped (`NotFound`), not reported.
1951                let Some(target) = serde_json::to_vec(&metadata).ok().and_then(|b| go(&b)) else {
1952                    return Err(PatchError::NotFound);
1953                };
1954                let existing = std::fs::read(path).map_err(|_| PatchError::NotFound)?;
1955                if go(&existing).as_ref() == Some(&target) {
1956                    return Ok(None);
1957                }
1958                Ok(Some((Some(path.clone()), target.marshal(), ())))
1959            },
1960            |inner, index, bytes, ()| {
1961                // Same revision: the file now says what memory already meant (absent
1962                // `disabled` is false, numbers are float64), so memory takes the written
1963                // form and the watcher's reload finds nothing changed.
1964                if let Ok(written) = serde_json::from_slice::<Map<String, Value>>(bytes) {
1965                    let mut next = Credential::clone(&inner.creds[index]);
1966                    next.metadata = written;
1967                    inner.creds[index] = Arc::new(next);
1968                }
1969                // A save's document stays Go's metadata, now held by these bytes. A load's
1970                // float64 reading is not provably these bytes' (they come from serde's
1971                // values), so it is forgotten.
1972                let id = &inner.creds[index].id;
1973                let mut provenance = self.provenance.lock().unwrap_or_else(PoisonError::into_inner);
1974                match provenance.get_mut(id) {
1975                    Some(p) if p.doc.is_some() => p.sha256 = sha256(bytes),
1976                    Some(_) => {
1977                        provenance.remove(id);
1978                    }
1979                    None => {}
1980                }
1981            },
1982        );
1983        match written {
1984            Ok(_) => {}
1985            Err(PatchError::Io(error)) => {
1986                tracing::warn!(id = %credential.id, %error, "failed to persist credential");
1987                return;
1988            }
1989            Err(_) => return,
1990        }
1991        self.persisted
1992            .lock()
1993            .unwrap_or_else(PoisonError::into_inner)
1994            .insert(credential.id.clone(), credential.revision);
1995    }
1996
1997    /// Go `updateInternalLocked` for a credential a reload, save, import or upsert
1998    /// replaced with other tokens or another API key (`CredentialsChanged`), enabled
1999    /// before and after: its unauthorized cooldowns go, so the new token is selectable
2000    /// at once, and every other cooldown (a model's 429, a live credential quota) stays.
2001    /// Called with the change, under the write lock; returns whether anything was
2002    /// cleared, for the caller to persist.
2003    fn clear_replaced_unauthorized(&self, before: &[Arc<Credential>], after: &[Arc<Credential>]) -> bool {
2004        let replaced: Vec<&str> = after
2005            .iter()
2006            .filter(|next| {
2007                before
2008                    .iter()
2009                    .find(|c| c.id == next.id)
2010                    .is_some_and(|previous| !previous.disabled && !next.disabled && credentials_changed(previous, next))
2011            })
2012            .map(|c| c.id.as_str())
2013            .collect();
2014        if replaced.is_empty() {
2015            return false;
2016        }
2017        let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2018        let now = Instant::now();
2019        let mut cleared = false;
2020        for id in replaced {
2021            cleared |= scheduler.clear_unauthorized(id, now);
2022        }
2023        cleared
2024    }
2025
2026    /// Go `clearDisabledCooldownStates` after a policy change.
2027    fn clear_disabled_cooldowns(&self) {
2028        let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
2029        let cleared = {
2030            let inner = self.read();
2031            self.scheduler
2032                .lock()
2033                .unwrap_or_else(PoisonError::into_inner)
2034                .clear_disabled(&inner.creds, &policy)
2035        };
2036        if cleared {
2037            self.persist_cooldowns();
2038        }
2039    }
2040
2041    pub fn snapshot(&self) -> Vec<Arc<Credential>> {
2042        self.read().creds.clone()
2043    }
2044
2045    pub fn get(&self, id: &str) -> Option<Arc<Credential>> {
2046        self.read().creds.iter().find(|c| c.id == id).cloned()
2047    }
2048
2049    /// Selects among the selection's providers without registry admission (tests and
2050    /// callers that already filtered credentials).
2051    pub fn select(self: &Arc<Self>, selection: Selection) -> Option<Lease> {
2052        let policy = self.policy.read().unwrap_or_else(PoisonError::into_inner).clone();
2053        let scope = selection.clone();
2054        let admit = standalone_admission(&scope, &policy);
2055        self.select_with(selection, policy.clone(), &admit).ok()
2056    }
2057
2058    pub fn select_with(
2059        self: &Arc<Self>,
2060        selection: Selection,
2061        policy: Arc<Policy>,
2062        admit: &Admit<'_>,
2063    ) -> Result<Lease, AcquireError> {
2064        self.select_ranked(selection, policy, admit, &|_| crate::scheduler::Windows::default())
2065    }
2066
2067    /// [`Self::select_with`] with each credential's usage windows (`soonest-reset`).
2068    pub fn select_ranked(
2069        self: &Arc<Self>,
2070        selection: Selection,
2071        policy: Arc<Policy>,
2072        admit: &Admit<'_>,
2073        ranks: &crate::scheduler::Ranks<'_>,
2074    ) -> Result<Lease, AcquireError> {
2075        self.select_picked(selection, policy, admit, ranks, Pick::Configured)
2076    }
2077
2078    /// The ready candidates a plugin scheduler chooses among (Go
2079    /// `availableAuthsForSelector`'s priority list): the highest priority tier, or every
2080    /// tier when the scheduler asks for them.
2081    pub(crate) fn scheduler_candidates(
2082        &self,
2083        selection: &Selection,
2084        policy: &Policy,
2085        admit: &Admit<'_>,
2086        across: bool,
2087    ) -> Result<Vec<Arc<Credential>>, AcquireError> {
2088        let inner = self.read();
2089        let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2090        let (_, ready) = ready(
2091            &inner.creds,
2092            &scheduler,
2093            selection,
2094            policy,
2095            admit,
2096            Instant::now(),
2097            false,
2098        )?;
2099        let tier = ready.iter().map(|(c, _)| crate::scheduler::priority(c)).max();
2100        let mut out: Vec<_> = ready
2101            .into_iter()
2102            .filter(|(c, _)| across || Some(crate::scheduler::priority(c)) == tier)
2103            .map(|(c, _)| Arc::clone(c))
2104            .collect();
2105        // Go `availableAuthsFromPriorityBuckets` orders both modes by ID.
2106        out.sort_by(|a, b| a.id.cmp(&b.id));
2107        Ok(out)
2108    }
2109
2110    /// [`Self::select_ranked`] with the choice a plugin scheduler made.
2111    pub(crate) fn select_picked(
2112        self: &Arc<Self>,
2113        selection: Selection,
2114        policy: Arc<Policy>,
2115        admit: &Admit<'_>,
2116        ranks: &crate::scheduler::Ranks<'_>,
2117        pick: Pick,
2118    ) -> Result<Lease, AcquireError> {
2119        let now = Instant::now();
2120        let credential = {
2121            let inner = self.read();
2122            let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2123            let configured = matches!(pick, Pick::Configured);
2124            let (eligible, candidates) = ready(&inner.creds, &scheduler, &selection, &policy, admit, now, configured)?;
2125            let refs: Vec<(&Credential, &str)> = candidates.iter().map(|(c, p)| (c.as_ref(), p.as_str())).collect();
2126            let chosen = match &pick {
2127                // Go serves the candidate from its pre-call snapshot without checking it
2128                // again; here one that stopped being ready during the call leaves the
2129                // choice to the configured selector (docs/DIFFERENCES-FROM-GO.md).
2130                Pick::Auth(id) => refs.iter().find(|(c, _)| c.id == *id).map(|(c, _)| (*c, None)),
2131                Pick::Builtin(strategy) => scheduler
2132                    .pick_builtin(&refs, &selection, *strategy, ranks)
2133                    .map(|c| (c, None)),
2134                Pick::Configured => None,
2135            };
2136            let (picked, lcp) = match chosen {
2137                Some(chosen) => chosen,
2138                None => {
2139                    let refs: Vec<_> = refs.into_iter().filter(|(c, _)| scheduler.admits(c, &policy)).collect();
2140                    if refs.is_empty() {
2141                        // Only the configured selector's filter removed them: its error.
2142                        return Err(ready(&inner.creds, &scheduler, &selection, &policy, admit, now, true)
2143                            .expect_err("no admitted candidate is ready"));
2144                    }
2145                    scheduler.pick_session(&refs, &selection, &policy, ranks, now).unwrap()
2146                }
2147            };
2148            let (c, m) = eligible.iter().find(|(c, _)| c.id == picked.id).unwrap();
2149            let generation = scheduler.reserve_probe(c, m, now);
2150            (Arc::clone(c), lcp, generation, scheduler.dispatch())
2151        };
2152        let (credential, lcp, picked, seq) = credential;
2153        let execution_model = admit(&credential).unwrap_or_else(|| selection.model.clone());
2154        Ok(Lease {
2155            store: self.clone(),
2156            credential,
2157            selection,
2158            execution_model,
2159            attempt: self.attempts.fetch_add(1, Ordering::Relaxed),
2160            policy,
2161            reported: false,
2162            picked,
2163            seq,
2164            remote: None,
2165            lcp,
2166        })
2167    }
2168
2169    /// The registry projection of this credential's cooldown state for `model`.
2170    pub fn suspension(&self, credential: &Credential, model: &str) -> crate::registry::Suspension {
2171        use crate::registry::Suspension;
2172        let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2173        let now = Instant::now();
2174        let live = |m: &str| {
2175            scheduler
2176                .cooldowns
2177                .get(&(credential.id.clone(), crate::scheduler::canonical_model(m).to_owned()))
2178                .filter(|s| s.deadline > now)
2179        };
2180        if let Some(state) = live(model) {
2181            return match (state.quota, state.status) {
2182                (true, 429) => Suspension::Quota,
2183                (quota_exceeded, _) => Suspension::Other { quota_exceeded },
2184            };
2185        }
2186        if live("").is_some() {
2187            // Credential-wide quota: Go reports these models as `credential_quota`.
2188            return Suspension::Other { quota_exceeded: false };
2189        }
2190        Suspension::None
2191    }
2192
2193    /// Whether `model` is cooling for this credential right now, or its tokens were
2194    /// refused for good. A probe reservation does not count: the lease that holds it is
2195    /// the one asking.
2196    pub fn blocked(&self, credential: &Credential, model: &str) -> bool {
2197        let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2198        scheduler.cooling(credential, model, Instant::now()) || scheduler.tokens_rejected(credential)
2199    }
2200
2201    /// Records that `refused`, the credential as an upstream refused it with a 401, can no
2202    /// longer recover, when `error` (its refresh's) is invalid_grant and the store still
2203    /// holds it with the same tokens and a non-empty access token (Go
2204    /// `accessTokenRejected`), under the registration the refresh started under
2205    /// (`registration`, Go `RegistrationEpoch`), so a credential removed and imported
2206    /// again while the exchange ran is not rejected. Checked and recorded under the
2207    /// store's read lock, so a replacement or removal lands entirely before or after.
2208    pub(crate) fn reject_refused(&self, refused: &Credential, registration: u64, error: &ExecError) -> bool {
2209        if !crate::classify::is_invalid_grant(error.status, &String::from_utf8_lossy(&error.body)) {
2210            return false;
2211        }
2212        let inner = self.read();
2213        let Some(current) = inner.creds.iter().find(|c| c.id == refused.id) else {
2214            return false;
2215        };
2216        let token = effective_access_token(refused);
2217        if current.disabled
2218            || self.registered(&current.id).registration != registration
2219            || token.is_empty()
2220            || effective_access_token(current) != token
2221            || credentials_changed(refused, current)
2222        {
2223            return false;
2224        }
2225        self.scheduler
2226            .lock()
2227            .unwrap_or_else(PoisonError::into_inner)
2228            .reject_tokens(current.clone());
2229        true
2230    }
2231
2232    /// Whether `c`'s tokens were refused for good ([`crate::scheduler::Scheduler::tokens_rejected`]).
2233    pub fn tokens_rejected(&self, c: &Credential) -> bool {
2234        self.scheduler
2235            .lock()
2236            .unwrap_or_else(PoisonError::into_inner)
2237            .tokens_rejected(c)
2238    }
2239
2240    /// The IDs in `snapshot` whose tokens were refused for good.
2241    fn rejected_ids(&self, snapshot: &[Arc<Credential>]) -> std::collections::HashSet<String> {
2242        let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2243        snapshot
2244            .iter()
2245            .filter(|c| scheduler.tokens_rejected(c))
2246            .map(|c| c.id.clone())
2247            .collect()
2248    }
2249
2250    /// Clears every cooldown and affinity binding of one credential (Go `ResetQuota`).
2251    /// Returns the model keys that were cooling.
2252    pub fn reset_cooldown(&self, id: &str) -> Vec<String> {
2253        let models = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner).reset(id);
2254        if !models.is_empty() {
2255            self.persist_cooldowns();
2256        }
2257        models
2258    }
2259
2260    /// Returns the next round's wait, if any credential still permits that round
2261    /// (Go `closestCooldownWaitWithAttempted`). A wait exceeding the cap is rejected,
2262    /// not shortened. `status` is the round error's status; `attempted` the credentials
2263    /// that executed this round (they get the 10s quota floor after a 429).
2264    pub fn retry_wait(
2265        &self,
2266        selection: &Selection,
2267        policy: &Policy,
2268        status: u16,
2269        attempted: &[String],
2270        admit: &Admit<'_>,
2271    ) -> Option<Duration> {
2272        self.retry_wait_at(selection, policy, status, attempted, admit, Instant::now())
2273    }
2274
2275    fn retry_wait_at(
2276        &self,
2277        selection: &Selection,
2278        policy: &Policy,
2279        status: u16,
2280        attempted: &[String],
2281        admit: &Admit<'_>,
2282        now: Instant,
2283    ) -> Option<Duration> {
2284        let inner = self.read();
2285        let scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2286        let wait = inner
2287            .creds
2288            .iter()
2289            .filter(|c| !c.disabled)
2290            .filter(|c| policy.retry_limit(c) > selection.retry_round && scheduler.admits(c, policy))
2291            .filter_map(|c| admit(c).map(|m| (c, m)))
2292            .filter(|(c, m)| scheduler.retry_eligible(c, m, now) && !scheduler.tokens_rejected(c))
2293            .map(|(c, m)| {
2294                let wait = scheduler.wait(c, &m, now).unwrap_or_default();
2295                if status == 429 && attempted.contains(&c.id) && !policy.cooling_disabled(c) {
2296                    wait.max(Duration::from_secs(10))
2297                } else {
2298                    wait
2299                }
2300            })
2301            .min()?;
2302        if wait > policy.max_retry_interval {
2303            None
2304        } else {
2305            Some(wait)
2306        }
2307    }
2308
2309    fn accept_probe(&self, lease: &Lease) {
2310        let inner = self.read();
2311        // Like results: a credential edited or re-created since the pick is left alone.
2312        if !inner
2313            .creds
2314            .iter()
2315            .any(|c| c.id == lease.credential.id && c.revision == lease.credential.revision)
2316        {
2317            return;
2318        }
2319        self.scheduler
2320            .lock()
2321            .unwrap_or_else(PoisonError::into_inner)
2322            .accept_probe_picked(&lease.credential, &lease.execution_model, lease.picked, Instant::now());
2323    }
2324
2325    fn record(&self, lease: &Lease, outcome: &Outcome) {
2326        self.record_model(lease, &lease.execution_model, outcome);
2327    }
2328
2329    fn record_model(&self, lease: &Lease, model: &str, outcome: &Outcome) {
2330        let slot = match outcome {
2331            Outcome::Success => 0,
2332            Outcome::Failure(_) | Outcome::Neutral(_) => 1,
2333            Outcome::Cancelled => 2,
2334        };
2335        self.stats[slot].fetch_add(1, Ordering::Relaxed);
2336        if !matches!(outcome, Outcome::Cancelled) {
2337            self.note_activity(&lease.credential.id, matches!(outcome, Outcome::Success));
2338        }
2339        let inner = self.read();
2340        // Outcomes from credentials deleted/re-created during an attempt must not
2341        // poison the replacement. Metadata edits likewise invalidate stale results.
2342        if inner
2343            .creds
2344            .iter()
2345            .any(|c| c.id == lease.credential.id && c.revision == lease.credential.revision)
2346        {
2347            let (now, wall) = (Instant::now(), std::time::SystemTime::now());
2348            let persist = self
2349                .cooldown_dir
2350                .read()
2351                .unwrap_or_else(PoisonError::into_inner)
2352                .is_some();
2353            let mut scheduler = self.scheduler.lock().unwrap_or_else(PoisonError::into_inner);
2354            let before = persist.then(|| scheduler.records(&lease.credential, now, wall));
2355            scheduler.record_dispatched(
2356                &lease.credential,
2357                model,
2358                outcome,
2359                &lease.policy,
2360                lease.picked,
2361                lease.seq,
2362                now,
2363            );
2364            scheduler.session_result(
2365                &lease.credential,
2366                &lease.selection,
2367                outcome,
2368                &lease.policy,
2369                lease.lcp.as_ref(),
2370                now,
2371            );
2372            // Go MarkResult: a 401 that cools the credential schedules a background
2373            // refresh of the rejected token (feb63984; the runtime checks for a refresh
2374            // credential). A forced cooldown counts even on a request-scoped 401.
2375            // ponytail: Codex only for now; other providers (Claude's included) still
2376            // refresh after a 401 inline only, until the post-replay port.
2377            let rejected = matches!(outcome, Outcome::Failure(error)
2378                if lease.credential.provider == "codex"
2379                    && crate::classify::names_unauthorized(error)
2380                    && lease.policy.error_action(&lease.credential, error).cooldown);
2381            // Go `publishErrorEvent` after MarkResult (and the availability-neutral record).
2382            if let (Outcome::Failure(error) | Outcome::Neutral(error), Some(queue)) = (outcome, self.error_events.get())
2383                && queue.wants_errors()
2384            {
2385                let records = scheduler.records(&lease.credential, now, wall);
2386                queue.enqueue_error(&crate::error_events::payload(&lease.credential, model, error, &records));
2387            }
2388            // Go MarkResult persists only when this credential's cooldown records changed.
2389            let changed = before.is_some_and(|before| before != scheduler.records(&lease.credential, now, wall));
2390            drop(scheduler);
2391            drop(inner);
2392            // After the store lock: a refresh commit holds this lock while it writes.
2393            if rejected {
2394                let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2395                let revision = lease.credential.revision;
2396                // Go sets `NextRefreshAfter` only when none is pending.
2397                if state
2398                    .pending
2399                    .get(&lease.credential.id)
2400                    .is_none_or(|(r, _)| *r != revision)
2401                {
2402                    state.pending.insert(lease.credential.id.clone(), (revision, now));
2403                    self.unauthorized_noted.send_replace(());
2404                }
2405            }
2406            if changed {
2407                self.persist_cooldowns();
2408            }
2409            // Go persists on every recorded result, a client-cancelled stream included.
2410            self.persist_credential(&lease.credential);
2411        }
2412    }
2413
2414    /// Resolves after the next change to the credential set or any credential.
2415    pub fn subscribe(&self) -> tokio::sync::watch::Receiver<()> {
2416        self.changed.subscribe()
2417    }
2418
2419    /// Pending 401 refreshes and the last rotations. Read it before the credential
2420    /// snapshot it is compared with.
2421    pub(crate) fn unauthorized(&self) -> Rejections {
2422        self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner).clone()
2423    }
2424
2425    /// Forgets a pending unauthorized refresh unless a 401 on a newer revision replaced it.
2426    pub(crate) fn forget_unauthorized(&self, id: &str, revision: u64) {
2427        let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2428        if state.pending.get(id).is_some_and(|(r, _)| *r <= revision) {
2429            state.pending.remove(id);
2430        }
2431    }
2432
2433    /// Drops the rejections `seen` (read before `snapshot`) whose credential the snapshot
2434    /// replaced or removed. A 401 recorded after `seen` is never touched.
2435    pub(crate) fn forget_stale_unauthorized(&self, seen: &Rejections, snapshot: &[Arc<Credential>]) {
2436        for (id, (revision, _)) in &seen.pending {
2437            if snapshot
2438                .iter()
2439                .find(|c| c.id == *id)
2440                .is_none_or(|c| c.revision > *revision)
2441            {
2442                self.forget_unauthorized(id, *revision);
2443            }
2444        }
2445        let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2446        state
2447            .refreshed
2448            .retain(|id, _| !seen.refreshed.contains_key(id) || snapshot.iter().any(|c| c.id == *id));
2449    }
2450
2451    /// [`Self::apply_patch`] for a token refresh that started under `fence`'s
2452    /// registration: refused (`NotFound`) once the credential's ID was registered again
2453    /// (Go's `RegistrationEpoch` check), and a plugin refresh's schedule is recorded
2454    /// with the commit ([`crate::plugins::refresh::committed`]). When `rotated` says the
2455    /// commit replaced the tokens, its time is recorded under the same lock that 401s
2456    /// are noted under, before any scan can see the new revision without it (Go
2457    /// `LastRefreshedAt`).
2458    pub(crate) fn apply_refresh(
2459        &self,
2460        id: &str,
2461        expected_revision: u64,
2462        fence: RefreshFence,
2463        patch: &MetadataPatch,
2464        edit: impl FnOnce(&mut Credential),
2465        rotated: impl FnOnce(&Credential) -> bool,
2466    ) -> Result<(Arc<Credential>, bool), PatchError> {
2467        let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2468        let committed = self.apply_patch_with(id, expected_revision, Some(fence), patch, edit)?;
2469        let rotated = rotated(&committed);
2470        if rotated {
2471            state
2472                .refreshed
2473                .insert(id.to_owned(), (committed.revision, Instant::now()));
2474        }
2475        Ok((committed, rotated))
2476    }
2477
2478    /// Go `updateInternal` for a Codex refresh of `original`, run while the caller holds
2479    /// the file's credential lock. Refused (`NotFound`) once the ID was registered again since
2480    /// `registered`. When the credential version moved or the credentials changed since
2481    /// `original` and no refreshed partition still holds a version `patch` may be applied
2482    /// over (Go `CanApplyTo`), the refresh is discarded with no side effect. Otherwise
2483    /// `patch` is applied to the file as it is now, written with legacy keys renamed (Go
2484    /// `PersistCodexAuthFile`), and that document published as the credential's
2485    /// metadata, with its proxy, its time is recorded as [`Self::apply_refresh`] records it,
2486    /// and new tokens clear unauthorized cooldowns. Blocking.
2487    fn admit_codex_refresh(
2488        &self,
2489        original: &Credential,
2490        registered: crate::plugins::refresh::Registered,
2491        patch: &cpa_exec::codex_refresh::RefreshPatch,
2492    ) -> Result<CodexAdmission, PatchError> {
2493        let id = original.id.as_str();
2494        let committed = {
2495            let mut state = self.unauthorized.lock().unwrap_or_else(PoisonError::into_inner);
2496            let committed = loop {
2497                let revision = self.get(id).ok_or(PatchError::NotFound)?.revision;
2498                let mut discarded = None;
2499                // The registration and version are checked here, not by the writer: a
2500                // moved version is a discard decision, not an edit to retry on. The
2501                // writer's gate holds off registrations, its lock this credential's writes.
2502                let admitted = self.write_unlocked(
2503                    id,
2504                    revision,
2505                    None,
2506                    false,
2507                    |current| {
2508                        let now = self.registered(id);
2509                        if now.registration != registered.registration {
2510                            return Err(PatchError::NotFound);
2511                        }
2512                        let moved = now.credential_version != registered.credential_version;
2513                        if (moved || credentials_changed(original, current)) && !patch.can_apply_to(&current.metadata) {
2514                            discarded = Some(current.clone());
2515                            return Ok(None);
2516                        }
2517                        let Source::File(path) = &current.source else {
2518                            return Err(PatchError::ConfigBacked);
2519                        };
2520                        let (out, written) = save_codex_refresh(path, current, patch);
2521                        let Ok(Value::Object(metadata)) = serde_json::from_slice::<Value>(&out) else {
2522                            unreachable!("Go's encoding of a JSON object");
2523                        };
2524                        let mut next = Credential::clone(current);
2525                        next.metadata = metadata;
2526                        next.refresh_derived();
2527                        publish_metadata_proxy(&mut next);
2528                        Ok(Some((None, out, (next, written))))
2529                    },
2530                    |inner, index, bytes, (mut next, written)| {
2531                        inner.generation += 1;
2532                        next.revision = inner.generation;
2533                        // The file holds Go's encoding of its own float64 reading; after a
2534                        // failed save it holds something else.
2535                        if written {
2536                            self.note_provenance(&next.id, next.revision, sha256(bytes), None);
2537                        } else {
2538                            self.provenance
2539                                .lock()
2540                                .unwrap_or_else(PoisonError::into_inner)
2541                                .remove(&next.id);
2542                        }
2543                        let next = Arc::new(next);
2544                        self.register(
2545                            std::slice::from_ref(&inner.creds[index]),
2546                            std::slice::from_ref(&next),
2547                            &[],
2548                        );
2549                        inner.creds[index] = next;
2550                        inner.epoch += 1;
2551                        self.changed.send_replace(());
2552                        inner.creds[index].clone()
2553                    },
2554                );
2555                match admitted {
2556                    // Edited in memory meanwhile: decide again on the edit.
2557                    Err(PatchError::Stale { .. }) => continue,
2558                    Ok(None) => {
2559                        let current = discarded.expect("only a discard writes nothing");
2560                        return Ok(CodexAdmission::Discarded(current));
2561                    }
2562                    other => break other?.expect("checked above"),
2563                }
2564            };
2565            if credentials_changed(original, &committed) {
2566                state
2567                    .refreshed
2568                    .insert(id.to_owned(), (committed.revision, Instant::now()));
2569            }
2570            committed
2571        };
2572        if credentials_changed(original, &committed) {
2573            self.recover_unauthorized(id);
2574        }
2575        Ok(CodexAdmission::Committed(committed))
2576    }
2577
2578    /// Changes whenever a 401 schedules a background refresh.
2579    pub(crate) fn subscribe_unauthorized(&self) -> tokio::sync::watch::Receiver<()> {
2580        self.unauthorized_noted.subscribe()
2581    }
2582
2583    /// Changes whenever the credential set or any credential changes.
2584    pub fn epoch(&self) -> u64 {
2585        self.read().epoch
2586    }
2587
2588    /// Attempts finished so far, by outcome.
2589    pub fn stats(&self) -> AttemptStats {
2590        let get = |i: usize| self.stats[i].load(Ordering::Relaxed);
2591        AttemptStats {
2592            success: get(0),
2593            failure: get(1),
2594            cancelled: get(2),
2595        }
2596    }
2597
2598    /// Serializes preparation of one credential across concurrent requests.
2599    pub fn prepare_lock(&self, id: &str) -> Arc<tokio::sync::Mutex<()>> {
2600        let mut locks = self.prepare_locks.lock().unwrap_or_else(PoisonError::into_inner);
2601        locks.entry(id.to_owned()).or_default().clone()
2602    }
2603
2604    /// Applies a metadata change if the credential is still at `expected_revision`.
2605    /// File-backed credentials are written atomically (0600) before memory changes.
2606    /// Blocking: call from `spawn_blocking` in async code.
2607    pub fn apply_patch(
2608        &self,
2609        id: &str,
2610        expected_revision: u64,
2611        patch: &MetadataPatch,
2612    ) -> Result<Arc<Credential>, PatchError> {
2613        self.apply_patch_with(id, expected_revision, None, patch, |_| {})
2614    }
2615
2616    /// [`Self::apply_patch`], with `edit` applied to the credential before it is
2617    /// published (fields outside the document, such as a plugin refresh's proxy URL),
2618    /// and a token refresh's `fence` ([`Self::apply_refresh`]).
2619    fn apply_patch_with(
2620        &self,
2621        id: &str,
2622        expected_revision: u64,
2623        fence: Option<RefreshFence>,
2624        patch: &MetadataPatch,
2625        edit: impl FnOnce(&mut Credential),
2626    ) -> Result<Arc<Credential>, PatchError> {
2627        let committed = self.write_unlocked(
2628            id,
2629            expected_revision,
2630            fence.map(|f| f.registered),
2631            false,
2632            |current| {
2633                if patch.remove.iter().any(|k| k == "type")
2634                    || patch
2635                        .set
2636                        .get("type")
2637                        .is_some_and(|t| t.as_str() != Some(current.provider.as_str()))
2638                {
2639                    return Err(PatchError::TypeIsImmutable);
2640                }
2641                let Source::File(path) = &current.source else {
2642                    return Err(PatchError::ConfigBacked);
2643                };
2644                let mut next = Credential::clone(current);
2645                patch.apply(&mut next.metadata);
2646                next.refresh_derived();
2647                edit(&mut next);
2648                // Go `FileTokenStore.Save` always records the disabled state.
2649                next.metadata.insert("disabled".into(), Value::Bool(next.disabled));
2650                // Go's `json.NewEncoder(f).Encode`: compact JSON and a newline.
2651                let mut bytes = serde_json::to_vec(&next.metadata).map_err(|e| PatchError::Io(e.to_string()))?;
2652                bytes.push(b'\n');
2653                let basis = self.patched_doc(current, path, patch, next.disabled);
2654                Ok(Some((Some(path.clone()), bytes, (next, basis))))
2655            },
2656            |inner, index, bytes, (mut next, basis)| {
2657                inner.generation += 1;
2658                next.revision = inner.generation;
2659                // The written bytes are serde's, not Go's: a later management save takes
2660                // its numbers from `doc`, Go's metadata after the refresh, or refuses.
2661                match basis {
2662                    Some((doc, loaded)) => {
2663                        let provenance = Provenance {
2664                            revision: next.revision,
2665                            sha256: sha256(bytes),
2666                            doc: Some(Arc::new(doc)),
2667                            loaded,
2668                        };
2669                        self.provenance
2670                            .lock()
2671                            .unwrap_or_else(PoisonError::into_inner)
2672                            .insert(next.id.clone(), provenance);
2673                    }
2674                    None => {
2675                        self.provenance
2676                            .lock()
2677                            .unwrap_or_else(PoisonError::into_inner)
2678                            .remove(&next.id);
2679                    }
2680                }
2681                let next = Arc::new(next);
2682                self.register(
2683                    std::slice::from_ref(&inner.creds[index]),
2684                    std::slice::from_ref(&next),
2685                    &[],
2686                );
2687                // Go publishes `LastRefreshedAt` with the refreshed auth (`MergeRefreshedAuth`
2688                // under `m.mu`), so a registration after this commit replaces it. The
2689                // writer checked the refresh's registration under the gate. Lock order:
2690                // `inner`, then the schedules, as in `register`.
2691                if fence.is_some_and(|f| f.plugin) {
2692                    crate::plugins::refresh::record(&self.refresh_schedules, &next, |c, s| {
2693                        crate::plugins::refresh::committed(c, s, chrono::Utc::now())
2694                    });
2695                }
2696                inner.creds[index] = next;
2697                inner.epoch += 1;
2698                self.changed.send_replace(());
2699                inner.creds[index].clone()
2700            },
2701        )?;
2702        Ok(committed.expect("apply_patch always writes"))
2703    }
2704
2705    /// Go 8a01bc02 (`persistLocked`): writes a file-backed credential with `inner`
2706    /// unlocked, so a slow disk never stalls selection or reads of other credentials.
2707    /// `prepare` sees the credential at `expected_revision` (`Stale` otherwise) and
2708    /// returns the file to write (`None` to write nothing) and the bytes, or `None` for
2709    /// nothing to do; `publish` then
2710    /// puts the result in place under the write lock. The credential's write lock and
2711    /// the reload gate (shared) are held throughout, so neither another write of it nor
2712    /// a set replacement can land between the write and its publication. A publication
2713    /// that touches another credential (an identity change) passes `exclusive` and holds
2714    /// the gate exclusively instead, so no write of any credential is in flight. A token
2715    /// refresh passes what it started under: it is refused (`NotFound`) once the
2716    /// credential's ID was registered again, before anything is merged, and meets a
2717    /// credential version moved since as an edit (`Stale`).
2718    fn write_unlocked<T, R>(
2719        &self,
2720        id: &str,
2721        expected_revision: u64,
2722        registered: Option<crate::plugins::refresh::Registered>,
2723        exclusive: bool,
2724        prepare: impl FnOnce(&Arc<Credential>) -> Result<Option<(Option<std::path::PathBuf>, Vec<u8>, T)>, PatchError>,
2725        publish: impl FnOnce(&mut Inner, usize, &[u8], T) -> R,
2726    ) -> Result<Option<R>, PatchError> {
2727        // Acquired once, before the check and the write: re-taking the gate while holding
2728        // it can deadlock behind a waiting reload.
2729        let lock = (!exclusive).then(|| {
2730            let mut locks = self.write_locks.lock().unwrap_or_else(PoisonError::into_inner);
2731            locks.entry(id.to_owned()).or_default().clone()
2732        });
2733        let _gate = if exclusive {
2734            (None, Some(self.reload.write().unwrap_or_else(PoisonError::into_inner)))
2735        } else {
2736            (Some(self.reload.read().unwrap_or_else(PoisonError::into_inner)), None)
2737        };
2738        let _own = lock.as_ref().map(|l| l.lock().unwrap_or_else(PoisonError::into_inner));
2739        let current = self
2740            .read()
2741            .creds
2742            .iter()
2743            .find(|c| c.id == id)
2744            .cloned()
2745            .ok_or(PatchError::NotFound)?;
2746        // Go `updateInternal`: "stale registration epoch" for a refresh, ahead of the
2747        // revision check. The gate holds off every set replacement.
2748        if let Some(registered) = registered {
2749            let now = self.registered(id);
2750            if now.registration != registered.registration {
2751                return Err(PatchError::NotFound);
2752            }
2753            if now.credential_version != registered.credential_version {
2754                return Err(PatchError::Stale {
2755                    current: current.revision,
2756                });
2757            }
2758        }
2759        if current.revision != expected_revision {
2760            return Err(PatchError::Stale {
2761                current: current.revision,
2762            });
2763        }
2764        let Some((path, bytes, value)) = prepare(&current)? else {
2765            return Ok(None);
2766        };
2767        // Go `Manager.persist` never writes a plugin-virtual credential (one of several a
2768        // plugin parsed from one file): its document is not the file's. A refresh or
2769        // patch of one is kept in memory.
2770        if let Some(path) = path.filter(|_| !marked(&current, "plugin_virtual")) {
2771            write_bytes_atomic(&path, &bytes).map_err(|e| PatchError::Io(e.to_string()))?;
2772        }
2773        let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
2774        // Every other change to this credential waits on the locks held here.
2775        let index = inner
2776            .creds
2777            .iter()
2778            .position(|c| c.id == id && c.revision == expected_revision)
2779            .ok_or(PatchError::NotFound)?;
2780        Ok(Some(publish(&mut inner, index, &bytes, value)))
2781    }
2782
2783    /// Replaces a config-backed or runtime-only credential in memory if it is still at
2784    /// `expected_revision`: Go's `Manager.Update` never persists config API keys (the
2785    /// next config publish re-synthesizes them) or `runtime_only` auths. `NotFound`
2786    /// unless `next.id` names such a credential.
2787    pub fn replace_config_backed(
2788        &self,
2789        next: Credential,
2790        expected_revision: u64,
2791    ) -> Result<Arc<Credential>, PatchError> {
2792        let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
2793        let generation = inner.generation + 1;
2794        let slot = inner
2795            .creds
2796            .iter_mut()
2797            .find(|c| c.id == next.id && matches!(c.source, Source::Config { .. } | Source::Runtime))
2798            .ok_or(PatchError::NotFound)?;
2799        if slot.revision != expected_revision {
2800            return Err(PatchError::Stale { current: slot.revision });
2801        }
2802        let mut next = next;
2803        next.source = slot.source.clone();
2804        next.revision = generation;
2805        let next = Arc::new(next);
2806        self.register(std::slice::from_ref(slot), std::slice::from_ref(&next), &[]);
2807        let cleared = self.clear_replaced_unauthorized(std::slice::from_ref(slot), std::slice::from_ref(&next));
2808        *slot = next;
2809        let committed = slot.clone();
2810        inner.generation = generation;
2811        // Registrations depend on the credential (prefix, models): invalidate the
2812        // registry cache like `apply_patch` does.
2813        inner.epoch += 1;
2814        self.changed.send_replace(());
2815        drop(inner);
2816        if cleared {
2817            self.persist_cooldowns();
2818        }
2819        // Go `Manager.Update` clears the cooldowns of an auth it stores disabled (or
2820        // with cooling off), so a later re-enable routes to it at once.
2821        self.clear_disabled_cooldowns();
2822        Ok(committed)
2823    }
2824
2825    /// Go `Manager.Modify` for an auth file: while `id` is still at `expected_revision`,
2826    /// writes `bytes` to its file and puts `next` (the credential re-synthesized from
2827    /// those bytes, possibly under a new provider or ID; `None` drops it) in its place.
2828    /// One write lock covers the check, the file and the swap, so a concurrent token
2829    /// refresh ([`Self::apply_patch`]) lands either before (`Stale`) or after, never
2830    /// under it, and nothing else in the set is re-read or replaced.
2831    pub fn replace_file_backed(
2832        &self,
2833        id: &str,
2834        expected_revision: u64,
2835        bytes: &[u8],
2836        next: Option<Credential>,
2837    ) -> Result<(), PatchError> {
2838        let commit = Commit {
2839            bytes: bytes.to_vec(),
2840            write: true,
2841            disk: None,
2842            doc: None,
2843        };
2844        self.commit_file_backed(id, expected_revision, commit, next)
2845    }
2846
2847    /// [`Self::replace_file_backed`] for a management save ([`Commit`]): the file check
2848    /// and the optional write run under the same lock and revision check, and the new
2849    /// revision records the save's [`Provenance`].
2850    pub fn commit_file_backed(
2851        &self,
2852        id: &str,
2853        expected_revision: u64,
2854        commit: Commit,
2855        next: Option<Credential>,
2856    ) -> Result<(), PatchError> {
2857        let Commit {
2858            bytes,
2859            write,
2860            disk,
2861            doc,
2862        } = commit;
2863        // A new ID can replace another credential, whose own write lock is not held here.
2864        let exclusive = next.as_ref().is_some_and(|n| n.id != id);
2865        let removed = self.write_unlocked(
2866            id,
2867            expected_revision,
2868            None,
2869            exclusive,
2870            |current| {
2871                let Source::File(path) = &current.source else {
2872                    return Err(PatchError::ConfigBacked);
2873                };
2874                if let Some(want) = disk {
2875                    let now = std::fs::read(path).ok().map(|b| sha256(&b));
2876                    if now != want {
2877                        return Err(PatchError::Stale {
2878                            current: current.revision,
2879                        });
2880                    }
2881                }
2882                // The file's hash after the commit.
2883                let held = if write { Some(sha256(&bytes)) } else { disk.flatten() };
2884                Ok(Some((write.then(|| path.clone()), bytes, (next, held))))
2885            },
2886            |inner, index, _, (next, held)| {
2887                let before = inner.creds.clone();
2888                let previous = inner.creds.remove(index);
2889                self.provenance
2890                    .lock()
2891                    .unwrap_or_else(PoisonError::into_inner)
2892                    .remove(&previous.id);
2893                if let Some(mut next) = next {
2894                    inner.generation += 1;
2895                    next.revision = inner.generation;
2896                    if let Some(held) = held {
2897                        self.note_provenance(&next.id, next.revision, held, doc);
2898                    }
2899                    inner.creds.retain(|c| c.id != next.id);
2900                    let at = index.min(inner.creds.len());
2901                    inner.creds.insert(at, Arc::new(next));
2902                }
2903                self.register(&before, &inner.creds, &[]);
2904                self.clear_replaced_unauthorized(&before, &inner.creds);
2905                inner.epoch += 1;
2906                self.changed.send_replace(());
2907                self.scheduler
2908                    .lock()
2909                    .unwrap_or_else(PoisonError::into_inner)
2910                    .reconcile(&inner.creds);
2911                removed_ids(&[previous], &inner.creds)
2912            },
2913        )?;
2914        self.close_sessions(removed.unwrap_or_default());
2915        self.clear_disabled_cooldowns();
2916        self.persist_cooldowns();
2917        Ok(())
2918    }
2919
2920    /// Go's metadata after `patch` (a token refresh) lands on `current`: Go's refresh
2921    /// changes only the patched keys of the auth it holds. From a save, that is the saved
2922    /// document with the patched values as Go holds them; from a load, the loaded text
2923    /// (while `path` still holds it) with the patched values as written, `true`. `None`
2924    /// when neither is known.
2925    fn patched_doc(
2926        &self,
2927        current: &Credential,
2928        path: &std::path::Path,
2929        patch: &MetadataPatch,
2930        disabled: bool,
2931    ) -> Option<(cpa_common::json::GoValue, bool)> {
2932        use cpa_common::json::GoValue;
2933        let prior = self
2934            .provenance(&current.id)
2935            .filter(|p| p.revision == current.revision)?;
2936        let (base, loaded) = match prior.doc {
2937            Some(doc) => (GoValue::clone(&doc), prior.loaded),
2938            None => {
2939                let bytes = std::fs::read(path).ok().filter(|b| sha256(b) == prior.sha256)?;
2940                (GoValue::parse(&bytes)?, true)
2941            }
2942        };
2943        let mut doc = base.into_object().ok()?;
2944        for key in &patch.remove {
2945            doc.remove(key);
2946        }
2947        for (key, value) in &patch.set {
2948            doc.insert(key.clone(), go_typed(value, loaded));
2949        }
2950        doc.insert("disabled".into(), GoValue::Bool(disabled));
2951        Some((GoValue::Object(doc), loaded))
2952    }
2953
2954    /// The [`Provenance`] of credential `id`, if one is known.
2955    pub fn provenance(&self, id: &str) -> Option<Provenance> {
2956        self.provenance
2957            .lock()
2958            .unwrap_or_else(PoisonError::into_inner)
2959            .get(id)
2960            .cloned()
2961    }
2962
2963    fn note_provenance(&self, id: &str, revision: u64, sha256: [u8; 32], doc: Option<Arc<cpa_common::json::GoValue>>) {
2964        self.provenance.lock().unwrap_or_else(PoisonError::into_inner).insert(
2965            id.to_owned(),
2966            Provenance {
2967                revision,
2968                sha256,
2969                doc,
2970                loaded: false,
2971            },
2972        );
2973    }
2974
2975    /// Records that the current file credentials were loaded from the bytes `read`
2976    /// hashed (an auth-folder scan's [`AuthScan::read`]). A credential whose recorded
2977    /// save already holds those bytes at its revision keeps that save's document.
2978    ///
2979    /// [`AuthScan::read`]: cpa_core::config::credentials::AuthScan
2980    pub fn note_loaded(&self, read: &std::collections::BTreeMap<std::path::PathBuf, [u8; 32]>) {
2981        let creds = self.snapshot();
2982        let mut provenance = self.provenance.lock().unwrap_or_else(PoisonError::into_inner);
2983        provenance.retain(|id, _| creds.iter().any(|c| c.id == *id));
2984        for c in creds.iter() {
2985            let Source::File(path) = &c.source else { continue };
2986            let Some(&hash) = read.get(path) else { continue };
2987            let same = provenance
2988                .get(&c.id)
2989                .is_some_and(|p| p.revision == c.revision && p.sha256 == hash);
2990            if !same {
2991                provenance.insert(
2992                    c.id.clone(),
2993                    Provenance {
2994                        revision: c.revision,
2995                        sha256: hash,
2996                        doc: None,
2997                        loaded: false,
2998                    },
2999                );
3000            }
3001        }
3002    }
3003
3004    /// Go `Service.removeAuth` (1e37b52b): a credential that left the set ends only the
3005    /// upstream WebSocket sessions it serves. Runs with the set's locks released.
3006    fn close_sessions(&self, removed: Vec<String>) {
3007        if let Some(close) = self.session_closer.get() {
3008            for id in removed {
3009                close(&id);
3010            }
3011        }
3012    }
3013
3014    /// Replaces the credential set (watcher reload, management import/delete). Unchanged
3015    /// credentials keep their revision; new and changed ones get fresh revisions.
3016    pub fn reconcile(&self, credentials: Vec<Credential>) {
3017        self.reconcile_scanned(credentials, &std::collections::BTreeMap::new());
3018    }
3019
3020    /// [`Self::reconcile`] for an auth-folder scan; `read` has the SHA-256 of the bytes
3021    /// each file credential was built from. Go c4c1d43e: a scanned credential that would
3022    /// replace the current one is applied only while its file still holds those bytes.
3023    /// Otherwise a write committed after the scan read the file (a token refresh, an
3024    /// edit) has changed it, and the current credential is kept: the stale snapshot
3025    /// would undo that write, and the watcher reloads the file it sees changed. A file
3026    /// that is gone or unreadable keeps the current credential too. The check runs
3027    /// under the exclusive reload gate, so no write lands between it and the swap, but
3028    /// before `inner` is taken, so its file reads never stall readers. Returns the files
3029    /// whose scanned credential was rejected.
3030    pub fn reconcile_scanned(
3031        &self,
3032        credentials: Vec<Credential>,
3033        read: &std::collections::BTreeMap<std::path::PathBuf, [u8; 32]>,
3034    ) -> Vec<std::path::PathBuf> {
3035        self.reconcile_scanned_with(credentials, read, |_, _| {})
3036    }
3037
3038    /// [`Self::reconcile_scanned`]; `keep` first adds to the set from the current
3039    /// credentials, under the gate, so no write lands between what it reads and the swap.
3040    pub fn reconcile_scanned_with(
3041        &self,
3042        credentials: Vec<Credential>,
3043        read: &std::collections::BTreeMap<std::path::PathBuf, [u8; 32]>,
3044        keep: impl FnOnce(&[Arc<Credential>], &mut Vec<Credential>),
3045    ) -> Vec<std::path::PathBuf> {
3046        use sha2::{Digest, Sha256};
3047        let reload = self.reload.write().unwrap_or_else(PoisonError::into_inner);
3048        let mut credentials = credentials;
3049        keep(&self.snapshot(), &mut credentials);
3050        // A credential this scan synthesized has no revision yet; one kept for an
3051        // incomplete or deferred file is the store's own record, not built from `read`.
3052        let built: std::collections::HashSet<std::path::PathBuf> = credentials
3053            .iter()
3054            .filter(|c| c.revision == 0)
3055            .filter_map(|c| match &c.source {
3056                Source::File(path) => Some(path.clone()),
3057                _ => None,
3058            })
3059            .collect();
3060        let mut rejected = Vec::new();
3061        if !read.is_empty() {
3062            let current = self.snapshot();
3063            let by_id: HashMap<&str, &Arc<Credential>> = current.iter().map(|c| (c.id.as_str(), c)).collect();
3064            for cred in &mut credentials {
3065                let Source::File(path) = &cred.source else { continue };
3066                let Some(seen) = read.get(path) else { continue };
3067                let Some(&existing) = by_id.get(cred.id.as_str()) else {
3068                    continue;
3069                };
3070                if unchanged(existing, cred) {
3071                    continue;
3072                }
3073                // The scan's own rule: a FIFO or device in the file's place is never read
3074                // (it could block or never end) and counts as a changed file.
3075                let on_disk: Option<[u8; 32]> = cpa_core::config::credentials::read_regular(path)
3076                    .filter(|data| !data.is_empty())
3077                    .map(|data| Sha256::digest(&data).into());
3078                if on_disk.as_ref() != Some(seen) {
3079                    rejected.push(path.clone());
3080                    *cred = Credential::clone(existing);
3081                }
3082            }
3083        }
3084        let mut guard = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3085        let inner = &mut *guard;
3086        let mut next = Vec::with_capacity(credentials.len());
3087        let mut by_id = HashMap::with_capacity(inner.creds.len());
3088        for c in &inner.creds {
3089            by_id.entry(c.id.as_str()).or_insert(c);
3090        }
3091        // A credential the startup load registered provisionally is registered again
3092        // when the first plugin-aware load parses another `NextRefreshAfter` for it,
3093        // whatever else it equals: Go's first load registers it with that one.
3094        let provisional = crate::plugins::refresh::provisional(&self.refresh_schedules);
3095        for mut cred in credentials {
3096            let same = |c: &Credential| {
3097                unchanged(c, &cred)
3098                    && !(provisional.contains(c.id.as_str()) && c.next_refresh_after != cred.next_refresh_after)
3099            };
3100            match by_id.get(cred.id.as_str()).filter(|c| same(c)) {
3101                Some(&existing) => next.push(existing.clone()),
3102                None => {
3103                    inner.generation += 1;
3104                    cred.revision = inner.generation;
3105                    next.push(Arc::new(cred));
3106                }
3107            }
3108        }
3109        // Runtime-only credentials (relay sessions) have no file or config entry; they
3110        // stay until their session ends.
3111        let runtime: Vec<_> = inner
3112            .creds
3113            .iter()
3114            .filter(|c| c.source == Source::Runtime && !next.iter().any(|n| n.id == c.id))
3115            .cloned()
3116            .collect();
3117        next.extend(runtime);
3118        let removed = removed_ids(&inner.creds, &next);
3119        self.register(&inner.creds, &next, &[]);
3120        self.clear_replaced_unauthorized(&inner.creds, &next);
3121        inner.creds = next;
3122        inner.epoch += 1;
3123        self.changed.send_replace(());
3124        self.scheduler
3125            .lock()
3126            .unwrap_or_else(PoisonError::into_inner)
3127            .reconcile(&inner.creds);
3128        drop(guard);
3129        // Under the gate still: no save lands between the scan's publication and this.
3130        let mut loaded = read.clone();
3131        loaded.retain(|path, _| built.contains(path) && !rejected.contains(path));
3132        self.note_loaded(&loaded);
3133        drop(reload);
3134        self.close_sessions(removed);
3135        self.clear_disabled_cooldowns();
3136        self.persist_cooldowns();
3137        rejected
3138    }
3139
3140    /// Replaces the config-sourced credentials (Home config API keys, Go
3141    /// `registerConfigAPIKeyAuths`) under a single write lock and keeps every other
3142    /// current entry, so a concurrent [`Self::upsert`] or relay session change is never
3143    /// undone the way a snapshot followed by [`Self::reconcile`] could undo it. An
3144    /// unchanged key keeps its revision, as in `reconcile`.
3145    pub fn replace_config(&self, keys: Vec<Credential>) {
3146        let reload = self.reload.write().unwrap_or_else(PoisonError::into_inner);
3147        let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3148        let mut next: Vec<_> = inner
3149            .creds
3150            .iter()
3151            .filter(|c| !matches!(c.source, Source::Config { .. }))
3152            .cloned()
3153            .collect();
3154        for mut key in keys {
3155            match inner
3156                .creds
3157                .iter()
3158                .find(|c| c.id == key.id && unchanged(c, &key))
3159                .cloned()
3160            {
3161                Some(existing) => next.push(existing),
3162                None => {
3163                    inner.generation += 1;
3164                    key.revision = inner.generation;
3165                    next.push(Arc::new(key));
3166                }
3167            }
3168        }
3169        let removed = removed_ids(&inner.creds, &next);
3170        self.register(&inner.creds, &next, &[]);
3171        self.clear_replaced_unauthorized(&inner.creds, &next);
3172        inner.creds = next;
3173        inner.epoch += 1;
3174        self.changed.send_replace(());
3175        self.scheduler
3176            .lock()
3177            .unwrap_or_else(PoisonError::into_inner)
3178            .reconcile(&inner.creds);
3179        drop(inner);
3180        drop(reload);
3181        self.close_sessions(removed);
3182        self.clear_disabled_cooldowns();
3183        self.persist_cooldowns();
3184    }
3185
3186    /// Inserts or replaces one credential (a plugin's `host.auth.save`) under a single
3187    /// write lock, so a concurrent `apply_patch` on another credential is never undone
3188    /// the way a snapshot followed by [`Self::reconcile`] could undo it. An unchanged
3189    /// credential keeps its revision, as in `reconcile`.
3190    pub fn upsert(&self, credential: Credential) {
3191        self.upsert_file(None, vec![credential], &[]);
3192    }
3193
3194    /// [`Self::upsert`] for every credential an auth file saved at `path` holds (Go
3195    /// `upsertAuthRecordsForPath`): the file's other credentials, parsed from it
3196    /// earlier, go, runtime-only ones excepted. `saved` has the refresh schedules a
3197    /// plugin's save registers its credentials with.
3198    pub fn upsert_file(
3199        &self,
3200        path: Option<&std::path::Path>,
3201        credentials: Vec<Credential>,
3202        saved: &[cpa_plugin::hostauth::SavedRefresh],
3203    ) {
3204        let reload = self.reload.write().unwrap_or_else(PoisonError::into_inner);
3205        let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3206        let before = inner.creds.clone();
3207        let mut changed = false;
3208        let mut removed = Vec::new();
3209        if let Some(path) = path {
3210            let clean = cpa_core::config::credentials::absolute_clean_path;
3211            let target = clean(path);
3212            let before = std::mem::take(&mut inner.creds);
3213            for credential in before {
3214                let stale = !marked(&credential, "runtime_only")
3215                    && !credentials.iter().any(|c| c.id == credential.id)
3216                    && matches!(&credential.source, Source::File(p) if clean(p) == target);
3217                if stale {
3218                    removed.push(credential.id.clone());
3219                } else {
3220                    inner.creds.push(credential);
3221                }
3222            }
3223            changed = !removed.is_empty();
3224        }
3225        for mut credential in credentials {
3226            let slot = inner.creds.iter().position(|c| c.id == credential.id);
3227            if slot.is_some_and(|i| unchanged(&inner.creds[i], &credential)) {
3228                continue;
3229            }
3230            inner.generation += 1;
3231            credential.revision = inner.generation;
3232            let credential = Arc::new(credential);
3233            match slot {
3234                Some(i) => inner.creds[i] = credential,
3235                None => inner.creds.push(credential),
3236            }
3237            changed = true;
3238        }
3239        if !changed {
3240            return;
3241        }
3242        self.register(&before, &inner.creds, saved);
3243        self.clear_replaced_unauthorized(&before, &inner.creds);
3244        inner.epoch += 1;
3245        self.changed.send_replace(());
3246        self.scheduler
3247            .lock()
3248            .unwrap_or_else(PoisonError::into_inner)
3249            .reconcile(&inner.creds);
3250        drop(inner);
3251        drop(reload);
3252        self.close_sessions(removed);
3253        self.clear_disabled_cooldowns();
3254        self.persist_cooldowns();
3255    }
3256
3257    /// Adds a runtime-only credential (Go's auth add for a `/v1/ws` relay session)
3258    /// unless one with its ID is already active. Additive API for the relay route.
3259    pub fn add_runtime(&self, mut credential: Credential) -> bool {
3260        let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3261        if inner.creds.iter().any(|c| c.id == credential.id && !c.disabled) {
3262            return false;
3263        }
3264        let before = inner.creds.clone();
3265        inner.creds.retain(|c| c.id != credential.id);
3266        inner.generation += 1;
3267        credential.revision = inner.generation;
3268        inner.creds.push(Arc::new(credential));
3269        self.register(&before, &inner.creds, &[]);
3270        inner.epoch += 1;
3271        self.changed.send_replace(());
3272        self.scheduler
3273            .lock()
3274            .unwrap_or_else(PoisonError::into_inner)
3275            .reconcile(&inner.creds);
3276        true
3277    }
3278
3279    /// Removes a credential when its relay session ends (Go's auth delete). Additive
3280    /// API for the relay route.
3281    pub fn remove_runtime(&self, id: &str) {
3282        let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner);
3283        let before = inner.creds.clone();
3284        inner.creds.retain(|c| !(c.id == id && c.source == Source::Runtime));
3285        if inner.creds.len() == before.len() {
3286            return;
3287        }
3288        self.register(&before, &inner.creds, &[]);
3289        inner.epoch += 1;
3290        self.changed.send_replace(());
3291        self.scheduler
3292            .lock()
3293            .unwrap_or_else(PoisonError::into_inner)
3294            .reconcile(&inner.creds);
3295    }
3296
3297    fn read(&self) -> std::sync::RwLockReadGuard<'_, Inner> {
3298        self.inner.read().unwrap_or_else(PoisonError::into_inner)
3299    }
3300}
3301
3302/// Per-credential outcome counters for management views (Go `Auth.Success`/`Failed`
3303/// and its 20 x 10-minute recent-request ring). Additive read API.
3304#[derive(Debug, Clone, Default, PartialEq, Eq)]
3305pub struct CredentialActivity {
3306    pub success: u64,
3307    pub failed: u64,
3308    /// `(bucket, success, failed)` for the most recent buckets, where `bucket` is
3309    /// Unix seconds / 600; at most 20 entries, oldest first.
3310    pub recent: Vec<(i64, u64, u64)>,
3311}
3312
3313pub const RECENT_BUCKET_SECONDS: i64 = 600;
3314const RECENT_BUCKETS: usize = 20;
3315
3316impl CredentialStore {
3317    fn note_activity(&self, id: &str, success: bool) {
3318        let bucket = std::time::SystemTime::now()
3319            .duration_since(std::time::UNIX_EPOCH)
3320            .map_or(0, |d| d.as_secs() as i64 / RECENT_BUCKET_SECONDS);
3321        let mut all = self.activity.lock().unwrap_or_else(PoisonError::into_inner);
3322        let entry = all.entry(id.to_owned()).or_default();
3323        if success {
3324            entry.success += 1;
3325        } else {
3326            entry.failed += 1;
3327        }
3328        match entry.recent.last_mut() {
3329            Some(last) if last.0 == bucket => {
3330                if success {
3331                    last.1 += 1;
3332                } else {
3333                    last.2 += 1;
3334                }
3335            }
3336            _ => entry.recent.push((bucket, u64::from(success), u64::from(!success))),
3337        }
3338        entry.recent.retain(|(b, _, _)| bucket - b < RECENT_BUCKETS as i64);
3339    }
3340
3341    /// Counters for one credential; zero when it has served nothing yet.
3342    pub fn activity(&self, id: &str) -> CredentialActivity {
3343        self.activity
3344            .lock()
3345            .unwrap_or_else(PoisonError::into_inner)
3346            .get(id)
3347            .cloned()
3348            .unwrap_or_default()
3349    }
3350
3351    /// Active cooldowns of one credential.
3352    pub fn cooldowns(&self, id: &str) -> Vec<crate::scheduler::CooldownState> {
3353        self.scheduler
3354            .lock()
3355            .unwrap_or_else(PoisonError::into_inner)
3356            .cooldowns_of(id, Instant::now())
3357    }
3358
3359    /// Clears the cooldowns of one credential (Go `Manager.ResetQuota`); returns the
3360    /// model keys that were cooling.
3361    pub fn reset_cooldowns(&self, id: &str) -> Vec<String> {
3362        let had = !self.cooldowns(id).is_empty();
3363        let models = self
3364            .scheduler
3365            .lock()
3366            .unwrap_or_else(PoisonError::into_inner)
3367            .reset_cooldowns(id);
3368        if had {
3369            // Go `ResetQuota` persists the cleared state.
3370            self.persist_cooldowns();
3371        }
3372        models
3373    }
3374
3375    /// The recovery Go runs after a token refresh and after `BindCodexSIWC` saves a
3376    /// binding: an enabled credential's unauthorized cooldowns are cleared and persisted,
3377    /// live quota blocks stay. Go also forgets the rejected access token; here a pending
3378    /// unauthorized refresh keys on the revision, which the new tokens replaced.
3379    pub fn recover_unauthorized(&self, id: &str) {
3380        if self.get(id).is_none_or(|c| c.disabled) {
3381            return;
3382        }
3383        let cleared = self
3384            .scheduler
3385            .lock()
3386            .unwrap_or_else(PoisonError::into_inner)
3387            .clear_unauthorized(id, Instant::now());
3388        if cleared {
3389            self.persist_cooldowns();
3390        }
3391    }
3392}
3393
3394impl Runtime {
3395    /// Runs the executor's single-flighted preparation now, due or not (management
3396    /// refresh, Go `RefreshAuthFile`). Returns the committed credential.
3397    pub async fn refresh_credential(&self, id: &str) -> Result<Arc<Credential>, ExecError> {
3398        let cfg = self.config();
3399        let revision = self.store.get(id).map(|c| c.revision);
3400        self.prepare_credential_with(id, &cfg, revision, true, None).await
3401    }
3402}
3403
3404/// Commits a token refresh of credential `id` that started from `current` under
3405/// `fence`'s registration: `patch` is what the refresh changed, merged with any edit
3406/// committed since (Go `UpdateRefreshedAuth`). Blocking.
3407pub(crate) fn commit_refresh(
3408    store: &CredentialStore,
3409    id: &str,
3410    current: &Credential,
3411    mut fence: RefreshFence,
3412    patch: &MetadataPatch,
3413    publish_proxy: impl Fn(&mut Credential) + Copy,
3414) -> Result<Arc<Credential>, PatchError> {
3415    let mut revision = current.revision;
3416    let mut attempt = patch.clone();
3417    loop {
3418        match store.apply_refresh(id, revision, fence, &attempt, publish_proxy, |c| {
3419            credentials_changed(current, c)
3420        }) {
3421            // Edited since the snapshot (the prepare lock does not hold off other writers),
3422            // within the refresh's registration: the writer refuses a re-registered ID
3423            // first (Go "stale registration epoch"), on every retry. Go 935aa6e3: an edit
3424            // that changed the credentials wins and this refresh is dropped; any other edit
3425            // is merged with it (Go `MergeRefreshedAuth`), so a rotated refresh token is
3426            // not lost. Credentials changed and put back since (A→B→A) count as changed:
3427            // the credential version moved (Go `CredentialVersion`).
3428            Err(PatchError::Stale { .. }) => {
3429                #[cfg(test)]
3430                if let Some(hook) = tests::AFTER_STALE.with(std::cell::Cell::take) {
3431                    hook();
3432                }
3433                // The writer's check, before anything here returns: an ID registered
3434                // again since then fails as Go's "stale registration epoch" does,
3435                // whatever its credentials.
3436                let (latest, now) = store.get_registered(id).ok_or(PatchError::NotFound)?;
3437                if now.registration != fence.registered.registration {
3438                    return Err(PatchError::NotFound);
3439                }
3440                let (rebased, partition_applies) =
3441                    rebase_patch(&current.provider, &current.metadata, &latest.metadata, patch);
3442                // Go 7b0319fb: a Codex token partition the refresh still matches
3443                // applies even when the other partition changed meanwhile
3444                // (`CredentialRefreshPatch.CanApplyTo`).
3445                let moved = now.credential_version != fence.registered.credential_version;
3446                if (moved || credentials_changed(current, &latest)) && !partition_applies {
3447                    return Ok(latest);
3448                }
3449                // Merged into `latest` from here, at its credential version.
3450                fence.registered = now;
3451                attempt = rebased;
3452                revision = latest.revision;
3453                if attempt.set.is_empty() && attempt.remove.is_empty() {
3454                    // Nothing left to write, but the refresh happened (Go records
3455                    // `LastRefreshedAt` whatever the merge changes), recorded while the
3456                    // credential is still `latest` under the refresh's registration. An
3457                    // edit since goes round again: its newer revision fails the writer's
3458                    // check, as revisions are never reused, so the empty patch is never
3459                    // written.
3460                    let at = (revision, fence.registered.credential_version);
3461                    match store.record_refresh(id, fence.registered.registration, Some(at), |c, s| {
3462                        if fence.plugin {
3463                            crate::plugins::refresh::committed(c, s, chrono::Utc::now());
3464                        }
3465                    }) {
3466                        Err(PatchError::Stale { .. }) => continue,
3467                        recorded => return recorded,
3468                    }
3469                }
3470            }
3471            other => {
3472                let (committed, rotated) = other?;
3473                // Go `updateInternal` after a refresh: new tokens clear the credential's
3474                // unauthorized cooldowns, keeping live quota blocks (50d5fe18). Here, so a
3475                // cancelled caller still does.
3476                if rotated {
3477                    store.recover_unauthorized(id);
3478                }
3479                return Ok(committed);
3480            }
3481        }
3482    }
3483}
3484
3485/// Go `mergeAuthContent` for a prepare patch computed from `base` (the snapshot) that
3486/// lost the race to another edit, `user` being the stored metadata now: the part of
3487/// `patch` to apply on top of `user`. A value the patch did not change keeps the edit;
3488/// a value both changed keeps the edit unless it is a token payload. A missing key is
3489/// not JSON null. Codex outer tokens move as one partition (Go `CredentialRefreshPatch`).
3490/// The Sign in with ChatGPT partition (`siwc`) moves the same way, inside its map. The
3491/// flag reports whether a refreshed Codex partition still matches the stored version.
3492// ponytail: no prepare patch sets `proxy_url` or `use_siwc`, so Go's dedicated merges for
3493// those keys are not ported; add them with the first producer.
3494fn rebase_patch(
3495    provider: &str,
3496    base: &Map<String, Value>,
3497    user: &Map<String, Value>,
3498    patch: &MetadataPatch,
3499) -> (MetadataPatch, bool) {
3500    // Go `codexOuterCredentialKeys` (internal/auth/codex/token.go).
3501    const CODEX_OUTER: [&str; 8] = [
3502        "id_token",
3503        "access_token",
3504        "refresh_token",
3505        "account_id",
3506        "last_refresh",
3507        "email",
3508        "expired",
3509        "plan_type",
3510    ];
3511    // Go `IsAuthTokenPayloadKey`.
3512    let token_payload = |k: &str| {
3513        matches!(
3514            k.trim().to_ascii_lowercase().as_str(),
3515            "access_token"
3516                | "refresh_token"
3517                | "id_token"
3518                | "session_id"
3519                | "expired"
3520                | "last_refresh"
3521                | "expires_in"
3522                | "timestamp"
3523                | "token_type"
3524                | "user_code"
3525                | "verification_uri"
3526                | "verification_uri_complete"
3527        )
3528    };
3529    let codex = provider == "codex";
3530    let kept_by_user = |k: &str| base.get(k) == user.get(k);
3531    let mut out = MetadataPatch::default();
3532    let mut partition_applies = false;
3533    for (k, v) in &patch.set {
3534        let outer = codex && (CODEX_OUTER.contains(&k.as_str()) || k == "siwc");
3535        if !outer && base.get(k) != Some(v) && (kept_by_user(k) || token_payload(k)) {
3536            out.set.insert(k.clone(), v.clone());
3537        }
3538    }
3539    out.remove = patch
3540        .remove
3541        .iter()
3542        .filter(|k| base.contains_key(*k) && kept_by_user(k))
3543        .cloned()
3544        .collect();
3545    if codex {
3546        // Go `CredentialRefreshPatch.Apply`: the refreshed tokens apply only while the
3547        // stored identity is still the one refreshed (or its result), and then only the
3548        // fields the refresh changed.
3549        let fields = |m: &Map<String, Value>, keys: &[&str]| -> Map<String, Value> {
3550            keys.iter()
3551                .filter_map(|k| m.get(*k).map(|v| ((*k).to_owned(), v.clone())))
3552                .collect()
3553        };
3554        let mut updated = base.clone();
3555        patch.apply(&mut updated);
3556        // Go `sameCredentialVersion` (`reflect.DeepEqual` on map reads: null is missing).
3557        let same = |current: &Map<String, Value>, b: &Map<String, Value>| {
3558            [
3559                "access_token",
3560                "refresh_token",
3561                "id_token",
3562                "account_id",
3563                "client_id",
3564                "device_id",
3565            ]
3566            .iter()
3567            .all(|k| current.get(*k).filter(|v| !v.is_null()) == b.get(*k).filter(|v| !v.is_null()))
3568        };
3569        // The changed fields of one partition, when `current` still holds the version
3570        // refreshed or its result.
3571        let changes = |current: &Map<String, Value>, before: &Map<String, Value>, after: &Map<String, Value>| {
3572            [before, after].into_iter().find(|b| same(current, b)).map(|baseline| {
3573                after
3574                    .iter()
3575                    .filter(|(k, v)| baseline.get(*k) != Some(*v))
3576                    .map(|(k, v)| (k.clone(), v.clone()))
3577                    .collect::<Map<String, Value>>()
3578            })
3579        };
3580        let (before, after) = (fields(base, &CODEX_OUTER), fields(&updated, &CODEX_OUTER));
3581        if before != after
3582            && let Some(changed) = changes(user, &before, &after)
3583        {
3584            partition_applies = true;
3585            out.set.extend(changed);
3586        }
3587        // Go `codexSIWCCredentialKeys`.
3588        const CODEX_SIWC: [&str; 9] = [
3589            "access_token",
3590            "refresh_token",
3591            "id_token",
3592            "client_id",
3593            "device_id",
3594            "expired",
3595            "last_refresh",
3596            "scope",
3597            "scopes",
3598        ];
3599        let siwc = |m: &Map<String, Value>| m.get("siwc").and_then(Value::as_object).cloned().unwrap_or_default();
3600        let (before, after) = (fields(&siwc(base), &CODEX_SIWC), fields(&siwc(&updated), &CODEX_SIWC));
3601        let current = siwc(user);
3602        if before != after
3603            && let Some(changed) = changes(&current, &before, &after)
3604        {
3605            partition_applies = true;
3606            if !changed.is_empty() {
3607                let mut merged = current;
3608                merged.extend(changed);
3609                out.set.insert("siwc".into(), Value::Object(merged));
3610            }
3611        }
3612    }
3613    (out, partition_applies)
3614}
3615
3616/// Go `CodexExecutor.Refresh` for credential `current`, whose file is `path`, as one
3617/// transaction: the file's credential lock (`<file>.lock`, shared with Go and
3618/// `--siwc-bind`) is taken, the file read under it, the grants sent from the tokens it
3619/// holds ([`cpa_exec::codex_refresh::Refresher::refresh_locked`]), their result written
3620/// onto the file as read again under the same lock when a grant was attempted
3621/// ([`codex_patched`]), and then admitted to the store, which writes the file again,
3622/// or discarded ([`CredentialStore::admit_codex_refresh`]), before the lock is released. A spawned
3623/// task owns it all, the preparation lock (`guard`) included: a cancelled caller cannot
3624/// release the lock while a grant is in flight and leave its new tokens unwritten.
3625#[allow(clippy::too_many_arguments)]
3626async fn refresh_codex_file(
3627    refreshes: &CodexRefreshes,
3628    store: Arc<CredentialStore>,
3629    refresher: cpa_exec::codex_refresh::Refresher,
3630    guard: tokio::sync::OwnedMutexGuard<()>,
3631    current: Arc<Credential>,
3632    registered: crate::plugins::refresh::Registered,
3633    view: Arc<Credential>,
3634    path: std::path::PathBuf,
3635    mode: cpa_exec::codex_refresh::RefreshMode,
3636    refused: Option<Arc<Credential>>,
3637) -> Result<Arc<Credential>, ExecError> {
3638    let local = |e: String| ExecError::local(500, FailureScope::Request, format!("codex executor: {e}"));
3639    let shutdown = refreshes.shutdown.clone();
3640    let transaction = refreshes.tasks.spawn(async move {
3641        let _guard = guard;
3642        let (lock, disk) = read_locked_codex_file(&path, &shutdown).await.map_err(local)?;
3643        // Recorded while the preparation lock is held ([`Runtime::prepare_credential_with`]).
3644        let patch = refresher.refresh_locked(&view, &disk, mode).await.inspect_err(|e| {
3645            if let Some(refused) = &refused {
3646                store.reject_refused(refused, registered.registration, e);
3647            }
3648        })?;
3649        if patch.is_empty() {
3650            return Ok(current);
3651        }
3652        let admitted = tokio::task::spawn_blocking(move || {
3653            let _lock = lock;
3654            // Go `refreshLocked` writes after a grant, whatever its manager then admits;
3655            // an adopt-only refresh reaches the file only when admitted.
3656            if patch.granted() {
3657                let bytes = std::fs::read(&path).map_err(|e| {
3658                    PatchError::Io(format!("re-read credential file {} before write: {e}", path.display()))
3659                })?;
3660                let out = codex_patched(&bytes, &patch, true).ok_or_else(|| {
3661                    PatchError::Io(format!("invalid credential JSON in {} before write", path.display()))
3662                })?;
3663                write_bytes_atomic(&path, &out)
3664                    .map_err(|e| PatchError::Io(format!("persist refreshed credentials to {}: {e}", path.display())))?;
3665            }
3666            #[cfg(test)]
3667            tests::before_admit(&path);
3668            store.admit_codex_refresh(&current, registered, &patch)
3669        })
3670        .await
3671        .map_err(|e| PatchError::Io(e.to_string()))
3672        .and_then(|r| r)
3673        .map_err(|e| {
3674            ExecError::local(
3675                500,
3676                FailureScope::Request,
3677                format!("committing prepared credential: {e:?}"),
3678            )
3679        })?;
3680        Ok(match admitted {
3681            CodexAdmission::Committed(c) | CodexAdmission::Discarded(c) => c,
3682        })
3683    });
3684    transaction.await.map_err(|e| local(e.to_string()))?
3685}
3686
3687/// The Codex credential document `bytes` with `patch` applied (each partition over a
3688/// version it may still be applied over, every other field as it is), as Go's
3689/// `json.Marshal` of the decoded map; `None` when it is not a JSON object. For
3690/// `refresh_write`, Go `refreshLocked`'s write after a grant: the document must be a
3691/// non-empty object and keys stay as written. Otherwise Go `PersistCodexAuthFile`'s: any
3692/// object, legacy keys renamed (Go `NormalizeCredentialMetadata`).
3693fn codex_patched(bytes: &[u8], patch: &cpa_exec::codex_refresh::RefreshPatch, refresh_write: bool) -> Option<Vec<u8>> {
3694    use cpa_common::json::GoValue;
3695    // ponytail: the file is read through serde for the partition baselines, so one
3696    // holding invalid UTF-8 or a lone surrogate escape fails here (DIFFERENCES-FROM-GO.md).
3697    let target = match serde_json::from_slice::<Value>(bytes) {
3698        Ok(Value::Object(target)) if !(refresh_write && target.is_empty()) => target,
3699        _ => return None,
3700    };
3701    let mut doc = GoValue::parse_f64(bytes)?.into_object().ok()?;
3702    patch.apply(&target, &mut doc);
3703    if !refresh_write {
3704        let legacy: Vec<String> = doc
3705            .keys()
3706            .filter(|k| cpa_plugin::auth::canonical_metadata_key(k) != k.as_str())
3707            .cloned()
3708            .collect();
3709        for key in legacy {
3710            let value = doc.remove(&key).expect("listed above");
3711            doc.entry(cpa_plugin::auth::canonical_metadata_key(&key).to_owned())
3712                .or_insert(value);
3713        }
3714    }
3715    Some(GoValue::Object(doc).marshal())
3716}
3717
3718/// Go `PersistCodexAuthFile` for an admitted refresh of `current`, while the caller holds
3719/// the file's credential lock: `patch` over the file as it is now, written. A file that
3720/// is gone is written from memory instead (the patch over `current`'s metadata, with its
3721/// `disabled` flag). A save that fails is logged and the patch over memory is published
3722/// unwritten, as Go's manager does (`failed to persist updated auth`), so the new tokens
3723/// are never dropped for a spent refresh token. Returns the document and whether the
3724/// file now holds it. Blocking.
3725fn save_codex_refresh(
3726    path: &Path,
3727    current: &Credential,
3728    patch: &cpa_exec::codex_refresh::RefreshPatch,
3729) -> (Vec<u8>, bool) {
3730    let memory = || {
3731        let mut metadata = current.metadata.clone();
3732        metadata.insert("disabled".into(), Value::Bool(current.disabled));
3733        let bytes = serde_json::to_vec(&metadata).expect("JSON values serialize");
3734        codex_patched(&bytes, patch, false).expect("a JSON object")
3735    };
3736    let warn = |error: String| {
3737        tracing::warn!(credential = %current.id, %error, "failed to persist updated codex auth");
3738    };
3739    let out = match std::fs::read(path) {
3740        Ok(bytes) => {
3741            codex_patched(&bytes, patch, false).ok_or_else(|| format!("invalid credential JSON in {}", path.display()))
3742        }
3743        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(memory()),
3744        Err(e) => Err(format!("read credential file {}: {e}", path.display())),
3745    };
3746    let saved = out.and_then(|out| match write_bytes_atomic(path, &out) {
3747        Ok(()) => Ok(out),
3748        Err(e) => Err(format!("write credential file {}: {e}", path.display())),
3749    });
3750    match saved {
3751        Ok(out) => (out, true),
3752        Err(error) => {
3753            warn(error);
3754            (memory(), false)
3755        }
3756    }
3757}
3758
3759/// What [`CredentialStore::admit_codex_refresh`] did with a refresh.
3760enum CodexAdmission {
3761    /// Published as the credential.
3762    Committed(Arc<Credential>),
3763    /// Dropped, nothing changed: the credential as it is.
3764    Discarded(Arc<Credential>),
3765}
3766
3767/// A credential's own proxy, published from its metadata (`proxy_url`, trimmed), or
3768/// none when the metadata has none (Go's canonical proxy merge).
3769fn publish_metadata_proxy(c: &mut Credential) {
3770    match c.str("proxy_url").map(str::trim).filter(|p| !p.is_empty()) {
3771        Some(proxy) => {
3772            let proxy = proxy.to_owned();
3773            c.attributes.insert("proxy_url".into(), proxy)
3774        }
3775        None => c.attributes.remove("proxy_url"),
3776    };
3777}
3778
3779/// Why a wait for a Codex credential lock ended without it.
3780const LOCK_WAIT_STOPPED: &str = "shutting down while waiting for the credential lock";
3781
3782/// The longest pause between two attempts at a Codex credential lock.
3783const LOCK_POLL_CEILING: Duration = Duration::from_millis(100);
3784
3785/// The credential lock of the Codex file at `path` (Go's blocking `flock`), waited for
3786/// without holding a thread; the wait gives up at `shutdown`.
3787async fn wait_codex_lock(path: &Path, shutdown: &tokio_util::sync::CancellationToken) -> Result<std::fs::File, String> {
3788    #[cfg(test)]
3789    tests::LOCK_ATTEMPTS
3790        .lock()
3791        .unwrap_or_else(PoisonError::into_inner)
3792        .push(path.to_owned());
3793    let stopped = || LOCK_WAIT_STOPPED.to_owned();
3794    if shutdown.is_cancelled() {
3795        return Err(stopped());
3796    }
3797    let lock = cpa_exec::codex_siwc_bind::open_lock_file(path)?;
3798    // ponytail: polled, not a blocking wait, so shutdown can stop it; up to 100 ms later
3799    // than Go's blocking flock once the holder releases. Upgrade: a lock wait on a
3800    // blocking thread that a signal interrupts.
3801    let mut delay = Duration::from_millis(5);
3802    loop {
3803        // Before every attempt: a shutdown during the sleep wins over a lock freed then.
3804        if shutdown.is_cancelled() {
3805            return Err(stopped());
3806        }
3807        if cpa_exec::codex_siwc_bind::try_lock(&lock)? {
3808            return Ok(lock);
3809        }
3810        tokio::select! {
3811            biased;
3812            _ = shutdown.cancelled() => return Err(stopped()),
3813            _ = tokio::time::sleep(delay) => {}
3814        }
3815        delay = (delay * 2).min(LOCK_POLL_CEILING);
3816    }
3817}
3818
3819/// A held credential lock and the file read under it.
3820type LockedFile = (std::fs::File, Map<String, Value>);
3821
3822/// Go `refreshLocked`'s read: the Codex credential file under its lock, which the
3823/// returned file holds. A file that is gone, or that is not a non-empty JSON object,
3824/// fails before anything refreshes. The wait for the lock gives up at `shutdown`.
3825async fn read_locked_codex_file(
3826    path: &Path,
3827    shutdown: &tokio_util::sync::CancellationToken,
3828) -> Result<LockedFile, String> {
3829    let lock = wait_codex_lock(path, shutdown).await?;
3830    let data = std::fs::read(path).map_err(|e| format!("read credential file {}: {e}", path.display()))?;
3831    match serde_json::from_slice(&data) {
3832        Ok(Value::Object(disk)) if !disk.is_empty() => Ok((lock, disk)),
3833        _ => Err(format!("invalid credential JSON in {}", path.display())),
3834    }
3835}
3836
3837/// Go `CredentialsChanged`: the tokens or API key differ.
3838pub(crate) fn credentials_changed(a: &Credential, b: &Credential) -> bool {
3839    let text = |c: &Credential, keys: &[&str]| {
3840        keys.iter()
3841            .find_map(|k| {
3842                c.metadata
3843                    .get(*k)
3844                    .and_then(Value::as_str)
3845                    .map(str::trim)
3846                    .filter(|v| !v.is_empty())
3847            })
3848            .unwrap_or_default()
3849            .to_owned()
3850    };
3851    let siwc = |c: &Credential, key: &str| {
3852        c.metadata
3853            .get("siwc")
3854            .and_then(|s| s.get(key))
3855            .and_then(Value::as_str)
3856            .map_or("", str::trim)
3857            .to_owned()
3858    };
3859    let api_key = |c: &Credential| {
3860        c.attributes
3861            .get("api_key")
3862            .filter(|k| !k.is_empty())
3863            .cloned()
3864            .unwrap_or_else(|| text(c, &["api_key"]))
3865    };
3866    // Go `parseBoolAny`.
3867    let parse_bool = |s: &str| match s.trim() {
3868        "1" | "t" | "T" | "TRUE" | "true" | "True" => Some(true),
3869        "0" | "f" | "F" | "FALSE" | "false" | "False" => Some(false),
3870        _ => None,
3871    };
3872    let use_siwc = |c: &Credential| {
3873        let value = c.metadata.get("use_siwc")?;
3874        value
3875            .as_bool()
3876            .or_else(|| parse_bool(value.as_str()?))
3877            .or_else(|| value.as_f64().map(|n| n != 0.0))
3878    };
3879    // Go `authAccessToken`: the token requests use, which follows `IsCodexUseSIWC` (a
3880    // true attribute, else the file's flag).
3881    let effective_access_token = |c: &Credential| {
3882        let attribute = c.attributes.get("use_siwc").and_then(|v| parse_bool(v)) == Some(true);
3883        if attribute || use_siwc(c) == Some(true) {
3884            siwc(c, "access_token")
3885        } else {
3886            text(c, &["access_token", "accessToken"])
3887        }
3888    };
3889    effective_access_token(a) != effective_access_token(b)
3890        || [
3891            &["access_token", "accessToken"][..],
3892            &["refresh_token", "refreshToken"],
3893            &["id_token", "idToken"],
3894        ]
3895        .iter()
3896        .any(|keys| text(a, keys) != text(b, keys))
3897        || api_key(a) != api_key(b)
3898        || ["access_token", "refresh_token", "id_token", "client_id"]
3899            .iter()
3900            .any(|k| siwc(a, k) != siwc(b, k))
3901        || use_siwc(a) != use_siwc(b)
3902}
3903
3904/// Replaces `path` with `bytes` through an exclusively created, uniquely named 0600
3905/// sibling and a rename. The temp name does not end in `.json`, so a crash never leaves
3906/// a loadable credential.
3907pub(crate) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
3908    #[cfg(test)]
3909    tests::stall_write(path);
3910    static COUNTER: AtomicU64 = AtomicU64::new(0);
3911    let dir = path.parent().unwrap_or(Path::new("."));
3912    let name = path.file_name().unwrap_or_default().to_string_lossy();
3913    let (tmp, mut file) = loop {
3914        let tmp = dir.join(format!(
3915            ".{name}.{}.{}.tmp",
3916            std::process::id(),
3917            COUNTER.fetch_add(1, Ordering::Relaxed)
3918        ));
3919        // create_new is O_CREAT|O_EXCL: it never follows or reuses an existing path.
3920        let mut options = std::fs::OpenOptions::new();
3921        options.write(true).create_new(true);
3922        // Go's 0600; on Windows Go ignores mode bits and creates the file plainly.
3923        #[cfg(unix)]
3924        std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
3925        match options.open(&tmp) {
3926            Ok(file) => break (tmp, file),
3927            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
3928            Err(e) => return Err(e),
3929        }
3930    };
3931    let result = file
3932        .write_all(bytes)
3933        .and_then(|()| file.sync_all())
3934        .and_then(|()| std::fs::rename(&tmp, path));
3935    if result.is_err() {
3936        let _ = std::fs::remove_file(&tmp);
3937    }
3938    result
3939}
3940
3941/// A response stream that reports its lease's outcome on end, on the first error (after
3942/// which it yields nothing more), or as `Cancelled` when dropped early.
3943///
3944/// A remote lease's cancellation (Go's selection-bound attempt context) is watched by
3945/// its own task: a draining dispatcher closes the upstream body and ends the lease even
3946/// while the client is not reading, and the reader then sees `context canceled`.
3947pub struct Completing {
3948    slot: Arc<Mutex<Slot>>,
3949    watcher: Option<tokio::task::JoinHandle<()>>,
3950}
3951
3952struct Slot {
3953    inner: ExecStream,
3954    lease: Option<Lease>,
3955    /// The watcher ended the lease; the next poll yields the cancellation error.
3956    cancelled: bool,
3957    waker: Option<std::task::Waker>,
3958}
3959
3960impl Completing {
3961    pub fn new(inner: ExecStream, lease: Lease) -> Self {
3962        let cancelled = lease.remote_cancelled();
3963        let slot = Arc::new(Mutex::new(Slot {
3964            inner,
3965            lease: Some(lease),
3966            cancelled: false,
3967            waker: None,
3968        }));
3969        let watcher = cancelled.map(|cancelled| {
3970            let slot = Arc::downgrade(&slot);
3971            tokio::spawn(async move {
3972                cancelled.await;
3973                let Some(slot) = slot.upgrade() else { return };
3974                let (inner, lease, waker) = {
3975                    let mut s = slot.lock().unwrap_or_else(PoisonError::into_inner);
3976                    let Some(lease) = s.lease.take() else { return };
3977                    s.cancelled = true;
3978                    let inner = std::mem::replace(&mut s.inner, Box::pin(futures_util::stream::empty()));
3979                    (inner, lease, s.waker.take())
3980                };
3981                // The upstream body closes first, then the lease ends.
3982                drop(inner);
3983                lease.complete(Outcome::Cancelled);
3984                if let Some(waker) = waker {
3985                    waker.wake();
3986                }
3987            })
3988        });
3989        Self { slot, watcher }
3990    }
3991}
3992
3993impl Drop for Completing {
3994    fn drop(&mut self) {
3995        if let Some(watcher) = &self.watcher {
3996            watcher.abort();
3997        }
3998    }
3999}
4000
4001impl Stream for Completing {
4002    type Item = Result<Bytes, ExecError>;
4003
4004    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
4005        let mut s = self.slot.lock().unwrap_or_else(PoisonError::into_inner);
4006        if std::mem::take(&mut s.cancelled) {
4007            return Poll::Ready(Some(Err(crate::remote::cancelled_error())));
4008        }
4009        if s.lease.is_none() {
4010            return Poll::Ready(None);
4011        }
4012        if self.watcher.is_some() {
4013            s.waker = Some(cx.waker().clone());
4014        }
4015        let item = ready!(s.inner.poll_next_unpin(cx));
4016        match &item {
4017            Some(Ok(_)) => {}
4018            None => s.lease.take().unwrap().complete(Outcome::Success),
4019            Some(Err(e)) => {
4020                // The upstream body closes before the lease ends: a remote lease's release
4021                // must not reach the control plane while the response is still open.
4022                s.inner = Box::pin(futures_util::stream::empty());
4023                s.lease.take().unwrap().complete(Outcome::Failure(e.clone()));
4024            }
4025        }
4026        Poll::Ready(item)
4027    }
4028}
4029
4030/// Go `configuredSignatureCacheEnabled` / `configuredSignatureBypassStrict`
4031/// (internal/api/server_reload.go): `oauth.providers.antigravity.signature-cache-enabled`
4032/// (default true) and `.signature-bypass-strict` (default false), legacy
4033/// `antigravity-signature-*` spellings included. Applied on every publish, as Go
4034/// applies them at startup and on each reload.
4035fn signature_cache_config(cfg: &Config) -> (bool, bool) {
4036    let flag = |key: &str| {
4037        cfg.document
4038            .get("oauth")
4039            .and_then(|o| o.get("providers"))
4040            .and_then(|p| p.get("antigravity"))
4041            .and_then(|a| a.get(key))
4042            .and_then(serde_yaml_ng::Value::as_bool)
4043    };
4044    (
4045        flag("signature-cache-enabled").unwrap_or(true),
4046        flag("signature-bypass-strict").unwrap_or(false),
4047    )
4048}
4049
4050/// Credentials the background refresh considers: enabled auth files, never API-key
4051/// kinds (Go `nextRefreshCheckAt` skips `AuthKind() == apikey` for every provider).
4052/// The access token requests send (Go `authAccessToken`): a Sign in with ChatGPT
4053/// credential's own, else `access_token`, trimmed.
4054fn effective_access_token(c: &Credential) -> &str {
4055    if cpa_core::registry::dynamic::uses_siwc(c) {
4056        return cpa_exec::codex_siwc::access_token(c);
4057    }
4058    c.str("access_token").map_or("", str::trim)
4059}
4060
4061fn refresh_candidate(c: &Credential) -> bool {
4062    !c.disabled && refreshable(c)
4063}
4064
4065/// [`refresh_candidate`] whether or not `c` is disabled.
4066fn refreshable(c: &Credential) -> bool {
4067    matches!(c.source, Source::File(_)) && cpa_core::registry::dynamic::auth_kind(c) != Some("apikey")
4068}
4069
4070/// Test hook at `site` for credential `id` ([`tests::hook`]); nothing otherwise.
4071fn test_hook(site: &'static str, id: &str) {
4072    #[cfg(test)]
4073    tests::hook(site, id);
4074    let _ = (site, id);
4075}
4076
4077#[cfg(test)]
4078pub(crate) mod tests {
4079    use super::*;
4080    #[cfg(unix)]
4081    use std::os::unix::fs::PermissionsExt;
4082
4083    thread_local! {
4084        /// Runs once where [`commit_refresh`] meets an edit made since its snapshot,
4085        /// before it reads the credential again.
4086        pub(super) static AFTER_STALE: std::cell::Cell<Option<Box<dyn FnOnce()>>> = const { std::cell::Cell::new(None) };
4087    }
4088
4089    /// Hook sites ([`test_hook`]) armed to panic once, by site and credential ID, and how
4090    /// often each was reached. Process-wide: refresh jobs run on any worker thread.
4091    static PANICS: Mutex<Vec<(&'static str, String)>> = Mutex::new(Vec::new());
4092    static REACHED: Mutex<Vec<(&'static str, String)>> = Mutex::new(Vec::new());
4093
4094    pub(super) fn hook(site: &'static str, id: &str) {
4095        REACHED.lock().unwrap().push((site, id.to_owned()));
4096        let mut panics = PANICS.lock().unwrap();
4097        if let Some(i) = panics.iter().position(|(s, i)| *s == site && i == id) {
4098            panics.remove(i);
4099            drop(panics);
4100            panic!("injected {site} panic");
4101        }
4102    }
4103
4104    /// The next time `site` is reached for credential `id`, it panics.
4105    fn arm_panic(site: &'static str, id: &str) {
4106        PANICS.lock().unwrap().push((site, id.to_owned()));
4107    }
4108
4109    /// How often `site` was reached for credential `id`.
4110    fn reached(site: &'static str, id: &str) -> usize {
4111        REACHED
4112            .lock()
4113            .unwrap()
4114            .iter()
4115            .filter(|(s, i)| *s == site && i == id)
4116            .count()
4117    }
4118
4119    /// Every Codex credential file a refresh or management write waited to lock, in
4120    /// order (all tests).
4121    pub(crate) static LOCK_ATTEMPTS: Mutex<Vec<std::path::PathBuf>> = Mutex::new(Vec::new());
4122
4123    /// Waits until something waited to lock the Codex file at `path`.
4124    pub(crate) async fn lock_waited(path: &Path) {
4125        let waited = async {
4126            while !LOCK_ATTEMPTS.lock().unwrap().iter().any(|p| p == path) {
4127                tokio::time::sleep(Duration::from_millis(5)).await;
4128            }
4129        };
4130        tokio::time::timeout(Duration::from_secs(5), waited)
4131            .await
4132            .expect("something waits for the credential lock");
4133    }
4134
4135    type AdmitHook = (std::path::PathBuf, Box<dyn FnOnce() + Send>);
4136
4137    /// Runs once, for its file, where a Codex refresh is about to be admitted.
4138    static BEFORE_ADMIT: Mutex<Vec<AdmitHook>> = Mutex::new(Vec::new());
4139
4140    pub(super) fn before_admit(path: &Path) {
4141        let hook = {
4142            let mut hooks = BEFORE_ADMIT.lock().unwrap();
4143            hooks.iter().position(|(p, _)| p == path).map(|i| hooks.remove(i).1)
4144        };
4145        if let Some(hook) = hook {
4146            hook();
4147        }
4148    }
4149
4150    /// Go checks a refresh's `RegistrationEpoch` before anything else in
4151    /// `updateInternal`: a refresh that meets an edit, after which its ID is registered
4152    /// again with other tokens, fails as a stale registration, not as a refresh dropped
4153    /// for changed credentials (which hands the caller the new credential).
4154    #[test]
4155    fn a_refresh_meeting_an_edit_then_a_new_registration_fails_as_stale() {
4156        let dir = std::env::temp_dir().join(format!("cpa-refresh-stale-{}", uuid::Uuid::new_v4()));
4157        std::fs::create_dir_all(&dir).unwrap();
4158        let credential = {
4159            let dir = dir.clone();
4160            move |token: &str| {
4161                let metadata = serde_json::json!({"type":"pp","access_token":token});
4162                let path = dir.join("a.json");
4163                std::fs::write(&path, metadata.to_string()).unwrap();
4164                Credential::from_file(&dir, &path, metadata.as_object().cloned().unwrap()).unwrap()
4165            }
4166        };
4167        let store = Arc::new(CredentialStore::new(vec![credential("a")]));
4168        let (current, registration) = store.get_registered("a.json").unwrap();
4169        // An unrelated edit under the same registration: the refresh meets it as stale.
4170        let mut note = MetadataPatch::default();
4171        note.set.insert("note".into(), "edited".into());
4172        store.apply_patch("a.json", current.revision, &note).unwrap();
4173        AFTER_STALE.with(|hook| {
4174            hook.set(Some(Box::new({
4175                let (store, credential) = (store.clone(), credential.clone());
4176                move || {
4177                    store.reconcile(vec![]);
4178                    store.reconcile(vec![credential("b")]);
4179                }
4180            })))
4181        });
4182        let mut patch = MetadataPatch::default();
4183        patch.set.insert("access_token".into(), "p2".into());
4184        let fence = RefreshFence {
4185            registered: registration,
4186            plugin: false,
4187        };
4188        let committed = commit_refresh(&store, "a.json", &current, fence, &patch, |_| {});
4189        assert!(AFTER_STALE.with(|hook| hook.take()).is_none(), "the hook ran");
4190        assert!(matches!(committed, Err(PatchError::NotFound)), "{committed:?}");
4191        assert_eq!(store.get("a.json").unwrap().metadata["access_token"], "b");
4192        let _ = std::fs::remove_dir_all(&dir);
4193    }
4194
4195    /// Go `nextRefreshCheckAt` skips `AuthKind() == apikey` for every provider. The
4196    /// kinds are Go's `AuthKind` goldens (tests/fixtures/server_go.json `auth_kind`).
4197    #[test]
4198    fn refresh_skips_api_key_kinds_like_go() {
4199        let fixture: serde_json::Value =
4200            serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
4201        let mut kinds = std::collections::BTreeSet::new();
4202        for case in fixture["auth_kind"].as_array().unwrap() {
4203            let meta = serde_json::json!({"type": "kimi"}).as_object().unwrap().clone();
4204            let mut c = Credential::from_file(Path::new("/a"), Path::new("/a/x.json"), meta).unwrap();
4205            c.metadata = case["metadata"].as_object().cloned().unwrap_or_default();
4206            c.attributes = case["attributes"]
4207                .as_object()
4208                .map(|a| {
4209                    a.iter()
4210                        .map(|(k, v)| (k.clone(), v.as_str().unwrap().to_owned()))
4211                        .collect()
4212                })
4213                .unwrap_or_default();
4214            let kind = case["kind"].as_str().unwrap();
4215            kinds.insert(kind);
4216            assert_eq!(refresh_candidate(&c), kind != "apikey", "case {case}");
4217            c.disabled = true;
4218            assert!(!refresh_candidate(&c));
4219        }
4220        assert!(kinds.contains("apikey") && kinds.contains("oauth"), "{kinds:?}");
4221    }
4222
4223    #[test]
4224    fn signature_cache_config_reads_v8_and_legacy_keys() {
4225        let at = |yaml: &str| signature_cache_config(&Config::parse(yaml).unwrap());
4226        assert_eq!(at("{}\n"), (true, false));
4227        assert_eq!(
4228            at("antigravity-signature-cache-enabled: false\nantigravity-signature-bypass-strict: true\n"),
4229            (false, true)
4230        );
4231        assert_eq!(
4232            at("oauth: {providers: {antigravity: {signature-cache-enabled: false, signature-bypass-strict: true}}}\n"),
4233            (false, true)
4234        );
4235        assert_eq!(
4236            at("oauth: {providers: {antigravity: {signature-cache-enabled: null}}}\n"),
4237            (true, false)
4238        );
4239    }
4240
4241    fn cred(id: &str, provider: &str, disabled: bool) -> Credential {
4242        let mut metadata = Map::new();
4243        metadata.insert("type".into(), provider.into());
4244        metadata.insert("disabled".into(), disabled.into());
4245        Credential::from_file(Path::new("/a"), &Path::new("/a").join(id), metadata).unwrap()
4246    }
4247
4248    fn sel(provider: &str) -> Selection {
4249        Selection {
4250            provider: provider.into(),
4251            ..Selection::default()
4252        }
4253    }
4254
4255    #[test]
4256    fn select_round_robins_enabled_credentials_and_honours_exclusions() {
4257        let store = CredentialStore::new(vec![
4258            cred("c1.json", "claude", false),
4259            cred("off.json", "claude", true),
4260            cred("x.json", "codex", false),
4261            cred("c2.json", "claude", false),
4262        ]);
4263        let picks: Vec<String> = (0..4)
4264            .map(|_| store.select(sel("claude")).unwrap().credential.id.clone())
4265            .collect();
4266        assert_eq!(picks, ["c1.json", "c2.json", "c1.json", "c2.json"]);
4267        assert!(store.select(sel("gemini")).is_none());
4268        let only = Selection {
4269            exclude: vec!["c1.json".into()],
4270            ..sel("claude")
4271        };
4272        for _ in 0..3 {
4273            assert_eq!(store.select(only.clone()).unwrap().credential.id, "c2.json");
4274        }
4275    }
4276
4277    /// Sequential cache invalidation of the derived registry: after a prefix change, a
4278    /// disable, a removal or a re-registration the next use rebuilds it, and a request
4279    /// result alone does not. This is not Go's publication fence (7c2133e2, 08e5762a,
4280    /// f75ea073): a build already in flight still returns its older snapshot.
4281    #[test]
4282    fn the_registry_follows_the_latest_credentials() {
4283        let rt = crate::testing::runtime(
4284            Config::parse("").unwrap(),
4285            Vec::new(),
4286            Executors {
4287                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
4288                codex: Default::default(),
4289                devices: Default::default(),
4290                openai: Default::default(),
4291                google: Default::default(),
4292            },
4293        );
4294        let with = |prefix: &str, disabled: bool| {
4295            let mut c = cred("a.json", "claude", disabled);
4296            c.metadata.insert("prefix".into(), prefix.into());
4297            c
4298        };
4299        rt.store.reconcile(vec![with("old", false)]);
4300        let model = cpa_core::registry::dynamic::models_for(&rt.config(), &HashMap::new(), &with("", false), 0)
4301            .first()
4302            .map(|spec| spec.id.clone())
4303            .expect("a Claude catalog model");
4304        let supports = |prefix: &str| rt.registry().client_supports("a.json", &format!("{prefix}/{model}"));
4305        assert!(supports("old"));
4306        let built = rt.registry();
4307        rt.store
4308            .select(Selection::new("claude", &model))
4309            .unwrap()
4310            .complete(Outcome::Success);
4311        assert!(Arc::ptr_eq(&built, &rt.registry()), "a result does not rebuild it");
4312        rt.store.reconcile(vec![with("new", false)]);
4313        assert!(supports("new") && !supports("old"), "a superseded prefix");
4314        rt.store.reconcile(vec![with("new", true)]);
4315        assert!(!rt.registry().has_client("a.json"), "disabled");
4316        rt.store.reconcile(vec![with("new", false)]);
4317        assert!(supports("new"), "enabled again");
4318        rt.store.reconcile(Vec::new());
4319        assert!(!rt.registry().has_client("a.json"), "removed");
4320        rt.store.reconcile(vec![with("fresh", false)]);
4321        assert!(supports("fresh") && !supports("new"), "re-registered");
4322    }
4323
4324    #[test]
4325    fn standalone_admission_reads_attributes_first() {
4326        let mut c = cred("a.json", "claude", false);
4327        c.metadata.insert("prefix".into(), "meta".into());
4328        c.metadata
4329            .insert("excluded_models".into(), serde_json::json!(["claude-sonnet*"]));
4330        c.attributes.insert("prefix".into(), "team".into());
4331        c.attributes.insert("excluded_models".into(), "claude-opus*".into());
4332        let policy = Policy::default();
4333        let admit = |model: &str| {
4334            let s = Selection::new("claude", model);
4335            standalone_admission(&s, &policy)(&c)
4336        };
4337        assert_eq!(
4338            admit("team/claude-sonnet-5(high)").as_deref(),
4339            Some("claude-sonnet-5(high)")
4340        );
4341        assert_eq!(admit("team/claude-opus-5"), None, "attribute exclusions win");
4342        assert_eq!(
4343            admit("meta/claude-sonnet-5"),
4344            None,
4345            "metadata prefix is not authoritative"
4346        );
4347        assert_eq!(admit("claude-sonnet-5").as_deref(), Some("claude-sonnet-5"));
4348        let forced = Policy {
4349            force_model_prefix: true,
4350            ..Policy::default()
4351        };
4352        let s = Selection::new("claude", "claude-sonnet-5");
4353        assert_eq!(standalone_admission(&s, &forced)(&c), None);
4354        assert_eq!(standalone_admission(&Selection::new("codex", "x"), &policy)(&c), None);
4355    }
4356
4357    #[test]
4358    fn patch_persists_atomically_preserves_unknown_fields_and_rejects_stale() {
4359        let dir = std::env::temp_dir().join(format!("cpa-store-{}", std::process::id()));
4360        std::fs::create_dir_all(&dir).unwrap();
4361        let path = dir.join("claude-a.json");
4362        std::fs::write(
4363            &path,
4364            r#"{"type":"claude","access_token":"old","zz_unknown":{"k":[1,2]}}"#,
4365        )
4366        .unwrap();
4367        // A stale temp-file-looking path and a symlink must not be reused or followed.
4368        let victim = dir.join("victim");
4369        std::fs::write(&victim, "keep").unwrap();
4370        #[cfg(unix)]
4371        std::os::unix::fs::symlink(
4372            &victim,
4373            dir.join(format!(".claude-a.json.{}.0.tmp", std::process::id())),
4374        )
4375        .unwrap();
4376        let mut cfg = cpa_core::config::Config::parse("").unwrap();
4377        cfg.auth_dir = dir.clone();
4378        let store = CredentialStore::new(cpa_core::config::credentials::from_auth_dir(&cfg));
4379        let rev = store.get("claude-a.json").unwrap().revision;
4380
4381        let mut patch = MetadataPatch::default();
4382        patch.set.insert("access_token".into(), "new".into());
4383        patch.set.insert("email".into(), "a@x.test".into());
4384        patch.set.insert("disabled".into(), true.into());
4385        let updated = store.apply_patch("claude-a.json", rev, &patch).unwrap();
4386        assert!(updated.revision > rev);
4387        assert!(updated.disabled);
4388        assert_eq!(updated.label, "a@x.test", "derived label follows the new email");
4389        assert_eq!(
4390            std::fs::read_to_string(&path).unwrap(),
4391            "{\"type\":\"claude\",\"access_token\":\"new\",\"zz_unknown\":{\"k\":[1,2]},\"email\":\"a@x.test\",\"disabled\":true}\n"
4392        );
4393        #[cfg(unix)]
4394        assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600);
4395        assert_eq!(
4396            std::fs::read_to_string(&victim).unwrap(),
4397            "keep",
4398            "symlink target untouched"
4399        );
4400        assert!(
4401            store.select(sel("claude")).is_none(),
4402            "patched-disabled credential leaves rotation"
4403        );
4404
4405        assert_eq!(
4406            store.apply_patch("claude-a.json", rev, &patch).unwrap_err(),
4407            PatchError::Stale {
4408                current: updated.revision
4409            }
4410        );
4411        assert_eq!(
4412            store.apply_patch("nope.json", 0, &patch).unwrap_err(),
4413            PatchError::NotFound
4414        );
4415        let mut retype = MetadataPatch::default();
4416        retype.set.insert("type".into(), "codex".into());
4417        assert_eq!(
4418            store
4419                .apply_patch("claude-a.json", updated.revision, &retype)
4420                .unwrap_err(),
4421            PatchError::TypeIsImmutable
4422        );
4423        let leftovers: Vec<_> = std::fs::read_dir(&dir)
4424            .unwrap()
4425            .map(|e| e.unwrap().file_name().into_string().unwrap())
4426            .filter(|n| n.ends_with(".tmp") && !n.ends_with(".0.tmp"))
4427            .collect();
4428        assert!(leftovers.is_empty(), "no temp files left behind: {leftovers:?}");
4429        std::fs::remove_dir_all(&dir).unwrap();
4430    }
4431
4432    /// A plugin save running alongside token refreshes of another credential never
4433    /// reinstalls a stale copy of it: every refresh finds the revision it left.
4434    #[test]
4435    fn upsert_never_undoes_a_concurrent_patch() {
4436        let dir = std::env::temp_dir().join(format!("cpa-upsert-{}", std::process::id()));
4437        let _ = std::fs::remove_dir_all(&dir);
4438        std::fs::create_dir_all(&dir).unwrap();
4439        std::fs::write(dir.join("refreshed.json"), r#"{"type":"claude","refresh_token":"r0"}"#).unwrap();
4440        std::fs::write(dir.join("saved.json"), r#"{"type":"claude"}"#).unwrap();
4441        let mut cfg = cpa_core::config::Config::parse("").unwrap();
4442        cfg.auth_dir = dir.clone();
4443        let store = Arc::new(CredentialStore::new(cpa_core::config::credentials::from_auth_dir(&cfg)));
4444        let saved = (*store.get("saved.json").unwrap()).clone();
4445        let saver = {
4446            let store = store.clone();
4447            std::thread::spawn(move || {
4448                for n in 0..500 {
4449                    let mut next = saved.clone();
4450                    next.metadata.insert("n".into(), n.into());
4451                    store.upsert(next);
4452                }
4453            })
4454        };
4455        let mut revision = store.get("refreshed.json").unwrap().revision;
4456        for n in 1..=500 {
4457            let mut patch = MetadataPatch::default();
4458            patch.set.insert("refresh_token".into(), format!("r{n}").into());
4459            revision = store
4460                .apply_patch("refreshed.json", revision, &patch)
4461                .unwrap_or_else(|e| panic!("refresh {n} was undone: {e:?}"))
4462                .revision;
4463        }
4464        saver.join().unwrap();
4465        let refreshed = store.get("refreshed.json").unwrap();
4466        assert_eq!(refreshed.revision, revision);
4467        assert_eq!(refreshed.metadata["refresh_token"], "r500");
4468        assert_eq!(store.get("saved.json").unwrap().metadata["n"], 499);
4469        assert_eq!(store.snapshot().len(), 2);
4470        std::fs::remove_dir_all(&dir).unwrap();
4471    }
4472
4473    /// Home config updates replacing the config keys never undo a concurrent plugin save
4474    /// or relay session change, and unchanged keys keep their revision.
4475    #[test]
4476    fn replace_config_keeps_concurrent_changes_to_other_credentials() {
4477        let keys = |key: &str| {
4478            let cfg = cpa_core::config::Config::parse(&format!(
4479                "codex-api-key:\n  - api-key: {key}\n    base-url: http://127.0.0.1:9\n"
4480            ))
4481            .unwrap();
4482            cpa_core::config::credentials::from_config(&cfg)
4483        };
4484        let store = Arc::new(CredentialStore::new(keys("sk-a")));
4485        let mut saved = Credential::relay_session("saved");
4486        saved.source = Source::File("saved.json".into());
4487        saved.id = "saved.json".into();
4488        // Each step checks that the previous one survived the concurrent replacements.
4489        let changer = {
4490            let store = store.clone();
4491            std::thread::spawn(move || {
4492                for n in 0..2000 {
4493                    if n > 0 {
4494                        assert_eq!(
4495                            store.get("saved.json").unwrap().metadata["n"],
4496                            n - 1,
4497                            "a save was undone"
4498                        );
4499                        assert!(store.get("relay").is_none(), "a closed relay session came back");
4500                    }
4501                    let mut next = saved.clone();
4502                    next.metadata.insert("n".into(), n.into());
4503                    store.upsert(next);
4504                    store.add_runtime(Credential::relay_session("relay"));
4505                    store.remove_runtime("relay");
4506                }
4507            })
4508        };
4509        let mut n = 0;
4510        while !changer.is_finished() {
4511            store.replace_config(keys(if n % 2 == 0 { "sk-b" } else { "sk-a" }));
4512            n += 1;
4513        }
4514        changer.join().unwrap();
4515        store.replace_config(keys("sk-a"));
4516        let revision = store.snapshot().iter().find(|c| c.id != "saved.json").unwrap().revision;
4517        store.replace_config(keys("sk-a"));
4518        let ids: Vec<_> = store.snapshot().iter().map(|c| (c.id.clone(), c.revision)).collect();
4519        assert_eq!(ids.len(), 2, "{ids:?}");
4520        assert!(ids.contains(&(keys("sk-a")[0].id.clone(), revision)), "{ids:?}");
4521        assert_eq!(store.get("saved.json").unwrap().metadata["n"], 1999);
4522    }
4523
4524    /// The next write of `path` (from any thread) stops inside [`write_bytes_atomic`],
4525    /// before the file changes, until the returned release is sent or dropped.
4526    fn stall_next_write(path: &Path) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
4527        let (entered_tx, entered) = std::sync::mpsc::channel();
4528        let (release, release_rx) = std::sync::mpsc::channel();
4529        STALLS
4530            .lock()
4531            .unwrap()
4532            .insert(path.to_path_buf(), (entered_tx, release_rx));
4533        (entered, release)
4534    }
4535
4536    type Stall = (std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>);
4537    static STALLS: std::sync::LazyLock<Mutex<HashMap<std::path::PathBuf, Stall>>> =
4538        std::sync::LazyLock::new(Mutex::default);
4539
4540    pub(super) fn stall_write(path: &Path) {
4541        let Some((entered, release)) = STALLS.lock().unwrap().remove(path) else {
4542            return;
4543        };
4544        let _ = entered.send(());
4545        let _ = release.recv();
4546    }
4547
4548    /// Runs `f` on its own thread; its result, or `None` while it is still running after
4549    /// `wait`.
4550    fn spawn_bounded<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> std::sync::mpsc::Receiver<T> {
4551        let (tx, rx) = std::sync::mpsc::channel();
4552        std::thread::spawn(move || {
4553            let _ = tx.send(f());
4554        });
4555        rx
4556    }
4557
4558    const BOUND: Duration = Duration::from_secs(5);
4559    const BLOCKED: Duration = Duration::from_millis(300);
4560
4561    fn stalled_store(
4562        name: &str,
4563        files: &[&str],
4564    ) -> (std::path::PathBuf, cpa_core::config::Config, Arc<CredentialStore>) {
4565        let dir = std::env::temp_dir().join(format!("cpa-{name}-{}", std::process::id()));
4566        let _ = std::fs::remove_dir_all(&dir);
4567        std::fs::create_dir_all(&dir).unwrap();
4568        for file in files {
4569            std::fs::write(dir.join(file), r#"{"type":"claude","access_token":"old"}"#).unwrap();
4570        }
4571        let mut cfg = cpa_core::config::Config::parse("").unwrap();
4572        cfg.auth_dir = dir.clone();
4573        let store = CredentialStore::new(cpa_core::config::credentials::from_auth_dir(&cfg));
4574        (dir, cfg, store)
4575    }
4576
4577    fn token_patch(token: &str) -> MetadataPatch {
4578        let mut patch = MetadataPatch::default();
4579        patch.set.insert("access_token".into(), token.into());
4580        patch
4581    }
4582
4583    /// Go 8a01bc02 (`TestManagerBlockedSaveAllowsUnrelatedCredential`,
4584    /// `TestManagerBlockedSaveSerializesLoad`): while a refresh commit's file write is
4585    /// stalled on disk, snapshots and selection proceed, before and while a reload is
4586    /// pending; another credential's write proceeds; and neither the reload nor another
4587    /// write of the same credential overtakes the stalled one. (A reload whose scan read
4588    /// the file before the write is Go c4c1d43e's stale-snapshot check, not ported here.)
4589    #[test]
4590    fn a_stalled_credential_write_blocks_only_its_reload() {
4591        let (dir, cfg, store) = stalled_store("stalled-write", &["a.json", "b.json"]);
4592        let before = store.get("a.json").unwrap().revision;
4593        let (entered, release) = stall_next_write(&dir.join("a.json"));
4594        let writer = spawn_bounded({
4595            let store = store.clone();
4596            move || {
4597                store
4598                    .apply_patch("a.json", before, &token_patch("new"))
4599                    .map(|c| c.revision)
4600            }
4601        });
4602        entered.recv_timeout(BOUND).expect("the write reached the disk");
4603        // Leases stay alive until the end: dropping one records a result and persists.
4604        let mut leases = Vec::new();
4605        let read_and_select = |leases: &mut Vec<Lease>, when: &str| {
4606            let reader = spawn_bounded({
4607                let store = store.clone();
4608                move || store.snapshot().len()
4609            });
4610            assert_eq!(reader.recv_timeout(BOUND).ok(), Some(2), "a snapshot {when}");
4611            let selector = spawn_bounded({
4612                let store = store.clone();
4613                move || {
4614                    store.select(Selection {
4615                        exclude: vec!["a.json".into()],
4616                        ..sel("claude")
4617                    })
4618                }
4619            });
4620            let lease = selector
4621                .recv_timeout(BOUND)
4622                .unwrap_or_else(|_| panic!("a selection {when}"));
4623            leases.push(lease.expect("b.json"));
4624        };
4625        read_and_select(&mut leases, "during the write");
4626        let other = spawn_bounded({
4627            let store = store.clone();
4628            move || {
4629                let revision = store.get("b.json").unwrap().revision;
4630                store.apply_patch("b.json", revision, &token_patch("b2")).map(|_| ())
4631            }
4632        });
4633        assert!(
4634            matches!(other.recv_timeout(BOUND), Ok(Ok(()))),
4635            "another credential's write"
4636        );
4637        // A scan made now still reads A's old file, which is c4c1d43e's stale snapshot;
4638        // this reload carries what the write is about to publish, so the end state shows
4639        // which one landed last.
4640        let mut rescan = cpa_core::config::credentials::from_auth_dir(&cfg);
4641        let in_scan = rescan.iter_mut().find(|c| c.id == "a.json").unwrap();
4642        in_scan.metadata.insert("access_token".into(), "new".into());
4643        in_scan.metadata.insert("reloaded".into(), true.into());
4644        let reload = spawn_bounded({
4645            let store = store.clone();
4646            move || store.reconcile(rescan)
4647        });
4648        assert!(
4649            reload.recv_timeout(BLOCKED).is_err(),
4650            "the reload overtook the stalled write"
4651        );
4652        read_and_select(&mut leases, "while a reload is pending");
4653        let same = spawn_bounded({
4654            let store = store.clone();
4655            move || store.apply_patch("a.json", before, &token_patch("other")).map(|_| ())
4656        });
4657        assert!(
4658            same.recv_timeout(BLOCKED).is_err(),
4659            "a write of the same credential overtook it"
4660        );
4661        release.send(()).unwrap();
4662        let published = writer
4663            .recv_timeout(BOUND)
4664            .expect("the write finished")
4665            .expect("and published");
4666        assert!(published > before);
4667        reload.recv_timeout(BOUND).expect("the reload finished");
4668        assert!(
4669            matches!(same.recv_timeout(BOUND), Ok(Err(PatchError::Stale { .. }))),
4670            "the same-credential write saw the published revision"
4671        );
4672        drop(leases);
4673        let file: Map<String, Value> = serde_json::from_slice(&std::fs::read(dir.join("a.json")).unwrap()).unwrap();
4674        assert_eq!(file["access_token"], "new");
4675        let a = store.get("a.json").unwrap();
4676        assert_eq!(
4677            a.metadata.get("reloaded"),
4678            Some(&Value::Bool(true)),
4679            "the reload landed last"
4680        );
4681        assert!(a.revision > published);
4682        assert_eq!(store.get("b.json").unwrap().metadata["access_token"], "b2");
4683        std::fs::remove_dir_all(&dir).unwrap();
4684    }
4685
4686    /// An edit that renames credential A to B's ID replaces B, so it must not run while
4687    /// a write of B is between its revision check and its publication.
4688    #[test]
4689    fn an_identity_change_waits_for_a_write_of_the_credential_it_replaces() {
4690        let (dir, _cfg, store) = stalled_store("identity-change", &["a.json", "b.json"]);
4691        let b = store.get("b.json").unwrap().revision;
4692        let a = store.get("a.json").unwrap();
4693        let (entered, release) = stall_next_write(&dir.join("b.json"));
4694        let writer = spawn_bounded({
4695            let store = store.clone();
4696            move || store.apply_patch("b.json", b, &token_patch("b2")).map(|c| c.revision)
4697        });
4698        entered.recv_timeout(BOUND).expect("B's write reached the disk");
4699        let mut renamed = Credential::clone(&a);
4700        renamed.id = "b.json".into();
4701        renamed.metadata.insert("access_token".into(), "renamed".into());
4702        let replace = spawn_bounded({
4703            let store = store.clone();
4704            move || {
4705                let bytes = br#"{"type":"claude","access_token":"renamed"}"#;
4706                store.replace_file_backed("a.json", a.revision, bytes, Some(renamed))
4707            }
4708        });
4709        assert!(replace.recv_timeout(BLOCKED).is_err(), "the rename overtook B's write");
4710        release.send(()).unwrap();
4711        let written = writer.recv_timeout(BOUND).expect("B's write finished");
4712        assert!(written.is_ok(), "B's write was published, not dropped: {written:?}");
4713        assert!(matches!(replace.recv_timeout(BOUND), Ok(Ok(()))));
4714        let ids: Vec<String> = store.snapshot().iter().map(|c| c.id.clone()).collect();
4715        assert_eq!(ids, ["b.json"]);
4716        assert_eq!(store.get("b.json").unwrap().metadata["access_token"], "renamed");
4717        std::fs::remove_dir_all(&dir).unwrap();
4718    }
4719
4720    /// Go c4c1d43e `TestFileSnapshotCannotOverwriteLaterPersistedPrefix`: a scan that
4721    /// read a file before a later write cannot undo that write; a real edit still
4722    /// applies; a file gone or unreadable since the scan keeps the current credential.
4723    #[test]
4724    fn a_stale_scan_cannot_undo_a_later_write() {
4725        let dir = std::env::temp_dir().join(format!("cpa-stale-scan-{}", std::process::id()));
4726        let _ = std::fs::remove_dir_all(&dir);
4727        std::fs::create_dir_all(&dir).unwrap();
4728        let path = dir.join("synthetic.json");
4729        let write = |prefix: &str| std::fs::write(&path, format!(r#"{{"type":"codex","prefix":"{prefix}"}}"#)).unwrap();
4730        let mut cfg = cpa_core::config::Config::parse("").unwrap();
4731        cfg.auth_dir = dir.clone();
4732        let scan = || cpa_core::config::credentials::scan_auth_dir(&cfg);
4733        write("pfx-8");
4734        let store = CredentialStore::new(scan().credentials);
4735        let stale = scan();
4736        let revision = store.get("synthetic.json").unwrap().revision;
4737        let mut patch = MetadataPatch::default();
4738        patch.set.insert("prefix".into(), "pfx-9".into());
4739        store.apply_patch("synthetic.json", revision, &patch).unwrap();
4740        let current = || store.get("synthetic.json").unwrap();
4741        let before = current();
4742        store.reconcile_scanned(stale.credentials, &stale.read);
4743        assert_eq!(
4744            current().metadata["prefix"],
4745            "pfx-9",
4746            "the stale scan overwrote the write"
4747        );
4748        assert_eq!(current().revision, before.revision);
4749        write("manual-edit");
4750        let manual = scan();
4751        store.reconcile_scanned(manual.credentials, &manual.read);
4752        assert_eq!(current().metadata["prefix"], "manual-edit", "a real edit was lost");
4753        let before = current();
4754        write("deleted-edit");
4755        let deleted = scan();
4756        std::fs::remove_file(&path).unwrap();
4757        store.reconcile_scanned(deleted.credentials.clone(), &deleted.read);
4758        assert_eq!(
4759            current().revision,
4760            before.revision,
4761            "a deleted file's snapshot was applied"
4762        );
4763        std::fs::create_dir(&path).unwrap();
4764        store.reconcile_scanned(deleted.credentials.clone(), &deleted.read);
4765        assert_eq!(
4766            current().revision,
4767            before.revision,
4768            "an unreadable file's snapshot was applied"
4769        );
4770        // A FIFO in its place is not opened (reading one blocks until a writer appears).
4771        #[cfg(unix)]
4772        {
4773            std::fs::remove_dir(&path).unwrap();
4774            let fifo = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
4775            // SAFETY: valid NUL-terminated path in this test's private directory.
4776            assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o600) }, 0);
4777            let done = spawn_bounded({
4778                let store = store.clone();
4779                move || store.reconcile_scanned(deleted.credentials, &deleted.read)
4780            });
4781            done.recv_timeout(BOUND).expect("the freshness check opened the FIFO");
4782            assert_eq!(current().revision, before.revision, "a FIFO's snapshot was applied");
4783        }
4784        std::fs::remove_dir_all(&dir).unwrap();
4785    }
4786
4787    #[test]
4788    fn reconcile_keeps_unchanged_revisions_and_never_reuses_old_ones() {
4789        let store = CredentialStore::new(vec![cred("a.json", "claude", false), cred("b.json", "claude", false)]);
4790        let a = store.get("a.json").unwrap();
4791        let b = store.get("b.json").unwrap();
4792        store.reconcile(vec![cred("a.json", "claude", false)]);
4793        assert!(
4794            Arc::ptr_eq(&store.get("a.json").unwrap(), &a),
4795            "unchanged credential kept as is"
4796        );
4797        assert!(store.get("b.json").is_none());
4798        store.reconcile(vec![cred("a.json", "claude", false), cred("b.json", "claude", false)]);
4799        assert!(
4800            store.get("b.json").unwrap().revision > b.revision,
4801            "re-created credential gets a new revision"
4802        );
4803    }
4804
4805    #[test]
4806    fn attribute_only_reload_changes_priority_and_rejects_stale_outcome() {
4807        let a = cred("a.json", "claude", false);
4808        let mut b = cred("b.json", "claude", false);
4809        b.attributes.insert("priority".into(), "1".into());
4810        let store = CredentialStore::new(vec![a.clone(), b.clone()]);
4811        let old = store.select(sel("claude")).unwrap();
4812        assert_eq!(old.credential.id, "b.json");
4813        let revision = old.credential.revision;
4814        b.attributes.insert("priority".into(), "2".into());
4815        store.reconcile(vec![a, b]);
4816        assert!(store.get("b.json").unwrap().revision > revision);
4817        old.complete(Outcome::Failure(ExecError::local(
4818            401,
4819            FailureScope::Credential,
4820            "expired",
4821        )));
4822        let current = store.select(sel("claude")).unwrap();
4823        assert_eq!(
4824            current.credential.id, "b.json",
4825            "stale outcome must not cool new revision"
4826        );
4827        assert_eq!(current.credential.attributes["priority"], "2");
4828        current.complete(Outcome::Success);
4829        assert_eq!(store.stats().failure, 1);
4830        assert_eq!(store.stats().success, 1);
4831    }
4832
4833    #[test]
4834    fn stale_lease_cannot_poison_deleted_and_recreated_credential() {
4835        let credential = cred("a.json", "claude", false);
4836        let store = CredentialStore::new(vec![credential.clone()]);
4837        let old = store.select(sel("claude")).unwrap();
4838        store.reconcile(Vec::new());
4839        store.reconcile(vec![credential]);
4840        old.complete(Outcome::Failure(ExecError::local(
4841            429,
4842            FailureScope::Credential,
4843            "quota",
4844        )));
4845        store.select(sel("claude")).unwrap().complete(Outcome::Success);
4846        assert_eq!(store.stats().failure, 1);
4847        assert_eq!(store.stats().success, 1);
4848    }
4849
4850    /// Go `TestRetryIntervalFiltersCooldownCredentials` and
4851    /// `TestRetryRoundAvailabilityRejectsStaleQuotaForNonRetryableStatus`: the next round
4852    /// waits for the earliest cooldown within max-retry-interval, skipping longer ones,
4853    /// and a credential cooling after a 402 or 404 never permits a round, whatever its
4854    /// deadline, while one cooling after a 429 does.
4855    #[test]
4856    fn retry_wait_skips_cooldowns_beyond_the_cap_and_non_retryable_statuses() {
4857        let now = Instant::now();
4858        let selection = sel("claude");
4859        let admit_all = |_: &Credential| Some(String::new());
4860        // One credential per (status, retry-after seconds), each cooling from `now`.
4861        let wait = |cap: u64, cooldowns: &[(u16, u64)]| {
4862            let policy = Policy {
4863                request_retry: 1,
4864                max_retry_interval: Duration::from_secs(cap),
4865                ..Policy::default()
4866            };
4867            let ids: Vec<String> = (0..cooldowns.len()).map(|i| format!("c{i}.json")).collect();
4868            let store = CredentialStore::new(ids.iter().map(|id| cred(id, "claude", false)).collect());
4869            for (id, (status, seconds)) in ids.iter().zip(cooldowns) {
4870                let mut error = ExecError::local(*status, FailureScope::Model, "cooling");
4871                error.retry_after = Some(Duration::from_secs(*seconds));
4872                let credential = store.get(id).unwrap();
4873                store.scheduler.lock().unwrap().record(
4874                    &credential,
4875                    &selection.model,
4876                    &Outcome::Failure(error),
4877                    &policy,
4878                    now,
4879                );
4880            }
4881            store.retry_wait_at(&selection, &policy, 429, &[], &admit_all, now)
4882        };
4883        let short_and_long = wait(30, &[(429, 10), (429, 60)]);
4884        assert!(
4885            short_and_long.is_some_and(|w| !w.is_zero() && w <= Duration::from_secs(10)),
4886            "{short_and_long:?}"
4887        );
4888        assert_eq!(wait(30, &[(429, 60)]), None, "only a cooldown beyond the cap");
4889        assert!(wait(3600, &[(429, 60)]).is_some(), "rate limit");
4890        assert_eq!(wait(3600, &[(402, 60)]), None, "payment required");
4891        assert_eq!(wait(3600, &[(404, 60)]), None, "not found");
4892    }
4893
4894    #[test]
4895    fn retry_wait_obeys_exact_cap_and_attempted_quota_floor() {
4896        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
4897        let now = Instant::now();
4898        let mut selection = sel("claude");
4899        let admit_all = |_: &Credential| Some(String::new());
4900        let tried = ["a.json".to_owned()];
4901        let mut policy = Policy {
4902            request_retry: 1,
4903            ..Policy::default()
4904        };
4905        // A transport fault (status 0) retries immediately.
4906        assert_eq!(
4907            store.retry_wait_at(&selection, &policy, 0, &tried, &admit_all, now),
4908            Some(Duration::ZERO)
4909        );
4910        // An attempted credential after a 429 waits at least 10s, which the default
4911        // max-retry-interval of 0 forbids; an untried one may go immediately.
4912        assert_eq!(
4913            store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, now),
4914            None
4915        );
4916        assert_eq!(
4917            store.retry_wait_at(&selection, &policy, 429, &[], &admit_all, now),
4918            Some(Duration::ZERO)
4919        );
4920        policy.max_retry_interval = Duration::from_secs(10);
4921        assert_eq!(
4922            store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, now),
4923            Some(Duration::from_secs(10))
4924        );
4925        let quota = ExecError::local(429, FailureScope::Model, "quota");
4926        let credential = store.get("a.json").unwrap();
4927        store.scheduler.lock().unwrap().record(
4928            &credential,
4929            &selection.model,
4930            &Outcome::Failure(quota.clone()),
4931            &policy,
4932            now,
4933        );
4934        let later = now + Duration::from_millis(1);
4935        assert_eq!(
4936            store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, later),
4937            Some(Duration::from_secs(10))
4938        );
4939        policy.max_retry_interval = Duration::from_secs(9);
4940        assert_eq!(
4941            store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, later),
4942            None
4943        );
4944        policy.disable_cooling = true;
4945        store.scheduler.lock().unwrap().record(
4946            &credential,
4947            &selection.model,
4948            &Outcome::Failure(quota.clone()),
4949            &policy,
4950            later,
4951        );
4952        policy.max_retry_interval = Duration::ZERO;
4953        assert_eq!(
4954            store.retry_wait_at(&selection, &policy, 429, &tried, &admit_all, later),
4955            Some(Duration::ZERO),
4956            "disabled cooling has no floor"
4957        );
4958        selection.retry_round = 1;
4959        assert_eq!(
4960            store.retry_wait_at(&selection, &policy, 0, &tried, &admit_all, later),
4961            None
4962        );
4963    }
4964
4965    /// A 429 with a reset six days away (`max-trusted-cooldown` bounds it).
4966    fn weekly_limit(scope: FailureScope) -> Outcome {
4967        let mut error = ExecError::local(429, scope, "usage_limit_reached");
4968        error.retry_after = Some(Duration::from_secs(6 * 24 * 3600));
4969        Outcome::Failure(error)
4970    }
4971
4972    /// A probe whose hold lapsed was taken over by a later pick: the first probe's
4973    /// cancellation leaves the account reserved for the second, whose own does not.
4974    #[test]
4975    fn a_stale_probe_cancellation_keeps_the_newer_reservation() {
4976        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
4977        let selection = Selection::new("claude", "m");
4978        let credential = store.get("a.json").unwrap();
4979        // A bounded window of the 10 s minimum that ended a second ago.
4980        let policy = Policy {
4981            max_trusted_cooldown: Duration::from_secs(10),
4982            ..Policy::default()
4983        };
4984        let opened = Instant::now().checked_sub(Duration::from_secs(11)).unwrap();
4985        store
4986            .scheduler
4987            .lock()
4988            .unwrap()
4989            .record(&credential, "m", &weekly_limit(FailureScope::Model), &policy, opened);
4990        let first = store.select(selection.clone()).expect("the probe");
4991        assert_eq!(first.execution_model, "m");
4992        assert!(store.select(selection.clone()).is_none(), "reserved");
4993        // The first probe's hold lapses, and the next pick probes again.
4994        let key = ("a.json".to_owned(), "m".to_owned());
4995        store
4996            .scheduler
4997            .lock()
4998            .unwrap()
4999            .cooldowns
5000            .get_mut(&key)
5001            .unwrap()
5002            .trust
5003            .probe = Some(Instant::now());
5004        let second = store.select(selection.clone()).expect("the second probe");
5005        first.complete(Outcome::Cancelled);
5006        assert!(store.select(selection.clone()).is_none(), "still reserved");
5007        second.complete(Outcome::Cancelled);
5008        assert!(store.select(selection).is_some(), "its own cancellation frees it");
5009    }
5010
5011    /// Go 35af74f9 (TestManager_Issue6643_InFlightSuccessesKeepFreshQuotaCooldown):
5012    /// successes of requests dispatched before a 429 was recorded, whether picked by
5013    /// selection or by the credits fallback outside it, leave the cooldown; a lease
5014    /// dispatched again after it (a refresh-and-retry) clears it.
5015    #[tokio::test]
5016    async fn successes_dispatched_before_a_cooldown_keep_it() {
5017        let rt = Arc::new(crate::testing::runtime(
5018            Config::parse("").unwrap(),
5019            vec![cred("a.json", "claude", false)],
5020            Executors {
5021                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5022                codex: Default::default(),
5023                devices: Default::default(),
5024                openai: Default::default(),
5025                google: Default::default(),
5026            },
5027        ));
5028        let selection = Selection::new("claude", "m");
5029        let cfg = rt.config();
5030        let in_flight = rt.store.select(selection.clone()).unwrap();
5031        let credits = rt
5032            .lease_credential("a.json", selection.clone(), &cfg, Credential::clone)
5033            .await
5034            .unwrap();
5035        let mut retried = rt.store.select(selection.clone()).unwrap();
5036        let mut quota = ExecError::local(429, FailureScope::Model, "rate limited");
5037        quota.retry_after = Some(Duration::from_secs(3600));
5038        rt.store
5039            .select(selection.clone())
5040            .unwrap()
5041            .complete(Outcome::Failure(quota));
5042        assert!(rt.store.select(selection.clone()).is_none(), "cooling");
5043        in_flight.complete(Outcome::Success);
5044        assert!(rt.store.select(selection.clone()).is_none(), "a stale success keeps it");
5045        credits.complete(Outcome::Success);
5046        assert!(
5047            rt.store.select(selection.clone()).is_none(),
5048            "a stale credits success keeps it"
5049        );
5050        retried.redispatch();
5051        retried.complete(Outcome::Success);
5052        assert!(rt.store.select(selection).is_some(), "a later dispatch clears it");
5053    }
5054
5055    /// A request picked before a bounded window opened succeeds after it (a stream that
5056    /// outlived the 429): its answer is older than the window, which stays, live or
5057    /// ended, until its own probe.
5058    #[test]
5059    fn a_late_success_keeps_a_newer_window() {
5060        let selection = Selection::new("claude", "m");
5061        // A model window, opened by a request picked after the stream.
5062        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
5063        let stream = store.select(selection.clone()).unwrap();
5064        let other = store.select(selection.clone()).unwrap();
5065        other.complete(weekly_limit(FailureScope::Model));
5066        assert!(store.select(selection.clone()).is_none(), "cooling");
5067        stream.accepted();
5068        stream.complete(Outcome::Success);
5069        assert!(store.select(selection.clone()).is_none(), "the window stands");
5070        // A credential-wide window that already ended keeps its probe.
5071        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
5072        let stream = store.select(selection.clone()).unwrap();
5073        let policy = Policy {
5074            max_trusted_cooldown: Duration::from_secs(10),
5075            ..Policy::default()
5076        };
5077        let opened = Instant::now().checked_sub(Duration::from_secs(11)).unwrap();
5078        store.scheduler.lock().unwrap().record(
5079            &store.get("a.json").unwrap(),
5080            "m",
5081            &weekly_limit(FailureScope::Credential),
5082            &policy,
5083            opened,
5084        );
5085        stream.accepted();
5086        stream.complete(Outcome::Success);
5087        let _probe = store.select(selection.clone()).expect("the window's probe");
5088        assert!(store.select(selection).is_none(), "one probe");
5089    }
5090
5091    #[test]
5092    fn config_routing_drives_policy_at_startup_and_on_publish() {
5093        let executors = || Executors {
5094            claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5095            codex: Default::default(),
5096            devices: Default::default(),
5097            openai: Default::default(),
5098            google: Default::default(),
5099        };
5100        // Legacy top-level keys and the canonical routing block both reach the scheduler.
5101        let rt = crate::testing::runtime(
5102            Config::parse("request-retry: 2\nrouting:\n  strategy: ff\n  session-affinity-ttl: 250ms\n").unwrap(),
5103            Vec::new(),
5104            executors(),
5105        );
5106        let policy = rt.policy();
5107        assert_eq!(policy.strategy, crate::scheduler::Strategy::FillFirst);
5108        assert_eq!(policy.request_retry, 2);
5109        assert_eq!(policy.session_affinity_ttl, Duration::from_secs(1));
5110
5111        rt.publish_config(Config::parse("routing:\n  strategy: wrr\n").unwrap());
5112        let policy = rt.policy();
5113        assert_eq!(policy.strategy, crate::scheduler::Strategy::WeightedRoundRobin);
5114        assert_eq!(policy.request_retry, 0);
5115        assert_eq!(policy.session_affinity_ttl, Duration::from_secs(3600));
5116    }
5117
5118    /// Go `nextModelPoolOffset` (conductor_models.go): per-key cursor, no advance for
5119    /// single-entry pools or blank keys, reset to zero at the int32 guard.
5120    #[test]
5121    fn pool_offsets_rotate_per_key_like_go() {
5122        let rt = crate::testing::runtime(
5123            Config::parse("").unwrap(),
5124            Vec::new(),
5125            Executors {
5126                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5127                codex: Default::default(),
5128                devices: Default::default(),
5129                openai: Default::default(),
5130                google: Default::default(),
5131            },
5132        );
5133        let offsets: Vec<usize> = (0..4).map(|_| rt.next_pool_offset("a|openai|m", 3)).collect();
5134        assert_eq!(offsets, [0, 1, 2, 0]);
5135        assert_eq!(rt.next_pool_offset(" a|openai|m ", 3), 1, "keys are trimmed");
5136        assert_eq!(rt.next_pool_offset("b|openai|m", 3), 0, "keys rotate independently");
5137        assert_eq!(rt.next_pool_offset("c", 1), 0);
5138        assert_eq!(rt.next_pool_offset("c", 2), 0, "a single-entry pool does not advance");
5139        assert_eq!(rt.next_pool_offset("  ", 2), 0);
5140        rt.pool_offsets.lock().unwrap().insert("d".into(), 2_147_483_641);
5141        assert_eq!(rt.next_pool_offset("d", 3), 0, "the guard resets before use");
5142        assert_eq!(rt.next_pool_offset("d", 3), 1);
5143    }
5144
5145    /// Go `Manager.persist` after `MarkResult`: the auth file gains `"disabled": false`
5146    /// in `json.Marshal` form (sorted keys, float64 numbers, HTML escaping, no newline)
5147    /// once, and is not rewritten while it already holds that JSON.
5148    #[test]
5149    fn first_use_persists_disabled_in_go_marshal_form() {
5150        let dir = std::env::temp_dir().join(format!(
5151            "persist-{}-{}",
5152            std::process::id(),
5153            std::time::SystemTime::now()
5154                .duration_since(std::time::UNIX_EPOCH)
5155                .unwrap()
5156                .as_nanos()
5157        ));
5158        std::fs::create_dir_all(&dir).unwrap();
5159        let path = dir.join("a.json");
5160        let original = "{\"type\":\"claude\",\"email\":\"a<b>@x.invalid\",\"n\":1.50,\"big\":1e3,\"nested\":{\"z\":1,\"a\":[true,null]}}\n";
5161        std::fs::write(&path, original).unwrap();
5162        let metadata: Map<String, Value> = serde_json::from_str(original).unwrap();
5163        let store = CredentialStore::new(vec![Credential::from_file(&dir, &path, metadata).unwrap()]);
5164        let revision = store.get("a.json").unwrap().revision;
5165        store
5166            .select(Selection::new("claude", "m"))
5167            .unwrap()
5168            .complete(Outcome::Success);
5169        let written = std::fs::read_to_string(&path).unwrap();
5170        assert_eq!(
5171            written,
5172            r#"{"big":1000,"disabled":false,"email":"a\u003cb\u003e@x.invalid","n":1.5,"nested":{"a":[true,null],"z":1},"type":"claude"}"#
5173        );
5174        let current = store.get("a.json").unwrap();
5175        assert_eq!(current.revision, revision, "a neutral write keeps the revision");
5176        assert_eq!(current.metadata.get("disabled"), Some(&Value::Bool(false)));
5177        assert_eq!(current.metadata.get("big"), Some(&Value::from(1000)));
5178        // The watcher's reload of the written file changes nothing.
5179        let reloaded: Map<String, Value> = serde_json::from_str(&written).unwrap();
5180        store.reconcile(vec![Credential::from_file(&dir, &path, reloaded).unwrap()]);
5181        assert_eq!(store.get("a.json").unwrap().revision, revision);
5182        // Same JSON in another layout is left alone.
5183        std::fs::write(&path, "{\"type\":\"claude\", \"disabled\":false}").unwrap();
5184        let mut metadata = Map::new();
5185        metadata.insert("type".into(), "claude".into());
5186        metadata.insert("disabled".into(), false.into());
5187        store.reconcile(vec![Credential::from_file(&dir, &path, metadata).unwrap()]);
5188        store
5189            .select(Selection::new("claude", "m"))
5190            .unwrap()
5191            .complete(Outcome::Success);
5192        assert_eq!(
5193            std::fs::read_to_string(&path).unwrap(),
5194            "{\"type\":\"claude\", \"disabled\":false}"
5195        );
5196        let _ = std::fs::remove_dir_all(&dir);
5197    }
5198
5199    /// A result recorded after a refresh committed a newer revision must not write the
5200    /// older token back (Go persists under the manager lock with generation checks).
5201    #[test]
5202    fn stale_result_does_not_overwrite_a_refreshed_credential() {
5203        let dir = std::env::temp_dir().join(format!(
5204            "stale-{}-{}",
5205            std::process::id(),
5206            std::time::SystemTime::now()
5207                .duration_since(std::time::UNIX_EPOCH)
5208                .unwrap()
5209                .as_nanos()
5210        ));
5211        std::fs::create_dir_all(&dir).unwrap();
5212        let path = dir.join("a.json");
5213        std::fs::write(&path, r#"{"type":"claude","access_token":"old"}"#).unwrap();
5214        let metadata: Map<String, Value> = serde_json::from_str(r#"{"type":"claude","access_token":"old"}"#).unwrap();
5215        let store = CredentialStore::new(vec![Credential::from_file(&dir, &path, metadata).unwrap()]);
5216        let stale = store.get("a.json").unwrap();
5217        let mut patch = MetadataPatch::default();
5218        patch.set.insert("access_token".into(), "new".into());
5219        store.apply_patch("a.json", stale.revision, &patch).unwrap();
5220        let refreshed = std::fs::read_to_string(&path).unwrap();
5221        store.persist_credential(&stale);
5222        assert_eq!(std::fs::read_to_string(&path).unwrap(), refreshed);
5223        assert!(refreshed.contains("\"new\"") && refreshed.contains("\"disabled\":false"));
5224        let _ = std::fs::remove_dir_all(&dir);
5225    }
5226
5227    /// Go `Manager.persist` skips plugin-virtual auths: a refresh of one of the
5228    /// credentials a plugin parsed from one file commits in memory and leaves the shared
5229    /// file alone.
5230    #[test]
5231    fn a_plugin_virtual_refresh_never_rewrites_the_shared_source_file() {
5232        let dir = std::env::temp_dir().join(format!("virtual-{}-{}", std::process::id(), line!()));
5233        std::fs::create_dir_all(&dir).unwrap();
5234        let path = dir.join("multi.json");
5235        let source = r#"{"type":"claude","accounts":[{"access_token":"a"},{"access_token":"b"}]}"#;
5236        std::fs::write(&path, source).unwrap();
5237        let metadata: Map<String, Value> = serde_json::from_str(r#"{"type":"claude","access_token":"a"}"#).unwrap();
5238        let mut credential = Credential::from_file(&dir, &path, metadata).unwrap();
5239        credential.id = "multi-a".into();
5240        credential.attributes.insert("plugin_virtual".into(), "true".into());
5241        let store = CredentialStore::new(vec![credential]);
5242        let current = store.get("multi-a").unwrap();
5243        let mut patch = MetadataPatch::default();
5244        patch.set.insert("access_token".into(), "refreshed".into());
5245        let committed = store.apply_patch("multi-a", current.revision, &patch).unwrap();
5246        assert_eq!(committed.metadata["access_token"], "refreshed");
5247        assert_eq!(store.get("multi-a").unwrap().metadata["access_token"], "refreshed");
5248        store.persist_credential(&committed);
5249        assert_eq!(std::fs::read_to_string(&path).unwrap(), source);
5250        let _ = std::fs::remove_dir_all(&dir);
5251    }
5252
5253    /// Go `save-cooldown-status`: a cooldown change writes `<auth-dir>/<file>.cds`, a new
5254    /// process restores it, reset clears it, and a file Go wrote restores too.
5255    #[test]
5256    fn cooldowns_persist_restore_and_reset_through_cds_files() {
5257        let dir = std::env::temp_dir().join(format!(
5258            "cds-rt-{}-{}",
5259            std::process::id(),
5260            std::time::SystemTime::now()
5261                .duration_since(std::time::UNIX_EPOCH)
5262                .unwrap()
5263                .as_nanos()
5264        ));
5265        std::fs::create_dir_all(&dir).unwrap();
5266        let config = || {
5267            Config::parse(&format!(
5268                "auth-dir: {}\nrouting:\n  cooldown:\n    save-cooldown-status: true\n",
5269                dir.display()
5270            ))
5271            .unwrap()
5272        };
5273        let creds = || {
5274            let metadata = |p: &str| {
5275                let mut m = Map::new();
5276                m.insert("type".into(), p.into());
5277                m
5278            };
5279            vec![
5280                Credential::from_file(&dir, &dir.join("a.json"), metadata("claude")).unwrap(),
5281                Credential::from_file(&dir, &dir.join("b.json"), metadata("claude")).unwrap(),
5282            ]
5283        };
5284        let executors = || Executors {
5285            claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5286            codex: Default::default(),
5287            openai: Default::default(),
5288            google: Default::default(),
5289            devices: Default::default(),
5290        };
5291        let rt = crate::testing::runtime(config(), creds(), executors());
5292        let mut selection = Selection::new("claude", "m1");
5293        selection.exclude.push("b.json".into());
5294        let lease = rt.store.select(selection.clone()).unwrap();
5295        let mut quota = ExecError::local(429, FailureScope::Model, "rate limited");
5296        quota.retry_after = Some(Duration::from_secs(120));
5297        lease.complete(Outcome::Failure(quota));
5298        let file = dir.join("a.cds");
5299        let written = std::fs::read_to_string(&file).expect("cooldown written");
5300        #[cfg(unix)]
5301        assert_eq!(
5302            std::fs::metadata(&file).unwrap().permissions().mode() & 0o777,
5303            0o600,
5304            "Go os.CreateTemp mode"
5305        );
5306        assert!(
5307            written.contains("\"model\": \"m1\"") && written.contains("\"reason\": \"quota\""),
5308            "{written}"
5309        );
5310        assert!(!dir.join("b.cds").exists());
5311        // A successful attempt on another credential changes no cooldown: no rewrite.
5312        let modified = std::fs::metadata(&file).unwrap().modified().unwrap();
5313        let mut other = Selection::new("claude", "m1");
5314        other.exclude.push("a.json".into());
5315        rt.store.select(other).unwrap().complete(Outcome::Success);
5316        assert_eq!(std::fs::metadata(&file).unwrap().modified().unwrap(), modified);
5317
5318        // A new process restores the cooldown.
5319        drop(rt);
5320        let rt = crate::testing::runtime(config(), creds(), executors());
5321        let a = rt.store.get("a.json").unwrap();
5322        let wait = rt
5323            .store
5324            .scheduler
5325            .lock()
5326            .unwrap()
5327            .wait(&a, "m1", Instant::now())
5328            .unwrap();
5329        assert!(
5330            wait > Duration::from_secs(100) && wait <= Duration::from_secs(120),
5331            "{wait:?}"
5332        );
5333        assert_eq!(rt.store.reset_cooldown("a.json"), ["m1"]);
5334        assert!(!file.exists(), "reset removes the stale file");
5335
5336        // A file CLIProxyAPI wrote (local time with an offset) restores model and
5337        // credential-wide quota records; an expired record is dropped on rewrite.
5338        let at = |secs: i64| {
5339            (chrono::Utc::now() + chrono::Duration::seconds(secs))
5340                .with_timezone(&chrono::FixedOffset::east_opt(7200).unwrap())
5341                .to_rfc3339_opts(chrono::SecondsFormat::Nanos, false)
5342        };
5343        let go = format!(
5344            r#"{{"version":1,"auth_id":"b.json","provider":"claude","updated_at":"{now}","records":[
5345{{"provider":"claude","auth_id":"b.json","status":"cooling","next_retry_after":"{late}","reason":"credential_quota","quota":{{"exceeded":true,"reason":"credential_quota","next_recover_at":"{late}","observed_at":"0001-01-01T00:00:00Z"}},"last_error":{{"message":"credential quota","retryable":false,"http_status":429}},"updated_at":"{now}"}},
5346{{"provider":"claude","auth_id":"b.json","model":"m2","status":"cooling","next_retry_after":"{soon}","reason":"unauthorized","quota":{{"exceeded":false,"next_recover_at":"0001-01-01T00:00:00Z","observed_at":"0001-01-01T00:00:00Z"}},"last_error":{{"message":"unauthorized","retryable":false,"http_status":401}},"updated_at":"{now}"}},
5347{{"provider":"claude","auth_id":"b.json","model":"m4","status":"cooling","next_retry_after":"{soon}","reason":"quota","quota":{{"exceeded":true,"reason":"quota","next_recover_at":"{late}","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}},
5348{{"provider":"claude","auth_id":"b.json","model":"m5","status":"cooling","next_retry_after":"{past}","reason":"quota","quota":{{"exceeded":true,"reason":"quota","next_recover_at":"{late}","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}},
5349{{"provider":"claude","auth_id":"b.json","model":"m3","status":"cooling","next_retry_after":"{past}","reason":"boom","quota":{{"exceeded":false,"next_recover_at":"0001-01-01T00:00:00Z","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}},
5350{{"provider":"claude","auth_id":"gone.json","model":"m1","status":"cooling","next_retry_after":"{late}","quota":{{"exceeded":false,"next_recover_at":"0001-01-01T00:00:00Z","observed_at":"0001-01-01T00:00:00Z"}},"updated_at":"{now}"}}]}}"#,
5351            now = at(0),
5352            late = at(600),
5353            soon = at(60),
5354            past = at(-5),
5355        );
5356        std::fs::write(dir.join("b.cds"), go).unwrap();
5357        drop(rt);
5358        let rt = crate::testing::runtime(config(), creds(), executors());
5359        let b = rt.store.get("b.json").unwrap();
5360        let now = Instant::now();
5361        let scheduler = rt.store.scheduler.lock().unwrap();
5362        let wait = |m: &str| scheduler.wait(&b, m, now).unwrap_or_default().as_secs();
5363        assert!((590..=600).contains(&wait("m9")), "credential quota blocks every model");
5364        assert!(scheduler.quota_cooling(&b, "m9", now));
5365        let records = scheduler.records(&b, now, std::time::SystemTime::now());
5366        drop(scheduler);
5367        let models: Vec<&str> = records.iter().map(|r| r.model.as_str()).collect();
5368        assert_eq!(
5369            models,
5370            ["", "m2", "m4"],
5371            "expired (by retry deadline) and unknown-credential records are dropped"
5372        );
5373        let m4 = records.iter().find(|r| r.model == "m4").unwrap();
5374        let left = m4
5375            .next_retry_after
5376            .unwrap()
5377            .duration_since(std::time::SystemTime::now())
5378            .unwrap();
5379        assert!(
5380            left > Duration::from_secs(590),
5381            "the later quota recovery wins: {left:?}"
5382        );
5383        let rewritten = std::fs::read_to_string(dir.join("b.cds")).unwrap();
5384        assert!(!rewritten.contains("gone.json") && !rewritten.contains("\"m3\""));
5385
5386        // Turning the option off persists the final state and stops writing.
5387        rt.publish_config(Config::parse(&format!("auth-dir: {}\n", dir.display())).unwrap());
5388        rt.store.reset_cooldown("b.json");
5389        assert!(dir.join("b.cds").exists(), "no writes once disabled");
5390        let _ = std::fs::remove_dir_all(&dir);
5391    }
5392
5393    #[tokio::test]
5394    async fn preparation_waiter_observes_deletion_and_refresh_loop_is_replaceable() {
5395        let rt = Arc::new(crate::testing::runtime(
5396            Config::parse("").unwrap(),
5397            vec![cred("a.json", "claude", false)],
5398            Executors {
5399                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5400                codex: Default::default(),
5401                devices: Default::default(),
5402                openai: Default::default(),
5403                google: Default::default(),
5404            },
5405        ));
5406        let lock = rt.store.prepare_lock("a.json");
5407        let guard = lock.lock().await;
5408        let worker = {
5409            let rt = rt.clone();
5410            tokio::spawn(async move { rt.prepare_credential("a.json", &rt.config(), None).await })
5411        };
5412        tokio::task::yield_now().await;
5413        assert!(!worker.is_finished());
5414        rt.store.reconcile(Vec::new());
5415        drop(guard);
5416        let error = worker.await.unwrap().unwrap_err();
5417        assert_eq!(error.scope, FailureScope::Request);
5418        assert!(rt.store.snapshot().is_empty());
5419        rt.start_auto_refresh();
5420        let first = rt.refresh_task.lock().unwrap().clone().unwrap();
5421        rt.start_auto_refresh();
5422        tokio::task::yield_now().await;
5423        assert!(first.is_finished());
5424        let second = rt.refresh_task.lock().unwrap().clone().unwrap();
5425        rt.stop_auto_refresh();
5426        tokio::task::yield_now().await;
5427        assert!(second.is_finished());
5428        assert!(rt.refresh_task.lock().unwrap().is_none());
5429    }
5430
5431    /// A refresh whose caller is cancelled while its commit waits keeps the preparation
5432    /// lock until the commit returns: a writer that serializes on the lock (an SIWC
5433    /// binding) lands after the rotated tokens instead of making them stale.
5434    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5435    async fn a_cancelled_refresh_holds_the_preparation_lock_through_its_commit() {
5436        let (hit_tx, mut hit) = tokio::sync::mpsc::unbounded_channel();
5437        let answer = Arc::new(tokio::sync::Semaphore::new(0));
5438        let app = axum::Router::new().route(
5439            "/oauth/token",
5440            axum::routing::post({
5441                let answer = answer.clone();
5442                move || async move {
5443                    hit_tx.send(()).unwrap();
5444                    answer.acquire().await.unwrap().forget();
5445                    r#"{"access_token":"rotated","refresh_token":"rotated-rt","expires_in":3600}"#
5446                }
5447            }),
5448        );
5449        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5450        let token_url = format!("http://{}/oauth/token", listener.local_addr().unwrap());
5451        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5452        let dir = std::env::temp_dir().join(format!("cpa-prepare-cancel-{}", uuid::Uuid::new_v4()));
5453        std::fs::create_dir_all(&dir).unwrap();
5454        let path = dir.join("codex.json");
5455        std::fs::write(
5456            &path,
5457            r#"{"type":"codex","access_token":"outer","refresh_token":"outer-rt"}"#,
5458        )
5459        .unwrap();
5460        let mut cfg = Config::parse("").unwrap();
5461        cfg.auth_dir = dir.clone();
5462        let client = wreq::Client::new();
5463        let oauth =
5464            cpa_exec::codex_oauth::CodexOAuth::with_endpoints(client.clone(), &token_url, &token_url, &token_url);
5465        let rt = Arc::new(crate::testing::runtime(
5466            cfg.clone(),
5467            cpa_core::config::credentials::from_auth_dir(&cfg)
5468                .into_iter()
5469                .map(crate::testing::local)
5470                .collect(),
5471            Executors {
5472                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5473                codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
5474                devices: Default::default(),
5475                openai: Default::default(),
5476                google: Default::default(),
5477            },
5478        ));
5479        let refresh = tokio::spawn({
5480            let rt = rt.clone();
5481            async move { rt.refresh_credential("codex.json").await }
5482        });
5483        if tokio::time::timeout(Duration::from_secs(5), hit.recv()).await.is_err() {
5484            panic!("no refresh request: {:?}", refresh.await);
5485        }
5486        // Hold the store so the refresh's commit job blocks inside `apply_patch`, then
5487        // let the token response through and cancel the caller while the job waits.
5488        let (held_tx, held) = std::sync::mpsc::channel();
5489        let (release, release_rx) = std::sync::mpsc::channel::<()>();
5490        let holder = std::thread::spawn({
5491            let store = rt.store.clone();
5492            move || {
5493                let _inner = store.inner.write().unwrap();
5494                held_tx.send(()).unwrap();
5495                release_rx.recv().unwrap();
5496            }
5497        });
5498        held.recv().unwrap();
5499        answer.add_permits(1);
5500        tokio::time::sleep(Duration::from_millis(500)).await;
5501        refresh.abort();
5502        assert!(refresh.await.unwrap_err().is_cancelled());
5503        let lock = rt.store.prepare_lock("codex.json");
5504        assert!(
5505            tokio::time::timeout(Duration::from_millis(300), lock.clone().lock_owned())
5506                .await
5507                .is_err(),
5508            "the detached commit still holds the lock"
5509        );
5510        release.send(()).unwrap();
5511        holder.join().unwrap();
5512        let _binding = tokio::time::timeout(Duration::from_secs(5), lock.lock_owned())
5513            .await
5514            .unwrap();
5515        let current = rt.store.get("codex.json").unwrap();
5516        assert_eq!(current.str("access_token"), Some("rotated"));
5517        let mut patch = MetadataPatch::default();
5518        patch.set.insert("use_siwc".into(), true.into());
5519        let bound = rt.store.apply_patch("codex.json", current.revision, &patch).unwrap();
5520        let disk: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5521        for meta in [&disk, &Value::Object(bound.metadata.clone())] {
5522            assert_eq!(
5523                (&meta["access_token"], &meta["refresh_token"], &meta["use_siwc"]),
5524                (
5525                    &serde_json::json!("rotated"),
5526                    &serde_json::json!("rotated-rt"),
5527                    &serde_json::json!(true)
5528                ),
5529                "{meta}"
5530            );
5531        }
5532        std::fs::remove_dir_all(dir).unwrap();
5533    }
5534
5535    /// Go 935aa6e3 (TestManager_RefreshAuth_DropsStaleConcurrentCredential): tokens saved
5536    /// while a refresh is in flight win over its result, in memory and on disk, and the
5537    /// refresh answers with them. An edit that leaves the credentials alone is merged with
5538    /// the refresh (Go `MergeRefreshedAuth`), so a rotated refresh token is not lost and a
5539    /// field the refresh did not change keeps the edit.
5540    #[tokio::test]
5541    async fn an_in_flight_refresh_does_not_overwrite_newer_credentials() {
5542        for (edit, want) in [
5543            (
5544                serde_json::json!({"access_token": "saved", "refresh_token": "saved-rt"}),
5545                ("saved", "saved-rt", None, "free"),
5546            ),
5547            (
5548                serde_json::json!({"note": "kept"}),
5549                ("rotated", "rotated-rt", Some("kept"), "free"),
5550            ),
5551            // The refresh leaves the plan as it was, so the edit stands.
5552            (
5553                serde_json::json!({"plan_type": "team"}),
5554                ("rotated", "rotated-rt", None, "team"),
5555            ),
5556        ] {
5557            let (hit_tx, mut hit) = tokio::sync::mpsc::unbounded_channel();
5558            let answer = Arc::new(tokio::sync::Semaphore::new(0));
5559            let app = axum::Router::new().route(
5560                "/oauth/token",
5561                axum::routing::post({
5562                    let answer = answer.clone();
5563                    move || async move {
5564                        hit_tx.send(()).unwrap();
5565                        answer.acquire().await.unwrap().forget();
5566                        r#"{"access_token":"rotated","refresh_token":"rotated-rt","expires_in":3600}"#
5567                    }
5568                }),
5569            );
5570            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5571            let token_url = format!("http://{}/oauth/token", listener.local_addr().unwrap());
5572            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5573            let dir = std::env::temp_dir().join(format!("cpa-refresh-stale-{}", uuid::Uuid::new_v4()));
5574            std::fs::create_dir_all(&dir).unwrap();
5575            let path = dir.join("codex.json");
5576            std::fs::write(
5577                &path,
5578                r#"{"type":"codex","access_token":"outer","refresh_token":"outer-rt","plan_type":"free"}"#,
5579            )
5580            .unwrap();
5581            let mut cfg = Config::parse("").unwrap();
5582            cfg.auth_dir = dir.clone();
5583            let client = wreq::Client::new();
5584            let oauth =
5585                cpa_exec::codex_oauth::CodexOAuth::with_endpoints(client.clone(), &token_url, &token_url, &token_url);
5586            let rt = Arc::new(crate::testing::runtime(
5587                cfg.clone(),
5588                cpa_core::config::credentials::from_auth_dir(&cfg)
5589                    .into_iter()
5590                    .map(crate::testing::local)
5591                    .collect(),
5592                Executors {
5593                    claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5594                    codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
5595                    devices: Default::default(),
5596                    openai: Default::default(),
5597                    google: Default::default(),
5598                },
5599            ));
5600            let refresh = tokio::spawn({
5601                let rt = rt.clone();
5602                async move { rt.refresh_credential("codex.json").await }
5603            });
5604            tokio::time::timeout(Duration::from_secs(5), hit.recv())
5605                .await
5606                .expect("a refresh request");
5607            let mut patch = MetadataPatch::default();
5608            for (k, v) in edit.as_object().unwrap() {
5609                patch.set.insert(k.clone(), v.clone());
5610            }
5611            let before = rt.store.get("codex.json").unwrap().revision;
5612            rt.store.apply_patch("codex.json", before, &patch).unwrap();
5613            answer.add_permits(1);
5614            let answered = refresh.await.unwrap().expect("the refresh answers");
5615            let current = rt.store.get("codex.json").unwrap();
5616            let disk: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5617            for meta in [
5618                &disk,
5619                &Value::Object(current.metadata.clone()),
5620                &Value::Object(answered.metadata.clone()),
5621            ] {
5622                assert_eq!(
5623                    (
5624                        meta["access_token"].as_str().unwrap(),
5625                        meta["refresh_token"].as_str().unwrap(),
5626                        meta["note"].as_str(),
5627                        meta["plan_type"].as_str().unwrap()
5628                    ),
5629                    want,
5630                    "{edit}: {meta}"
5631                );
5632            }
5633            std::fs::remove_dir_all(dir).unwrap();
5634        }
5635    }
5636
5637    /// A runtime over Codex credential files in a private auth-dir, refreshing against a
5638    /// loopback mock that records each grant as (partition, refresh token, client ID).
5639    struct CodexDisk {
5640        rt: Arc<Runtime>,
5641        dir: std::path::PathBuf,
5642        grants: Arc<Mutex<Grants>>,
5643        /// Each grant takes a permit before the mock answers.
5644        gate: Arc<tokio::sync::Semaphore>,
5645        base: String,
5646    }
5647
5648    type Grants = Vec<(&'static str, String, String)>;
5649
5650    impl CodexDisk {
5651        async fn new(files: &[(&str, &Value)]) -> Self {
5652            Self::gated(files, 1000).await
5653        }
5654
5655        async fn gated(files: &[(&str, &Value)], permits: usize) -> Self {
5656            let grants = Arc::new(Mutex::new(Vec::new()));
5657            let gate = Arc::new(tokio::sync::Semaphore::new(permits));
5658            let route = |partition: &'static str, answer: &'static str| {
5659                let (grants, gate) = (grants.clone(), gate.clone());
5660                axum::routing::post(move |body: String| async move {
5661                    let field = |key: &str| {
5662                        url::form_urlencoded::parse(body.as_bytes())
5663                            .find(|(k, _)| k == key)
5664                            .map(|(_, v)| v.into_owned())
5665                            .unwrap_or_default()
5666                    };
5667                    grants
5668                        .lock()
5669                        .unwrap()
5670                        .push((partition, field("refresh_token"), field("client_id")));
5671                    gate.acquire().await.unwrap().forget();
5672                    answer
5673                })
5674            };
5675            let app = axum::Router::new()
5676                .route(
5677                    "/oauth/token",
5678                    route(
5679                        "outer",
5680                        r#"{"access_token":"outer-access-refreshed","refresh_token":"outer-ref-refreshed","expires_in":864000}"#,
5681                    ),
5682                )
5683                .route(
5684                    "/siwc/token",
5685                    route(
5686                        "siwc",
5687                        r#"{"access_token":"siwc-access-refreshed","refresh_token":"siwc-ref-refreshed","scope":"openid chatgpt.tokens.use.direct","expires_in":3600}"#,
5688                    ),
5689                );
5690            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5691            let base = format!("http://{}", listener.local_addr().unwrap());
5692            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5693            let dir = std::env::temp_dir().join(format!("cpa-codex-disk-{}", uuid::Uuid::new_v4()));
5694            std::fs::create_dir_all(&dir).unwrap();
5695            for (name, body) in files {
5696                std::fs::write(dir.join(name), body.to_string()).unwrap();
5697            }
5698            let rt = Self::runtime(&dir, &base);
5699            Self {
5700                rt,
5701                dir,
5702                grants,
5703                gate,
5704                base,
5705            }
5706        }
5707
5708        /// A runtime over the auth-dir as it is now, with its own refresh coordination:
5709        /// another process sharing the files and the mock.
5710        fn other_process(&self) -> Arc<Runtime> {
5711            Self::runtime(&self.dir, &self.base)
5712        }
5713
5714        fn runtime(dir: &Path, base: &str) -> Arc<Runtime> {
5715            let mut cfg = Config::parse("").unwrap();
5716            cfg.auth_dir = dir.to_owned();
5717            let client = wreq::Client::new();
5718            let oauth = cpa_exec::codex_oauth::CodexOAuth::with_endpoints(
5719                client.clone(),
5720                &format!("{base}/oauth/token"),
5721                "",
5722                "",
5723            )
5724            .with_siwc_token_url(&format!("{base}/siwc/token"))
5725            .with_retry_delay(Duration::from_millis(1));
5726            Arc::new(crate::testing::runtime(
5727                cfg.clone(),
5728                cpa_core::config::credentials::from_auth_dir(&cfg)
5729                    .into_iter()
5730                    .map(crate::testing::local)
5731                    .collect(),
5732                Executors {
5733                    claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
5734                    codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
5735                    devices: Default::default(),
5736                    openai: Default::default(),
5737                    google: Default::default(),
5738                },
5739            ))
5740        }
5741
5742        /// Waits until `n` refreshes in all went to lock this auth-dir's `name`.
5743        async fn lock_attempts(&self, name: &str, n: usize) {
5744            let path = self.dir.join(name);
5745            let attempted = async {
5746                while LOCK_ATTEMPTS.lock().unwrap().iter().filter(|p| **p == path).count() < n {
5747                    tokio::time::sleep(Duration::from_millis(5)).await;
5748                }
5749            };
5750            tokio::time::timeout(Duration::from_secs(5), attempted)
5751                .await
5752                .expect("the refresh went to lock the file");
5753        }
5754
5755        /// Waits until a grant reached the mock.
5756        async fn granted(&self) {
5757            let granted = async {
5758                while self.grants.lock().unwrap().is_empty() {
5759                    tokio::time::sleep(Duration::from_millis(5)).await;
5760                }
5761            };
5762            tokio::time::timeout(Duration::from_secs(5), granted)
5763                .await
5764                .expect("a grant");
5765        }
5766
5767        fn grants(&self) -> Grants {
5768            std::mem::take(&mut *self.grants.lock().unwrap())
5769        }
5770
5771        /// The file and the stored credential, which must agree on `fields`.
5772        fn tokens(&self, name: &str, fields: &[&str]) -> Vec<Value> {
5773            let disk: Value = serde_json::from_slice(&std::fs::read(self.dir.join(name)).unwrap()).unwrap();
5774            let stored = Value::Object(self.rt.store.get(name).unwrap().metadata.clone());
5775            let pick = |m: &Value| {
5776                fields
5777                    .iter()
5778                    .map(|f| m.pointer(f).cloned().unwrap_or(Value::Null))
5779                    .collect()
5780            };
5781            let (disk, stored): (Vec<Value>, Vec<Value>) = (pick(&disk), pick(&stored));
5782            assert_eq!(disk, stored, "file and store differ");
5783            disk
5784        }
5785    }
5786
5787    fn codex_file(outer: (&str, chrono::Duration), siwc: (&str, &str, chrono::Duration)) -> Value {
5788        let now = chrono::Utc::now();
5789        serde_json::json!({
5790            "type": "codex",
5791            "access_token": format!("outer-{}", outer.0),
5792            "refresh_token": format!("outer-ref-{}", outer.0),
5793            "expired": (now + outer.1).to_rfc3339(),
5794            "use_siwc": true,
5795            "siwc": {
5796                "access_token": format!("siwc-{}", siwc.0),
5797                "refresh_token": format!("siwc-ref-{}", siwc.0),
5798                "client_id": siwc.1,
5799                "device_id": "e61bbe28-07ef-466d-8e5d-a344f94ab305",
5800                "expired": (now + siwc.2).to_rfc3339(),
5801            },
5802        })
5803    }
5804
5805    /// A shutdown that lands while a credential lock wait sleeps stops it, even when
5806    /// the lock comes free and the sleep ends at the same moment: no lock is taken (and
5807    /// no grant started) after shutdown. Polled by hand, so both are ready at once.
5808    #[tokio::test(start_paused = true)]
5809    async fn a_lock_wait_stops_at_shutdown_even_when_the_lock_frees() {
5810        let dir = std::env::temp_dir().join(format!("cpa-lock-wait-{}", uuid::Uuid::new_v4()));
5811        std::fs::create_dir_all(&dir).unwrap();
5812        let path = dir.join("codex.json");
5813        let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
5814        for _ in 0..32 {
5815            let held = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
5816            let shutdown = tokio_util::sync::CancellationToken::new();
5817            let mut wait = std::pin::pin!(wait_codex_lock(&path, &shutdown));
5818            assert!(wait.as_mut().poll(&mut cx).is_pending(), "the lock is held");
5819            drop(held);
5820            tokio::time::advance(LOCK_POLL_CEILING).await;
5821            shutdown.cancel();
5822            let std::task::Poll::Ready(waited) = wait.as_mut().poll(&mut cx) else {
5823                panic!("the wait ends");
5824            };
5825            assert_eq!(waited.map(drop), Err(LOCK_WAIT_STOPPED.to_owned()));
5826        }
5827        std::fs::remove_dir_all(dir).unwrap();
5828    }
5829
5830    /// Go `TestCodexExecutorRefreshSerializesUnderFileLockAndSkipsAlreadyRotatedToken`
5831    /// (7b0319fb): a due refresh waits while another process holds the file's credential
5832    /// lock, then reads the file: the Sign in with ChatGPT partition that process rotated
5833    /// is adopted, in memory and on disk, and its refresh token is not spent again.
5834    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5835    async fn a_codex_refresh_waits_for_the_file_lock_and_adopts_what_another_process_rotated() {
5836        let (days, minutes) = (chrono::Duration::days(5), chrono::Duration::minutes(2));
5837        let f = CodexDisk::new(&[(
5838            "codex.json",
5839            &codex_file(("keep", days), ("old", "oaiapp_shared", minutes)),
5840        )])
5841        .await;
5842        let path = f.dir.join("codex.json");
5843        let (locked_tx, locked) = std::sync::mpsc::channel();
5844        let (release_tx, release) = std::sync::mpsc::channel::<()>();
5845        let other = std::thread::spawn({
5846            let path = path.clone();
5847            move || {
5848                let lock = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
5849                locked_tx.send(()).unwrap();
5850                release.recv().unwrap();
5851                let rotated = codex_file(("keep", days), ("other", "oaiapp_shared", chrono::Duration::hours(1)));
5852                std::fs::write(&path, rotated.to_string()).unwrap();
5853                drop(lock);
5854            }
5855        });
5856        locked.recv().unwrap();
5857        let refresh = tokio::spawn({
5858            let rt = f.rt.clone();
5859            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
5860        });
5861        f.lock_attempts("codex.json", 1).await;
5862        tokio::time::sleep(Duration::from_millis(50)).await;
5863        assert!(!refresh.is_finished(), "the refresh waits for the lock");
5864        assert_eq!(f.grants(), [], "nothing refreshes under another process's lock");
5865        release_tx.send(()).unwrap();
5866        other.join().unwrap();
5867        tokio::time::timeout(Duration::from_secs(5), refresh)
5868            .await
5869            .unwrap()
5870            .unwrap()
5871            .unwrap();
5872        assert_eq!(f.grants(), [], "the rotated token is not refreshed again");
5873        assert_eq!(
5874            f.tokens(
5875                "codex.json",
5876                &["/access_token", "/siwc/access_token", "/siwc/refresh_token"]
5877            ),
5878            ["outer-keep", "siwc-other", "siwc-ref-other"]
5879        );
5880        std::fs::remove_dir_all(&f.dir).unwrap();
5881    }
5882
5883    /// The A→B→C case of the same Go test: memory holds version A, the file version B,
5884    /// both due. The refresh spends B's refresh tokens (and B's client ID), and C reaches
5885    /// memory and disk.
5886    #[tokio::test]
5887    async fn a_codex_refresh_spends_the_tokens_on_disk_not_the_stale_snapshot() {
5888        let hour = chrono::Duration::hours(1);
5889        let minute = chrono::Duration::minutes(1);
5890        let f = CodexDisk::new(&[("codex.json", &codex_file(("A", hour), ("A", "oaiapp_A", minute)))]).await;
5891        let disk_b = codex_file(("B", hour), ("B", "oaiapp_B", minute));
5892        std::fs::write(f.dir.join("codex.json"), disk_b.to_string()).unwrap();
5893        f.rt.prepare_credential("codex.json", &f.rt.config(), None)
5894            .await
5895            .unwrap();
5896        assert_eq!(
5897            f.grants(),
5898            [
5899                (
5900                    "outer",
5901                    "outer-ref-B".to_owned(),
5902                    cpa_exec::codex_oauth::CLIENT_ID.to_owned()
5903                ),
5904                ("siwc", "siwc-ref-B".to_owned(), "oaiapp_B".to_owned()),
5905            ]
5906        );
5907        assert_eq!(
5908            f.tokens(
5909                "codex.json",
5910                &[
5911                    "/access_token",
5912                    "/refresh_token",
5913                    "/siwc/access_token",
5914                    "/siwc/client_id"
5915                ]
5916            ),
5917            [
5918                "outer-access-refreshed",
5919                "outer-ref-refreshed",
5920                "siwc-access-refreshed",
5921                "oaiapp_B"
5922            ]
5923        );
5924        std::fs::remove_dir_all(&f.dir).unwrap();
5925    }
5926
5927    /// The A→B→C case when the watcher loads B while B refreshes: the commit merges
5928    /// against B, the tokens the refresh started from, so C is kept, not dropped as a
5929    /// stale refresh of A.
5930    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5931    async fn a_codex_refresh_keeps_its_result_when_the_watcher_loads_the_adopted_tokens() {
5932        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
5933        let f = CodexDisk::gated(
5934            &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_A", minute)))],
5935            0,
5936        )
5937        .await;
5938        let disk_b = codex_file(("keep", days), ("B", "oaiapp_B", minute));
5939        std::fs::write(f.dir.join("codex.json"), disk_b.to_string()).unwrap();
5940        let refresh = tokio::spawn({
5941            let rt = f.rt.clone();
5942            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
5943        });
5944        f.granted().await;
5945        // The watcher's reload of B, while the grant for B is in flight.
5946        let mut reload = MetadataPatch::default();
5947        reload.set.insert("siwc".into(), disk_b["siwc"].clone());
5948        let revision = f.rt.store.get("codex.json").unwrap().revision;
5949        f.rt.store.apply_patch("codex.json", revision, &reload).unwrap();
5950        f.gate.add_permits(1);
5951        tokio::time::timeout(Duration::from_secs(5), refresh)
5952            .await
5953            .unwrap()
5954            .unwrap()
5955            .unwrap();
5956        assert_eq!(f.grants(), [("siwc", "siwc-ref-B".to_owned(), "oaiapp_B".to_owned())]);
5957        assert_eq!(
5958            f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
5959            ["siwc-access-refreshed", "siwc-ref-refreshed"]
5960        );
5961        std::fs::remove_dir_all(&f.dir).unwrap();
5962    }
5963
5964    /// A file whose `type` is spelled differently from the provider it loaded as
5965    /// (`Codex`) still takes an adopted outer partition: the stored spelling stays.
5966    #[tokio::test]
5967    async fn a_codex_refresh_adopts_a_rotated_file_whatever_its_type_spelling() {
5968        let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
5969        let mut file = codex_file(("A", days), ("keep", "oaiapp_x", hour));
5970        file["type"] = "Codex".into();
5971        file["use_siwc"] = false.into();
5972        let f = CodexDisk::new(&[("codex.json", &file)]).await;
5973        let mut rotated = file.clone();
5974        rotated["access_token"] = "outer-B".into();
5975        rotated["refresh_token"] = "outer-ref-B".into();
5976        std::fs::write(f.dir.join("codex.json"), rotated.to_string()).unwrap();
5977        let current = f.rt.store.get("codex.json").unwrap();
5978        let cfg = f.rt.config();
5979        assert!(f.rt.refresh_after_unauthorized(&current, &cfg).await.is_some());
5980        assert_eq!(f.grants(), []);
5981        assert_eq!(
5982            f.tokens("codex.json", &["/type", "/access_token", "/refresh_token"]),
5983            ["Codex", "outer-B", "outer-ref-B"]
5984        );
5985        std::fs::remove_dir_all(&f.dir).unwrap();
5986    }
5987
5988    /// After a 401 on the Sign in with ChatGPT token, a file whose partition another
5989    /// process already rotated is adopted without forcing a refresh of the new token
5990    /// (Go `RejectedAccessToken` names the old one). A file that is not valid JSON fails
5991    /// the refresh before any grant (Go: "invalid credential JSON").
5992    #[tokio::test]
5993    async fn a_rejected_codex_token_adopts_a_rotated_file_and_a_corrupt_file_refreshes_nothing() {
5994        let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
5995        let f = CodexDisk::new(&[(
5996            "codex.json",
5997            &codex_file(("keep", days), ("rejected", "oaiapp_x", hour)),
5998        )])
5999        .await;
6000        let path = f.dir.join("codex.json");
6001        std::fs::write(
6002            &path,
6003            codex_file(("keep", days), ("other", "oaiapp_x", hour)).to_string(),
6004        )
6005        .unwrap();
6006        let cfg = f.rt.config();
6007        let current = f.rt.store.get("codex.json").unwrap();
6008        let refreshed = f.rt.refresh_after_unauthorized(&current, &cfg).await.expect("adopted");
6009        assert_eq!(f.grants(), []);
6010        assert_eq!(refreshed.metadata["siwc"]["access_token"], "siwc-other");
6011        assert_eq!(
6012            f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6013            ["siwc-other", "siwc-ref-other"]
6014        );
6015        std::fs::write(&path, "{corrupt").unwrap();
6016        assert!(f.rt.refresh_after_unauthorized(&refreshed, &cfg).await.is_none());
6017        assert_eq!(f.grants(), []);
6018        std::fs::remove_dir_all(&f.dir).unwrap();
6019    }
6020
6021    /// The commit takes the partition as the file held it under the lock as well as the
6022    /// snapshot's (Go `matchesBaselineOrAfter`): memory holds A and is edited while the
6023    /// refresh waits for another process, which rotates the file to B. C, refreshed from
6024    /// B, lands over B with the edit kept, instead of being dropped as a refresh of an A
6025    /// that changed meanwhile, which would leave B's spent refresh token on disk.
6026    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6027    async fn a_codex_refresh_of_an_adopted_partition_lands_after_an_edit_to_the_snapshot() {
6028        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6029        let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))]).await;
6030        let path = f.dir.join("codex.json");
6031        let (locked_tx, locked) = std::sync::mpsc::channel();
6032        let (release_tx, release) = std::sync::mpsc::channel::<()>();
6033        let other = std::thread::spawn({
6034            let path = path.clone();
6035            move || {
6036                let lock = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
6037                locked_tx.send(()).unwrap();
6038                release.recv().unwrap();
6039                let mut rotated = codex_file(("keep", days), ("B", "oaiapp_x", minute));
6040                rotated["priority"] = 7.into();
6041                std::fs::write(&path, rotated.to_string()).unwrap();
6042                drop(lock);
6043            }
6044        });
6045        locked.recv().unwrap();
6046        let refresh = tokio::spawn({
6047            let rt = f.rt.clone();
6048            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6049        });
6050        f.lock_attempts("codex.json", 1).await;
6051        // An edit of A that keeps its tokens (the watcher, a management save).
6052        let mut edit = MetadataPatch::default();
6053        edit.set.insert("priority".into(), 7.into());
6054        let revision = f.rt.store.get("codex.json").unwrap().revision;
6055        f.rt.store.apply_patch("codex.json", revision, &edit).unwrap();
6056        release_tx.send(()).unwrap();
6057        other.join().unwrap();
6058        tokio::time::timeout(Duration::from_secs(5), refresh)
6059            .await
6060            .unwrap()
6061            .unwrap()
6062            .unwrap();
6063        assert_eq!(f.grants(), [("siwc", "siwc-ref-B".to_owned(), "oaiapp_x".to_owned())]);
6064        assert_eq!(
6065            f.tokens(
6066                "codex.json",
6067                &["/siwc/access_token", "/siwc/refresh_token", "/priority"]
6068            ),
6069            [
6070                Value::from("siwc-access-refreshed"),
6071                Value::from("siwc-ref-refreshed"),
6072                Value::from(7)
6073            ]
6074        );
6075        std::fs::remove_dir_all(&f.dir).unwrap();
6076    }
6077
6078    /// A refresh whose caller is cancelled once its grant is sent keeps the file's
6079    /// credential lock until the new tokens are written (Go `refreshLocked` writes before
6080    /// `WithFileLock` returns): another process waiting on the lock then reads them and
6081    /// does not spend the old refresh token again.
6082    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6083    async fn a_cancelled_codex_refresh_writes_its_tokens_before_another_process_reads_the_file() {
6084        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6085        let f = CodexDisk::gated(
6086            &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6087            0,
6088        )
6089        .await;
6090        let other = f.other_process();
6091        let refresh = tokio::spawn({
6092            let rt = f.rt.clone();
6093            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6094        });
6095        f.granted().await;
6096        refresh.abort();
6097        assert!(refresh.await.unwrap_err().is_cancelled());
6098        let waiting = tokio::spawn({
6099            let other = other.clone();
6100            async move { other.prepare_credential("codex.json", &other.config(), None).await }
6101        });
6102        f.lock_attempts("codex.json", 2).await;
6103        f.gate.add_permits(10);
6104        let adopted = tokio::time::timeout(Duration::from_secs(5), waiting)
6105            .await
6106            .unwrap()
6107            .unwrap()
6108            .unwrap();
6109        assert_eq!(f.grants(), [("siwc", "siwc-ref-A".to_owned(), "oaiapp_x".to_owned())]);
6110        assert_eq!(adopted.metadata["siwc"]["refresh_token"], "siwc-ref-refreshed");
6111        assert_eq!(
6112            f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6113            ["siwc-access-refreshed", "siwc-ref-refreshed"]
6114        );
6115        std::fs::remove_dir_all(&f.dir).unwrap();
6116    }
6117
6118    /// Fields other than the refreshed tokens are written as the file holds them under
6119    /// the lock (Go applies the refresh patch to the file it reads again): a credential
6120    /// disabled in the file, not yet reloaded, stays disabled after a refresh of the
6121    /// tokens it adopted from that file.
6122    #[tokio::test]
6123    async fn a_codex_refresh_keeps_a_disabled_flag_it_finds_in_the_file() {
6124        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6125        let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))]).await;
6126        let mut disk_b = codex_file(("keep", days), ("B", "oaiapp_x", minute));
6127        disk_b["disabled"] = true.into();
6128        std::fs::write(f.dir.join("codex.json"), disk_b.to_string()).unwrap();
6129        let refreshed =
6130            f.rt.prepare_credential("codex.json", &f.rt.config(), None)
6131                .await
6132                .unwrap();
6133        assert_eq!(f.grants(), [("siwc", "siwc-ref-B".to_owned(), "oaiapp_x".to_owned())]);
6134        assert!(refreshed.disabled);
6135        assert_eq!(
6136            f.tokens("codex.json", &["/disabled", "/siwc/access_token"]),
6137            [Value::Bool(true), Value::from("siwc-access-refreshed")]
6138        );
6139        std::fs::remove_dir_all(&f.dir).unwrap();
6140    }
6141
6142    /// After a 401, a file that rotated only the refresh token of the partition requests
6143    /// use still holds the rejected access token, so the refresh is forced, with the
6144    /// file's refresh token (Go compares the adopted access token with
6145    /// `RejectedAccessToken`). Both partitions.
6146    #[tokio::test]
6147    async fn a_rejected_codex_token_refreshes_when_the_file_rotated_only_its_refresh_token() {
6148        let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
6149        for use_siwc in [true, false] {
6150            let mut file = codex_file(("A", days), ("A", "oaiapp_x", hour));
6151            file["use_siwc"] = use_siwc.into();
6152            let f = CodexDisk::new(&[("codex.json", &file)]).await;
6153            let (prefix, client) = match use_siwc {
6154                true => ("siwc", "oaiapp_x"),
6155                false => ("outer", cpa_exec::codex_oauth::CLIENT_ID),
6156            };
6157            let pointer = if use_siwc {
6158                "/siwc/refresh_token"
6159            } else {
6160                "/refresh_token"
6161            };
6162            *file.pointer_mut(pointer).unwrap() = format!("{prefix}-ref-B").into();
6163            std::fs::write(f.dir.join("codex.json"), file.to_string()).unwrap();
6164            let current = f.rt.store.get("codex.json").unwrap();
6165            let cfg = f.rt.config();
6166            let refreshed = f.rt.refresh_after_unauthorized(&current, &cfg).await;
6167            assert_eq!(
6168                f.grants(),
6169                [(
6170                    if use_siwc { "siwc" } else { "outer" },
6171                    format!("{prefix}-ref-B"),
6172                    client.to_owned()
6173                )],
6174                "use_siwc {use_siwc}"
6175            );
6176            let access = if use_siwc {
6177                "/siwc/access_token"
6178            } else {
6179                "/access_token"
6180            };
6181            let refreshed = Value::Object(refreshed.expect("refreshed").metadata.clone());
6182            assert_eq!(
6183                refreshed.pointer(access).unwrap(),
6184                &format!("{prefix}-access-refreshed")
6185            );
6186            assert_eq!(
6187                f.tokens("codex.json", &[access]),
6188                [format!("{prefix}-access-refreshed")]
6189            );
6190            std::fs::remove_dir_all(&f.dir).unwrap();
6191        }
6192    }
6193
6194    /// A credential file removed while the refresh waits for its lock fails the refresh
6195    /// before any grant (Go: "read credential file").
6196    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6197    async fn a_codex_refresh_fails_when_its_file_is_removed_while_it_waits_for_the_lock() {
6198        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6199        let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))]).await;
6200        let path = f.dir.join("codex.json");
6201        let (locked_tx, locked) = std::sync::mpsc::channel();
6202        let (release_tx, release) = std::sync::mpsc::channel::<()>();
6203        let other = std::thread::spawn({
6204            let path = path.clone();
6205            move || {
6206                let lock = cpa_exec::codex_siwc_bind::lock_file(&path).unwrap();
6207                locked_tx.send(()).unwrap();
6208                release.recv().unwrap();
6209                std::fs::remove_file(&path).unwrap();
6210                drop(lock);
6211            }
6212        });
6213        locked.recv().unwrap();
6214        let refresh = tokio::spawn({
6215            let rt = f.rt.clone();
6216            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6217        });
6218        f.lock_attempts("codex.json", 1).await;
6219        release_tx.send(()).unwrap();
6220        other.join().unwrap();
6221        let result = tokio::time::timeout(Duration::from_secs(5), refresh)
6222            .await
6223            .unwrap()
6224            .unwrap();
6225        assert!(result.is_err(), "{result:?}");
6226        assert_eq!(f.grants(), []);
6227        std::fs::remove_dir_all(&f.dir).unwrap();
6228    }
6229
6230    /// The commit applies a partition over the snapshot's version too (Go
6231    /// `matchesBaselineOrAfter`): memory holds A, the refresh reads B under the lock and
6232    /// spends B's outer refresh token, and a writer that does not take the lock puts A
6233    /// back meanwhile. B's adopted Sign in with ChatGPT partition then lands over A, its
6234    /// numbers written as Go's float64 reading of them, and legacy keys renamed.
6235    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6236    async fn a_codex_refresh_lands_over_the_snapshot_tokens_restored_while_it_ran() {
6237        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6238        let file_a = codex_file(("A", minute), ("A", "oaiapp_x", days));
6239        let f = CodexDisk::gated(&[("codex.json", &file_a)], 0).await;
6240        let path = f.dir.join("codex.json");
6241        let mut file_b = codex_file(("A", minute), ("B", "oaiapp_x", days));
6242        file_b["siwc"]["expired"] = serde_json::from_str("9007199254740993").unwrap();
6243        file_b["siwc"]["last_refresh"] = serde_json::from_str("2000000000.0").unwrap();
6244        std::fs::write(&path, serde_json::to_string(&file_b).unwrap()).unwrap();
6245        let refresh = tokio::spawn({
6246            let rt = f.rt.clone();
6247            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6248        });
6249        f.granted().await;
6250        let mut restored = file_a.clone();
6251        restored["openapi-location"] = "x".into();
6252        std::fs::write(&path, restored.to_string()).unwrap();
6253        f.gate.add_permits(1);
6254        tokio::time::timeout(Duration::from_secs(5), refresh)
6255            .await
6256            .unwrap()
6257            .unwrap()
6258            .unwrap();
6259        assert_eq!(
6260            f.grants(),
6261            [(
6262                "outer",
6263                "outer-ref-A".to_owned(),
6264                cpa_exec::codex_oauth::CLIENT_ID.to_owned()
6265            )]
6266        );
6267        assert_eq!(
6268            f.tokens(
6269                "codex.json",
6270                &["/access_token", "/siwc/access_token", "/siwc/refresh_token"]
6271            ),
6272            ["outer-access-refreshed", "siwc-B", "siwc-ref-B"]
6273        );
6274        let written = std::fs::read_to_string(&path).unwrap();
6275        for want in [
6276            r#""expired":9007199254740992,"#,
6277            r#""last_refresh":2000000000,"#,
6278            r#""openapi_location":"x""#,
6279        ] {
6280            assert!(written.contains(want), "{want} in {written}");
6281        }
6282        std::fs::remove_dir_all(&f.dir).unwrap();
6283    }
6284
6285    /// A credential replaced in place while its refresh ran (same registration, other
6286    /// tokens) keeps the replacement: no refreshed partition applies over it, so the
6287    /// store discards the refresh without publishing anything (Go `updateInternal`).
6288    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6289    async fn a_codex_refresh_is_discarded_when_the_credential_was_replaced_meanwhile() {
6290        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6291        let f = CodexDisk::gated(
6292            &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6293            0,
6294        )
6295        .await;
6296        let refresh = tokio::spawn({
6297            let rt = f.rt.clone();
6298            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6299        });
6300        f.granted().await;
6301        let mut replace = MetadataPatch::default();
6302        replace.set.insert(
6303            "siwc".into(),
6304            codex_file(("keep", days), ("D", "oaiapp_x", days))["siwc"].clone(),
6305        );
6306        let before = f.rt.store.get("codex.json").unwrap().revision;
6307        let replaced = f.rt.store.apply_patch("codex.json", before, &replace).unwrap();
6308        f.gate.add_permits(1);
6309        let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6310            .await
6311            .unwrap()
6312            .unwrap()
6313            .unwrap();
6314        assert_eq!(f.grants(), [("siwc", "siwc-ref-A".to_owned(), "oaiapp_x".to_owned())]);
6315        assert_eq!(answered.revision, replaced.revision);
6316        assert_eq!(f.rt.store.get("codex.json").unwrap().revision, replaced.revision);
6317        assert_eq!(
6318            f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6319            ["siwc-D", "siwc-ref-D"]
6320        );
6321        std::fs::remove_dir_all(&f.dir).unwrap();
6322    }
6323
6324    /// A refresh is discarded when the credential version moved while it ran and no
6325    /// refreshed partition still matches (Go `updateInternal`), even though the tokens
6326    /// were put back: here the outer partition's account changed with its token, and
6327    /// only the token came back.
6328    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6329    async fn a_codex_refresh_is_discarded_when_the_credential_version_moved_meanwhile() {
6330        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6331        let mut file = codex_file(("A", minute), ("keep", "oaiapp_x", days));
6332        file["use_siwc"] = false.into();
6333        file["account_id"] = "acct-A".into();
6334        let f = CodexDisk::gated(&[("codex.json", &file)], 0).await;
6335        let refresh = tokio::spawn({
6336            let rt = f.rt.clone();
6337            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6338        });
6339        f.granted().await;
6340        let mut last = None;
6341        for edit in [
6342            serde_json::json!({"access_token": "outer-B", "account_id": "acct-X"}),
6343            serde_json::json!({"access_token": "outer-A"}),
6344        ] {
6345            let mut patch = MetadataPatch::default();
6346            for (k, v) in edit.as_object().unwrap() {
6347                patch.set.insert(k.clone(), v.clone());
6348            }
6349            let revision = f.rt.store.get("codex.json").unwrap().revision;
6350            last = Some(f.rt.store.apply_patch("codex.json", revision, &patch).unwrap());
6351        }
6352        let edited = last.unwrap();
6353        f.gate.add_permits(1);
6354        let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6355            .await
6356            .unwrap()
6357            .unwrap()
6358            .unwrap();
6359        assert_eq!(f.grants().len(), 1);
6360        assert_eq!(answered.revision, edited.revision);
6361        assert_eq!(f.rt.store.get("codex.json").unwrap().revision, edited.revision);
6362        assert_eq!(
6363            f.tokens("codex.json", &["/access_token", "/account_id"]),
6364            ["outer-A", "acct-X"]
6365        );
6366        std::fs::remove_dir_all(&f.dir).unwrap();
6367    }
6368
6369    /// A refresh that only adopted tokens another process rotated on disk (no grant) and
6370    /// is then discarded leaves the file as it was, and the credential's cooldowns and
6371    /// refresh record too: Go writes such a refresh only through the persist of an
6372    /// admitted update.
6373    #[tokio::test]
6374    async fn a_discarded_adopt_only_codex_refresh_changes_nothing() {
6375        let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
6376        let f = CodexDisk::new(&[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", hour)))]).await;
6377        let path = f.dir.join("codex.json");
6378        let select = |model: &str| f.rt.store.select(Selection::new("codex", model)).unwrap();
6379        select("m").complete(Outcome::Failure(ExecError::local(
6380            401,
6381            FailureScope::Credential,
6382            "unauthorized",
6383        )));
6384        // Written after the 401, whose cooldown state is persisted too.
6385        let mut file_b = codex_file(("keep", days), ("B", "oaiapp_x", hour));
6386        file_b["request-retry"] = 2.into();
6387        let bytes_b = file_b.to_string();
6388        std::fs::write(&path, &bytes_b).unwrap();
6389        // The store takes other tokens without writing the file (a reload of another
6390        // copy), once the refresh adopted B.
6391        let cooldowns = Arc::new(Mutex::new(None));
6392        BEFORE_ADMIT.lock().unwrap().push((
6393            path.clone(),
6394            Box::new({
6395                let (store, dir, path, cooldowns) =
6396                    (f.rt.store.clone(), f.dir.clone(), path.clone(), cooldowns.clone());
6397                move || {
6398                    let d = codex_file(("keep", days), ("D", "oaiapp_x", days));
6399                    let d = Credential::from_file(&dir, &path, d.as_object().cloned().unwrap()).unwrap();
6400                    store.reconcile(vec![crate::testing::local(d)]);
6401                    // The reload's new tokens cleared the 401 (Go `updateInternalLocked`);
6402                    // upstream rejects D too (recorded without the file write a lease's
6403                    // result makes).
6404                    let rejected = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
6405                    let (d, policy) = (store.get("codex.json").unwrap(), store.policy.read().unwrap().clone());
6406                    let mut scheduler = store.scheduler.lock().unwrap();
6407                    scheduler.record(&d, "m", &rejected, &policy, Instant::now());
6408                    drop(scheduler);
6409                    *cooldowns.lock().unwrap() = Some(store.cooldowns("codex.json").len());
6410                }
6411            }),
6412        ));
6413        let current = f.rt.store.get("codex.json").unwrap();
6414        let cfg = f.rt.config();
6415        assert!(f.rt.refresh_after_unauthorized(&current, &cfg).await.is_some());
6416        assert_eq!(f.grants(), []);
6417        let stored = f.rt.store.get("codex.json").unwrap();
6418        assert_eq!(stored.metadata["siwc"]["access_token"], "siwc-D");
6419        assert_ne!(stored.revision, current.revision);
6420        assert_eq!(std::fs::read_to_string(&path).unwrap(), bytes_b);
6421        let before = cooldowns.lock().unwrap().expect("the hook ran");
6422        assert_eq!(before, 1, "the 401 cooldown is still there");
6423        assert_eq!(f.rt.store.cooldowns("codex.json").len(), before);
6424        assert!(
6425            !f.rt
6426                .store
6427                .unauthorized
6428                .lock()
6429                .unwrap()
6430                .refreshed
6431                .contains_key("codex.json")
6432        );
6433        std::fs::remove_dir_all(&f.dir).unwrap();
6434    }
6435
6436    /// Tokens replaced and put back while the grant ran (A→D→A) move the credential
6437    /// version, but the refreshed partition still holds the version it started from, so
6438    /// the refresh is admitted (Go `CanApplyTo`).
6439    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6440    async fn a_codex_refresh_is_admitted_over_tokens_put_back_meanwhile() {
6441        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6442        let file_a = codex_file(("keep", days), ("A", "oaiapp_x", minute));
6443        let f = CodexDisk::gated(&[("codex.json", &file_a)], 0).await;
6444        let refresh = tokio::spawn({
6445            let rt = f.rt.clone();
6446            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6447        });
6448        f.granted().await;
6449        let version = || f.rt.store.get_registered("codex.json").unwrap().1.credential_version;
6450        let started = version();
6451        for siwc in [
6452            codex_file(("keep", days), ("D", "oaiapp_x", days))["siwc"].clone(),
6453            file_a["siwc"].clone(),
6454        ] {
6455            let mut patch = MetadataPatch::default();
6456            patch.set.insert("siwc".into(), siwc);
6457            let revision = f.rt.store.get("codex.json").unwrap().revision;
6458            f.rt.store.apply_patch("codex.json", revision, &patch).unwrap();
6459        }
6460        assert_ne!(version(), started, "the put-back moved the credential version");
6461        f.gate.add_permits(1);
6462        let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6463            .await
6464            .unwrap()
6465            .unwrap()
6466            .unwrap();
6467        assert_eq!(answered.metadata["siwc"]["access_token"], "siwc-access-refreshed");
6468        assert_eq!(
6469            f.tokens("codex.json", &["/siwc/access_token", "/siwc/refresh_token"]),
6470            ["siwc-access-refreshed", "siwc-ref-refreshed"]
6471        );
6472        std::fs::remove_dir_all(&f.dir).unwrap();
6473    }
6474
6475    /// A save that fails after the grant (a program that does not take the lock removed
6476    /// or truncated the file) does not fail the refresh: Go logs it and publishes the new
6477    /// tokens, writing a removed file again from memory, so the spent refresh token is
6478    /// never kept in memory.
6479    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6480    async fn a_codex_refresh_publishes_its_tokens_when_the_save_fails() {
6481        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6482        for truncate in [false, true] {
6483            let f = CodexDisk::gated(
6484                &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6485                0,
6486            )
6487            .await;
6488            let path = f.dir.join("codex.json");
6489            BEFORE_ADMIT.lock().unwrap().push((
6490                path.clone(),
6491                Box::new({
6492                    let path = path.clone();
6493                    move || match truncate {
6494                        true => std::fs::write(&path, r#"{"type":"#).unwrap(),
6495                        false => std::fs::remove_file(&path).unwrap(),
6496                    }
6497                }),
6498            ));
6499            let refresh = tokio::spawn({
6500                let rt = f.rt.clone();
6501                async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6502            });
6503            f.granted().await;
6504            f.gate.add_permits(1);
6505            let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6506                .await
6507                .unwrap()
6508                .unwrap()
6509                .expect("the refresh succeeds");
6510            assert_eq!(answered.metadata["siwc"]["refresh_token"], "siwc-ref-refreshed");
6511            let stored = f.rt.store.get("codex.json").unwrap();
6512            assert_eq!(stored.metadata["siwc"]["refresh_token"], "siwc-ref-refreshed");
6513            let disk = std::fs::read_to_string(&path).unwrap();
6514            if truncate {
6515                assert_eq!(disk, r#"{"type":"#);
6516            } else {
6517                let disk: Value = serde_json::from_str(&disk).unwrap();
6518                assert_eq!(
6519                    (&disk["siwc"]["refresh_token"], &disk["disabled"]),
6520                    (&"siwc-ref-refreshed".into(), &false.into())
6521                );
6522            }
6523            std::fs::remove_dir_all(&f.dir).unwrap();
6524        }
6525    }
6526
6527    /// At shutdown, a Codex refresh still waiting for the file's lock (another process
6528    /// holds it) gives up without a grant, and one whose grant is in flight is waited for
6529    /// until its tokens are written: an exit cannot strand a spent refresh token.
6530    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6531    async fn shutdown_stops_codex_lock_waits_and_drains_started_grants() {
6532        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6533        let f = CodexDisk::gated(
6534            &[
6535                ("locked.json", &codex_file(("keep", days), ("L", "oaiapp_x", minute))),
6536                ("granting.json", &codex_file(("keep", days), ("G", "oaiapp_x", minute))),
6537            ],
6538            0,
6539        )
6540        .await;
6541        let held = cpa_exec::codex_siwc_bind::lock_file(&f.dir.join("locked.json")).unwrap();
6542        let refresh = |id: &'static str| {
6543            let rt = f.rt.clone();
6544            tokio::spawn(async move { rt.prepare_credential(id, &rt.config(), None).await })
6545        };
6546        let (waiting, granting) = (refresh("locked.json"), refresh("granting.json"));
6547        f.lock_attempts("locked.json", 1).await;
6548        f.granted().await;
6549        let shutdown = tokio::spawn({
6550            let rt = f.rt.clone();
6551            async move { rt.shutdown_codex_refreshes().await }
6552        });
6553        let waited = tokio::time::timeout(Duration::from_secs(5), waiting)
6554            .await
6555            .unwrap()
6556            .unwrap();
6557        assert!(waited.is_err(), "{waited:?}");
6558        tokio::time::sleep(Duration::from_millis(100)).await;
6559        assert!(!shutdown.is_finished(), "shutdown waits for the started grant");
6560        f.gate.add_permits(1);
6561        tokio::time::timeout(Duration::from_secs(5), shutdown)
6562            .await
6563            .unwrap()
6564            .unwrap();
6565        assert!(granting.await.unwrap().is_ok());
6566        assert_eq!(f.grants(), [("siwc", "siwc-ref-G".to_owned(), "oaiapp_x".to_owned())]);
6567        assert_eq!(
6568            f.tokens("granting.json", &["/siwc/refresh_token"]),
6569            ["siwc-ref-refreshed"]
6570        );
6571        drop(held);
6572        std::fs::remove_dir_all(&f.dir).unwrap();
6573    }
6574
6575    /// The new tokens reach the file even when the store refuses the refresh: here the
6576    /// file was removed and created again with the spent tokens, and the ID registered
6577    /// again, while the grant ran. Go writes under the lock before its manager refuses
6578    /// the stale registration, so the next load reads the new tokens, not spent ones.
6579    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6580    async fn a_codex_refresh_writes_its_tokens_when_the_id_was_registered_again_meanwhile() {
6581        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6582        let mut file_a = codex_file(("keep", days), ("A", "oaiapp_x", minute));
6583        file_a["request-retry"] = 2.into();
6584        let f = CodexDisk::gated(&[("codex.json", &file_a)], 0).await;
6585        let path = f.dir.join("codex.json");
6586        let refresh = tokio::spawn({
6587            let rt = f.rt.clone();
6588            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6589        });
6590        f.granted().await;
6591        std::fs::remove_file(&path).unwrap();
6592        std::fs::write(&path, file_a.to_string()).unwrap();
6593        let cfg = f.rt.config();
6594        let load = || {
6595            cpa_core::config::credentials::from_auth_dir(&cfg)
6596                .into_iter()
6597                .map(crate::testing::local)
6598                .collect::<Vec<_>>()
6599        };
6600        f.rt.store.reconcile(vec![]);
6601        f.rt.store.reconcile(load());
6602        f.gate.add_permits(1);
6603        let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6604            .await
6605            .unwrap()
6606            .unwrap();
6607        assert!(answered.is_err(), "{answered:?}");
6608        let disk: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6609        assert_eq!(disk["siwc"]["refresh_token"], "siwc-ref-refreshed");
6610        // Go's write after a grant keeps the file's keys as written.
6611        assert_eq!(
6612            (&disk["request-retry"], &disk["request_retry"]),
6613            (&2.into(), &Value::Null)
6614        );
6615        assert_eq!(
6616            f.rt.store.get("codex.json").unwrap().metadata["siwc"]["refresh_token"],
6617            "siwc-ref-A"
6618        );
6619        // The watcher's next reload takes the new tokens.
6620        f.rt.store.reconcile(load());
6621        assert_eq!(
6622            f.rt.store.get("codex.json").unwrap().metadata["siwc"]["refresh_token"],
6623            "siwc-ref-refreshed"
6624        );
6625        std::fs::remove_dir_all(&f.dir).unwrap();
6626    }
6627
6628    /// A refresh publishes the proxy of the file it adopted, as requests route through
6629    /// it: a changed proxy replaces the loaded one, and a removed one is gone.
6630    #[tokio::test]
6631    async fn a_codex_refresh_routes_through_the_proxy_of_the_file_it_adopted() {
6632        let (days, hour) = (chrono::Duration::days(5), chrono::Duration::hours(1));
6633        let mut file = codex_file(("keep", days), ("A", "oaiapp_x", hour));
6634        file["proxy_url"] = "http://127.0.0.1:9/".into();
6635        let f = CodexDisk::new(&[("codex.json", &file)]).await;
6636        let cfg = f.rt.config();
6637        let route = || cpa_exec::proxy::Proxy::effective_url(&f.rt.store.get("codex.json").unwrap(), &cfg);
6638        assert_eq!(route(), "http://127.0.0.1:9/");
6639        for (version, proxy) in [("B", Some("http://127.0.0.1:8/")), ("C", None)] {
6640            let mut rotated = codex_file(("keep", days), (version, "oaiapp_x", hour));
6641            if let Some(proxy) = proxy {
6642                rotated["proxy_url"] = proxy.into();
6643            }
6644            std::fs::write(f.dir.join("codex.json"), rotated.to_string()).unwrap();
6645            let current = f.rt.store.get("codex.json").unwrap();
6646            assert!(f.rt.refresh_after_unauthorized(&current, &cfg).await.is_some());
6647            assert_eq!(route(), proxy.unwrap_or(""), "version {version}");
6648        }
6649        assert_eq!(f.grants(), []);
6650        std::fs::remove_dir_all(&f.dir).unwrap();
6651    }
6652
6653    /// A credential file emptied to `{}` while the grant ran fails the refresh at the
6654    /// write (Go: "invalid credential JSON ... before write"): nothing is written or
6655    /// published.
6656    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6657    async fn a_codex_refresh_fails_when_its_file_is_emptied_while_the_grant_runs() {
6658        let (days, minute) = (chrono::Duration::days(5), chrono::Duration::minutes(1));
6659        let f = CodexDisk::gated(
6660            &[("codex.json", &codex_file(("keep", days), ("A", "oaiapp_x", minute)))],
6661            0,
6662        )
6663        .await;
6664        let path = f.dir.join("codex.json");
6665        let refresh = tokio::spawn({
6666            let rt = f.rt.clone();
6667            async move { rt.prepare_credential("codex.json", &rt.config(), None).await }
6668        });
6669        f.granted().await;
6670        std::fs::write(&path, "{}").unwrap();
6671        f.gate.add_permits(1);
6672        let answered = tokio::time::timeout(Duration::from_secs(5), refresh)
6673            .await
6674            .unwrap()
6675            .unwrap();
6676        assert!(answered.is_err(), "{answered:?}");
6677        assert_eq!(std::fs::read_to_string(&path).unwrap(), "{}");
6678        assert_eq!(
6679            f.rt.store.get("codex.json").unwrap().metadata["siwc"]["refresh_token"],
6680            "siwc-ref-A"
6681        );
6682        std::fs::remove_dir_all(&f.dir).unwrap();
6683    }
6684
6685    /// Go 50d5fe18 and feb63984 (conductor_issue6415_test, conductor_issue6742_test): a
6686    /// 401 on a Codex token that is not due schedules a background refresh at once, which
6687    /// rotates it and clears the unauthorized cooldown, not another model's quota. A 401
6688    /// right after that refresh waits `refreshFailureBackoff`.
6689    #[tokio::test]
6690    async fn a_rejected_token_is_refreshed_in_the_background() {
6691        let (hit_tx, mut hit) = tokio::sync::mpsc::unbounded_channel();
6692        let app = axum::Router::new().route(
6693            "/oauth/token",
6694            axum::routing::post(move || async move {
6695                hit_tx.send(()).unwrap();
6696                r#"{"access_token":"rotated","refresh_token":"rotated-rt","expires_in":864000}"#
6697            }),
6698        );
6699        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6700        let token_url = format!("http://{}/oauth/token", listener.local_addr().unwrap());
6701        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6702        let dir = std::env::temp_dir().join(format!("cpa-refresh-401-{}", uuid::Uuid::new_v4()));
6703        std::fs::create_dir_all(&dir).unwrap();
6704        let expires = (chrono::Utc::now() + chrono::Duration::days(9)).to_rfc3339();
6705        std::fs::write(
6706            dir.join("codex.json"),
6707            serde_json::json!({"type": "codex", "access_token": "outer", "refresh_token": "outer-rt", "expired": expires})
6708                .to_string(),
6709        )
6710        .unwrap();
6711        let mut cfg = Config::parse("").unwrap();
6712        cfg.auth_dir = dir.clone();
6713        let client = wreq::Client::new();
6714        let oauth =
6715            cpa_exec::codex_oauth::CodexOAuth::with_endpoints(client.clone(), &token_url, &token_url, &token_url);
6716        let rt = Arc::new(crate::testing::runtime(
6717            cfg.clone(),
6718            cpa_core::config::credentials::from_auth_dir(&cfg)
6719                .into_iter()
6720                .map(crate::testing::local)
6721                .collect(),
6722            Executors {
6723                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
6724                codex: cpa_exec::codex::CodexExecutor::with_client(client, oauth),
6725                devices: Default::default(),
6726                openai: Default::default(),
6727                google: Default::default(),
6728            },
6729        ));
6730        rt.start_auto_refresh();
6731        async fn quiet(hit: &mut tokio::sync::mpsc::UnboundedReceiver<()>) {
6732            let pending = tokio::time::timeout(Duration::from_millis(1500), hit.recv());
6733            assert!(pending.await.is_err(), "no refresh");
6734        }
6735        quiet(&mut hit).await; // Nine days left: not due under the 96h lead.
6736        let mut quota = ExecError::local(429, FailureScope::Model, "rate limited");
6737        quota.retry_after = Some(Duration::from_secs(3600));
6738        let select = |model: &str| rt.store.select(Selection::new("codex", model)).unwrap();
6739        select("q").complete(Outcome::Failure(quota));
6740        let unauthorized = || Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
6741        select("m").complete(unauthorized());
6742        assert_eq!(rt.store.cooldowns("codex.json").len(), 2);
6743        tokio::time::timeout(Duration::from_secs(5), hit.recv())
6744            .await
6745            .expect("a refresh of the rejected token");
6746        let rotated = async {
6747            loop {
6748                let c = rt.store.get("codex.json").unwrap();
6749                if c.str("access_token") == Some("rotated") {
6750                    break c;
6751                }
6752                tokio::time::sleep(Duration::from_millis(20)).await;
6753            }
6754        };
6755        let rotated = tokio::time::timeout(Duration::from_secs(5), rotated).await.unwrap();
6756        let left = rt.store.cooldowns("codex.json");
6757        assert_eq!(
6758            left.iter().map(|c| (c.model.as_str(), c.status)).collect::<Vec<_>>(),
6759            [("q", 429)]
6760        );
6761        // Rejected again right after: the refresh waits out the backoff.
6762        rt.store
6763            .select(Selection::new("codex", "m"))
6764            .unwrap()
6765            .complete(unauthorized());
6766        assert_eq!(
6767            rt.store.unauthorized().pending.get("codex.json").map(|(r, _)| *r),
6768            Some(rotated.revision)
6769        );
6770        quiet(&mut hit).await;
6771        std::fs::remove_dir_all(dir).unwrap();
6772    }
6773
6774    /// Which 401s schedule a background refresh: Codex ones that cool the credential,
6775    /// including a request-scoped 401 a `*-and-cooldown` rule cools; not Claude's (that
6776    /// waits for the post-replay port). A scan whose snapshot predates a newer revision
6777    /// keeps the 401 recorded on it.
6778    #[test]
6779    fn which_401s_schedule_a_background_refresh() {
6780        let codex = |note: &str| {
6781            let mut c = cred("codex.json", "codex", false);
6782            c.metadata.insert("refresh_token".into(), "rt".into());
6783            c.metadata.insert("note".into(), note.into());
6784            c.metadata.insert(
6785                "request_scoped_errors".into(),
6786                serde_json::json!([{"status": 401, "match": ["revoked"], "action": "stop-and-cooldown"}]),
6787            );
6788            c
6789        };
6790        let mut claude = cred("claude.json", "claude", false);
6791        claude.metadata.insert("refresh_token".into(), "rt".into());
6792        let store = CredentialStore::new(vec![codex("a"), claude]);
6793        let fail = |provider: &str, model: &str, scope, text: &str| {
6794            store
6795                .select(Selection::new(provider, model))
6796                .unwrap()
6797                .complete(Outcome::Failure(ExecError::local(401, scope, text)));
6798        };
6799        let pending = || {
6800            let mut ids: Vec<_> = store
6801                .unauthorized()
6802                .pending
6803                .into_iter()
6804                .map(|(id, (r, _))| (id, r))
6805                .collect();
6806            ids.sort();
6807            ids
6808        };
6809        fail("claude", "m", FailureScope::Credential, "unauthorized");
6810        fail("codex", "m", FailureScope::Request, "bad request");
6811        assert_eq!(pending(), [], "Claude, and an unforced request-scoped 401");
6812        fail("codex", "m", FailureScope::Request, "token revoked");
6813        let first = store.get("codex.json").unwrap().revision;
6814        assert_eq!(pending(), [("codex.json".to_owned(), first)], "a forced cooldown");
6815        // The scan's snapshot holds the first revision; a refresh commits a newer one,
6816        // which an upstream rejects too, before the scan reads the rejections.
6817        let old = store.snapshot();
6818        store.reconcile(vec![
6819            codex("b"),
6820            store.get("claude.json").map(|c| (*c).clone()).unwrap(),
6821        ]);
6822        let second = store.get("codex.json").unwrap().revision;
6823        assert!(second > first);
6824        fail("codex", "n", FailureScope::Request, "token revoked");
6825        store.forget_stale_unauthorized(&store.unauthorized(), &old);
6826        assert_eq!(
6827            pending(),
6828            [("codex.json".to_owned(), second)],
6829            "a stale snapshot keeps it"
6830        );
6831        store.forget_stale_unauthorized(&store.unauthorized(), &store.snapshot());
6832        assert_eq!(pending(), [("codex.json".to_owned(), second)], "still current");
6833        let seen = store.unauthorized();
6834        store.reconcile(vec![
6835            codex("c"),
6836            store.get("claude.json").map(|c| (*c).clone()).unwrap(),
6837        ]);
6838        store.forget_stale_unauthorized(&seen, &store.snapshot());
6839        assert_eq!(pending(), [], "replaced");
6840    }
6841
6842    /// Go `mergeAuthContent` and `CredentialRefreshPatch.Apply` for a refresh that lost
6843    /// the race to another edit (metadata_merge_test.go, refresh_patch_test.go).
6844    #[test]
6845    fn a_stale_refresh_merges_like_go() {
6846        use serde_json::json;
6847        let map = |v: Value| v.as_object().unwrap().clone();
6848        let codex = json!({"type": "codex", "access_token": "a", "refresh_token": "r", "account_id": "acct", "email": "e@x", "plan_type": "plus"});
6849        let refresh = |plan: &str, email: &str| json!({"type": "codex", "access_token": "a2", "refresh_token": "r2", "account_id": "acct", "email": email, "plan_type": plan});
6850        let edit = |base: &Value, k: &str, v: Value| {
6851            let mut m = map(base.clone());
6852            m.insert(k.into(), v);
6853            Value::Object(m)
6854        };
6855        let kimi = json!({"type": "kimi", "note": "a", "expired": "t0", "gone": "a"});
6856        let siwc = json!({"type": "codex", "access_token": "a", "refresh_token": "r", "account_id": "acct",
6857            "siwc": {"access_token": "s", "refresh_token": "sr", "client_id": "c", "expired": "t0", "note": "n"}});
6858        let siwc_refresh = json!({"siwc": {"access_token": "s2", "refresh_token": "sr2", "client_id": "c", "expired": "t1", "note": "n"}});
6859        let siwc_edit = |k: &str, v: Value| {
6860            let mut m = map(siwc.clone());
6861            m["siwc"][k] = v;
6862            Value::Object(m)
6863        };
6864        for (name, provider, base, user, set, remove, want_set, want_remove) in [
6865            (
6866                "an unchanged value keeps the edit, a changed token is applied",
6867                "kimi",
6868                &kimi,
6869                edit(&kimi, "note", json!("b")),
6870                json!({"type": "kimi", "note": "a", "access_token": "new"}),
6871                vec![],
6872                json!({"access_token": "new"}),
6873                vec![],
6874            ),
6875            (
6876                "both changed a normal field: the edit wins",
6877                "kimi",
6878                &kimi,
6879                edit(&kimi, "note", json!("b")),
6880                json!({"note": "x"}),
6881                vec![],
6882                json!({}),
6883                vec![],
6884            ),
6885            (
6886                "both changed a token payload: the refresh wins",
6887                "kimi",
6888                &kimi,
6889                edit(&kimi, "expired", json!("t2")),
6890                json!({"expired": "t1"}),
6891                vec![],
6892                json!({"expired": "t1"}),
6893                vec![],
6894            ),
6895            (
6896                "a removal of an edited key is dropped, of an untouched key kept",
6897                "kimi",
6898                &kimi,
6899                edit(&kimi, "expired", json!("t2")),
6900                json!({}),
6901                vec!["expired", "gone"],
6902                json!({}),
6903                vec!["gone"],
6904            ),
6905            (
6906                "JSON null is an edit of a missing key",
6907                "kimi",
6908                &kimi,
6909                edit(&kimi, "fresh", Value::Null),
6910                json!({"fresh": "x"}),
6911                vec![],
6912                json!({}),
6913                vec![],
6914            ),
6915            (
6916                "Codex: an unchanged plan and email keep the edit",
6917                "codex",
6918                &codex,
6919                edit(&codex, "plan_type", json!("team")),
6920                refresh("plus", "e@x"),
6921                vec![],
6922                json!({"access_token": "a2", "refresh_token": "r2"}),
6923                vec![],
6924            ),
6925            (
6926                "Codex: a changed plan and email are applied over the edit",
6927                "codex",
6928                &codex,
6929                edit(&edit(&codex, "plan_type", json!("team")), "email", json!("u@x")),
6930                refresh("pro", "n@x"),
6931                vec![],
6932                json!({"access_token": "a2", "refresh_token": "r2", "plan_type": "pro", "email": "n@x"}),
6933                vec![],
6934            ),
6935            (
6936                "Codex SIWC: the refreshed fields land over an edit inside the partition",
6937                "codex",
6938                &siwc,
6939                siwc_edit("note", json!("edited")),
6940                siwc_refresh.clone(),
6941                vec![],
6942                json!({"siwc": {"access_token": "s2", "refresh_token": "sr2", "client_id": "c", "expired": "t1", "note": "edited"}}),
6943                vec![],
6944            ),
6945            (
6946                "Codex SIWC: another client drops the partition",
6947                "codex",
6948                &siwc,
6949                siwc_edit("client_id", json!("other")),
6950                siwc_refresh.clone(),
6951                vec![],
6952                json!({}),
6953                vec![],
6954            ),
6955            (
6956                "Codex SIWC: an edit already holding the result keeps it",
6957                "codex",
6958                &siwc,
6959                siwc_edit("access_token", json!("s2")),
6960                json!({"siwc": {"access_token": "s2", "refresh_token": "sr", "client_id": "c", "expired": "t0", "note": "n"}}),
6961                vec![],
6962                json!({}),
6963                vec![],
6964            ),
6965            (
6966                "Codex: another account drops the whole partition",
6967                "codex",
6968                &codex,
6969                edit(&codex, "account_id", json!("other")),
6970                refresh("pro", "n@x"),
6971                vec![],
6972                json!({}),
6973                vec![],
6974            ),
6975        ] {
6976            let patch = MetadataPatch {
6977                set: map(set),
6978                remove: remove.into_iter().map(str::to_owned).collect(),
6979            };
6980            let (out, _) = rebase_patch(provider, &map(base.clone()), &map(user), &patch);
6981            assert_eq!(Value::Object(out.set), want_set, "{name}");
6982            assert_eq!(out.remove, want_remove, "{name}");
6983        }
6984    }
6985
6986    /// Go `CredentialRefreshPatch.CanApplyTo`: a refreshed partition that still matches
6987    /// the stored version applies even after the other partition's tokens changed.
6988    #[test]
6989    fn a_matching_codex_partition_applies_after_the_other_changed() {
6990        use serde_json::json;
6991        let map = |v: Value| v.as_object().unwrap().clone();
6992        let base = json!({"type": "codex", "access_token": "a", "refresh_token": "r",
6993            "siwc": {"access_token": "s", "refresh_token": "sr", "client_id": "c"}});
6994        let patch = MetadataPatch {
6995            set: map(json!({"siwc": {"access_token": "s2", "refresh_token": "sr2", "client_id": "c"}})),
6996            remove: vec![],
6997        };
6998        let mut outer_changed = map(base.clone());
6999        outer_changed.insert("access_token".into(), "other".into());
7000        let (out, applies) = rebase_patch("codex", &map(base.clone()), &outer_changed, &patch);
7001        assert!(applies);
7002        assert_eq!(out.set["siwc"]["access_token"], "s2");
7003        let mut siwc_changed = map(base.clone());
7004        siwc_changed["siwc"]["refresh_token"] = "rotated-elsewhere".into();
7005        let (out, applies) = rebase_patch("codex", &map(base.clone()), &siwc_changed, &patch);
7006        assert!(!applies && out.set.is_empty());
7007        // Only Codex partitions qualify.
7008        assert!(!rebase_patch("claude", &map(base.clone()), &outer_changed, &patch).1);
7009    }
7010
7011    /// Go `CredentialsChanged` compares the token requests use, which a `use_siwc`
7012    /// attribute alone can switch.
7013    #[test]
7014    fn switching_to_the_siwc_token_changes_the_credentials() {
7015        let mut a = cred("codex.json", "codex", false);
7016        a.metadata.insert("access_token".into(), "outer".into());
7017        a.metadata
7018            .insert("siwc".into(), serde_json::json!({"access_token": "nested"}));
7019        let mut b = a.clone();
7020        assert!(!credentials_changed(&a, &b));
7021        b.attributes.insert("use_siwc".into(), "true".into());
7022        assert!(credentials_changed(&a, &b));
7023    }
7024
7025    /// Log lines written on this thread while the guard lives.
7026    #[derive(Clone, Default)]
7027    struct Logs(Arc<Mutex<Vec<u8>>>);
7028    impl std::io::Write for Logs {
7029        fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
7030            self.0.lock().unwrap().extend_from_slice(bytes);
7031            Ok(bytes.len())
7032        }
7033        fn flush(&mut self) -> std::io::Result<()> {
7034            Ok(())
7035        }
7036    }
7037    impl Logs {
7038        fn install(&self) -> tracing::subscriber::DefaultGuard {
7039            let writer = self.clone();
7040            let subscriber = tracing_subscriber::fmt()
7041                .with_ansi(false)
7042                .without_time()
7043                .with_writer(move || writer.clone())
7044                .finish();
7045            tracing::subscriber::set_default(subscriber)
7046        }
7047        fn text(&self) -> String {
7048            String::from_utf8(self.0.lock().unwrap().clone()).unwrap()
7049        }
7050        async fn wait_for(&self, needle: &str, count: usize) {
7051            let found = tokio::time::timeout(Duration::from_secs(10), async {
7052                while self.text().matches(needle).count() < count {
7053                    tokio::time::sleep(Duration::from_millis(20)).await;
7054                }
7055            });
7056            found
7057                .await
7058                .unwrap_or_else(|_| panic!("{needle:?} not logged {count} times:\n{}", self.text()));
7059        }
7060    }
7061
7062    /// A due Claude refresh that fails logs one warning with the status, the OAuth error
7063    /// and the next attempt, and is tried again when its backoff ends: invalid_grant
7064    /// after Go's 1 min, a 5xx (retried in place twice) after 5 min. A job that panics
7065    /// fails alone, a scan that panics restarts the loop, and the next success commits.
7066    /// Backoffs are ended by hand ([`RefreshState::expire`]) instead of waited for.
7067    #[tokio::test]
7068    async fn a_failed_background_refresh_is_logged_and_retried_after_its_backoff() {
7069        let answers = Arc::new(Mutex::new(std::collections::VecDeque::from([
7070            (
7071                400,
7072                r#"{"error":"invalid_grant","error_description":"Refresh token sk-ant-ort01-leaked is invalid"}"#,
7073            ),
7074            (500, r#"{"error":"server_error"}"#),
7075            (500, r#"{"error":"server_error"}"#),
7076            (500, r#"{"error":"server_error"}"#),
7077            (
7078                200,
7079                r#"{"access_token":"sk-ant-oat-new-fake","refresh_token":"fake-rt-2","expires_in":28800}"#,
7080            ),
7081        ])));
7082        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7083        let app = axum::Router::new().route(
7084            "/token",
7085            axum::routing::post({
7086                let (answers, calls) = (answers.clone(), calls.clone());
7087                move || async move {
7088                    calls.fetch_add(1, Ordering::SeqCst);
7089                    let (status, body) = answers.lock().unwrap().pop_front().expect("an unexpected exchange");
7090                    (axum::http::StatusCode::from_u16(status).unwrap(), body)
7091                }
7092            }),
7093        );
7094        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7095        let base = format!("http://{}", listener.local_addr().unwrap());
7096        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7097        let dir = std::env::temp_dir().join(format!("cpa-refresh-fail-{}", uuid::Uuid::new_v4()));
7098        std::fs::create_dir_all(&dir).unwrap();
7099        let path = dir.join("claude-backoff.json");
7100        std::fs::write(
7101            &path,
7102            serde_json::json!({"type": "claude", "access_token": "sk-ant-oat-old-fake", "refresh_token": "fake-rt-1",
7103                "expired": (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc3339(),
7104                "account_uuid": "acct-fake", "claude_device_ids": ["b".repeat(64)]})
7105            .to_string(),
7106        )
7107        .unwrap();
7108        let mut cfg = Config::parse("").unwrap();
7109        cfg.auth_dir = dir.clone();
7110        let (token, profile) = (format!("{base}/token"), format!("{base}/profile"));
7111        let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), &token, &profile, &profile);
7112        let rt = Arc::new(crate::testing::runtime(
7113            cfg.clone(),
7114            cpa_core::config::credentials::from_auth_dir(&cfg)
7115                .into_iter()
7116                .map(crate::testing::local)
7117                .collect(),
7118            Executors {
7119                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7120                    .unwrap()
7121                    .with_oauth(oauth),
7122                codex: Default::default(),
7123                devices: Default::default(),
7124                openai: Default::default(),
7125                google: Default::default(),
7126            },
7127        ));
7128        let logs = Logs::default();
7129        let _guard = logs.install();
7130        let failed = "credential refresh failed for claude (claude-backoff.json): status";
7131        let retry_in = |rt: &Runtime| {
7132            let c = rt.store.get("claude-backoff.json").unwrap();
7133            let at = rt.refresh_state.lock().unwrap().retry_at(&c).unwrap();
7134            at.saturating_duration_since(Instant::now())
7135        };
7136        // Ends the current backoff and wakes the loop.
7137        let expire = |rt: &Runtime| {
7138            let c = rt.store.get("claude-backoff.json").unwrap();
7139            rt.refresh_state.lock().unwrap().expire(&c);
7140            rt.config_changed.send_replace(());
7141        };
7142        rt.start_auto_refresh();
7143        logs.wait_for(failed, 1).await;
7144        assert!(
7145            logs.text().contains(&format!(
7146                "{failed} 400: Claude OAuth acquisition failed: invalid_grant; next attempt in 60s"
7147            )),
7148            "{}",
7149            logs.text()
7150        );
7151        assert!(
7152            (55..=60).contains(&retry_in(&rt).as_secs()),
7153            "Go invalidGrantBackoffBase"
7154        );
7155        // A job that panics is that job's failure; the loop goes on.
7156        tests::arm_panic("prepare", "claude-backoff.json");
7157        expire(&rt);
7158        logs.wait_for(failed, 2).await;
7159        assert!(
7160            logs.text()
7161                .contains(&format!("{failed} 500: refresh panicked: injected prepare panic"))
7162        );
7163        expire(&rt);
7164        logs.wait_for(failed, 3).await;
7165        assert!(
7166            logs.text().contains(&format!(
7167                "{failed} 500: Claude OAuth acquisition failed: server_error; next attempt in 300s"
7168            )),
7169            "{}",
7170            logs.text()
7171        );
7172        assert!(
7173            (295..=300).contains(&retry_in(&rt).as_secs()),
7174            "Go refreshFailureBackoff"
7175        );
7176        assert_eq!(
7177            calls.load(Ordering::SeqCst),
7178            4,
7179            "one invalid_grant, then three 5xx tries in place"
7180        );
7181        // A scan that panics restarts the loop, which then refreshes.
7182        tests::arm_panic("scan", "claude-backoff.json");
7183        expire(&rt);
7184        logs.wait_for("credential auto-refresh loop panicked (injected scan panic)", 1)
7185            .await;
7186        let committed = tokio::time::timeout(Duration::from_secs(10), async {
7187            loop {
7188                let c = rt.store.get("claude-backoff.json").unwrap();
7189                if c.str("access_token") == Some("sk-ant-oat-new-fake") {
7190                    break c;
7191                }
7192                tokio::time::sleep(Duration::from_millis(20)).await;
7193            }
7194        })
7195        .await
7196        .expect("the retry after the backoff commits");
7197        assert_eq!(committed.str("refresh_token"), Some("fake-rt-2"));
7198        let saved: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
7199        assert_eq!(saved["access_token"], "sk-ant-oat-new-fake");
7200        assert_eq!(logs.text().matches(failed).count(), 3, "{}", logs.text());
7201        let text = logs.text();
7202        assert!(
7203            !text.contains("fake-rt") && !text.contains("sk-ant-o"),
7204            "no token in the logs: {text}"
7205        );
7206        rt.stop_auto_refresh();
7207        std::fs::remove_dir_all(dir).unwrap();
7208    }
7209
7210    /// A Claude runtime over one credential file in a private auth-dir whose token
7211    /// endpoint answers every refresh with `answer`; counts the exchanges.
7212    async fn claude_refusing(
7213        name: &str,
7214        file: Value,
7215        answer: (u16, &'static str),
7216    ) -> (Arc<Runtime>, std::path::PathBuf, Arc<std::sync::atomic::AtomicUsize>) {
7217        claude_gated(name, file, answer, None).await
7218    }
7219
7220    /// Holds [`claude_gated`]'s token exchanges: each notifies `entered`, then answers
7221    /// once it takes a `release` permit.
7222    #[derive(Clone)]
7223    struct Gate {
7224        entered: Arc<tokio::sync::Notify>,
7225        release: Arc<tokio::sync::Semaphore>,
7226    }
7227
7228    impl Gate {
7229        fn new() -> Self {
7230            Self {
7231                entered: Arc::new(tokio::sync::Notify::new()),
7232                release: Arc::new(tokio::sync::Semaphore::new(0)),
7233            }
7234        }
7235    }
7236
7237    /// [`claude_refusing`] whose exchanges wait on `gate`.
7238    async fn claude_gated(
7239        name: &str,
7240        file: Value,
7241        answer: (u16, &'static str),
7242        gate: Option<Gate>,
7243    ) -> (Arc<Runtime>, std::path::PathBuf, Arc<std::sync::atomic::AtomicUsize>) {
7244        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7245        let app = axum::Router::new().route(
7246            "/token",
7247            axum::routing::post({
7248                let calls = calls.clone();
7249                move || async move {
7250                    calls.fetch_add(1, Ordering::SeqCst);
7251                    if let Some(gate) = gate {
7252                        gate.entered.notify_one();
7253                        gate.release.acquire().await.unwrap().forget();
7254                    }
7255                    (axum::http::StatusCode::from_u16(answer.0).unwrap(), answer.1)
7256                }
7257            }),
7258        );
7259        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7260        let base = format!("http://{}", listener.local_addr().unwrap());
7261        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7262        let dir = std::env::temp_dir().join(format!("cpa-{name}-{}", uuid::Uuid::new_v4()));
7263        std::fs::create_dir_all(&dir).unwrap();
7264        std::fs::write(dir.join(format!("{name}.json")), file.to_string()).unwrap();
7265        let mut cfg = Config::parse("").unwrap();
7266        cfg.auth_dir = dir.clone();
7267        let (token, profile) = (format!("{base}/token"), format!("{base}/profile"));
7268        let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), &token, &profile, &profile);
7269        let rt = Arc::new(crate::testing::runtime(
7270            cfg.clone(),
7271            cpa_core::config::credentials::from_auth_dir(&cfg)
7272                .into_iter()
7273                .map(crate::testing::local)
7274                .collect(),
7275            Executors {
7276                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7277                    .unwrap()
7278                    .with_oauth(oauth),
7279                codex: Default::default(),
7280                devices: Default::default(),
7281                openai: Default::default(),
7282                google: Default::default(),
7283            },
7284        ));
7285        (rt, dir, calls)
7286    }
7287
7288    fn claude_file(access: &str, expires_in: chrono::Duration, disabled: bool) -> Value {
7289        serde_json::json!({"type": "claude", "access_token": access, "refresh_token": "fake-rt-1",
7290            "expired": (chrono::Utc::now() + expires_in).to_rfc3339(), "disabled": disabled,
7291            "account_uuid": "acct-fake", "claude_device_ids": ["b".repeat(64)]})
7292    }
7293
7294    /// Go `refreshAuthForRequestAtEpoch`: when the refresh after an upstream 401 is
7295    /// refused with invalid_grant while the refused access token is still current, the
7296    /// credential is neither selected nor refreshed again until its tokens change; a
7297    /// file with new tokens lifts that at once.
7298    #[tokio::test]
7299    async fn an_invalid_grant_after_a_401_blocks_the_credential_until_its_tokens_change() {
7300        let name = "claude-revoked";
7301        let id = format!("{name}.json");
7302        let refused = (400, r#"{"error":"invalid_grant","error_description":"revoked"}"#);
7303        let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7304        let (rt, dir, calls) = claude_refusing(name, file, refused).await;
7305        let cfg = rt.config();
7306        let claude = || Selection {
7307            provider: "claude".into(),
7308            ..Selection::default()
7309        };
7310        let rejected = rt.store.get(&id).unwrap();
7311        assert!(rt.store.select(claude()).is_some(), "selectable before the 401");
7312        // Two requests that both got the 401: the second waits on the preparation lock,
7313        // then finds the rejection and sends nothing.
7314        let both = [0, 1].map(|_| {
7315            let (rt, rejected, cfg) = (rt.clone(), rejected.clone(), cfg.clone());
7316            tokio::spawn(async move { rt.refresh_after_unauthorized(&rejected, &cfg).await })
7317        });
7318        for job in both {
7319            assert!(job.await.unwrap().is_none());
7320        }
7321        assert_eq!(calls.load(Ordering::SeqCst), 1, "one token request for both 401s");
7322        let current = rt.store.get(&id).unwrap();
7323        assert!(rt.store.tokens_rejected(&current));
7324        assert!(rt.store.blocked(&current, ""));
7325        assert!(rt.store.select(claude()).is_none(), "no longer selected");
7326        // A refresh that starts after the rejection refuses too.
7327        assert_eq!(rt.prepare_credential(&id, &cfg, None).await.unwrap_err().status, 401);
7328        assert_eq!(calls.load(Ordering::SeqCst), 1);
7329        // The background loop leaves it alone too, and so does the credits fallback.
7330        assert!(rt.store.rejected_ids(&rt.store.snapshot()).contains(&id));
7331        let lease = rt.lease_credential(&id, claude(), &cfg, |c| c.clone()).await;
7332        assert!(lease.is_none(), "the credits fallback leased a refused credential");
7333        // Deleted, then imported again with the same tokens: a new credential.
7334        let reload = |rt: &Runtime| {
7335            rt.store.reconcile(
7336                cpa_core::config::credentials::from_auth_dir(&cfg)
7337                    .into_iter()
7338                    .map(crate::testing::local)
7339                    .collect(),
7340            )
7341        };
7342        let bytes = std::fs::read(dir.join(&id)).unwrap();
7343        std::fs::remove_file(dir.join(&id)).unwrap();
7344        reload(&rt);
7345        assert!(rt.store.get(&id).is_none());
7346        std::fs::write(dir.join(&id), &bytes).unwrap();
7347        reload(&rt);
7348        let imported = rt.store.get(&id).unwrap();
7349        assert!(
7350            !rt.store.tokens_rejected(&imported),
7351            "a re-import inherited the rejection"
7352        );
7353        assert!(rt.store.select(claude()).is_some());
7354        assert!(rt.refresh_after_unauthorized(&imported, &cfg).await.is_none());
7355        assert!(rt.store.tokens_rejected(&rt.store.get(&id).unwrap()));
7356        // A new sign-in writes other tokens: selectable again.
7357        let file = claude_file("sk-ant-oat-new-fake", chrono::Duration::hours(1), false);
7358        std::fs::write(dir.join(&id), file.to_string()).unwrap();
7359        rt.store.reconcile(
7360            cpa_core::config::credentials::from_auth_dir(&cfg)
7361                .into_iter()
7362                .map(crate::testing::local)
7363                .collect(),
7364        );
7365        let replaced = rt.store.get(&id).unwrap();
7366        assert!(!rt.store.tokens_rejected(&replaced));
7367        assert_eq!(rt.store.select(claude()).unwrap().credential.id, id);
7368        std::fs::remove_dir_all(dir).unwrap();
7369    }
7370
7371    /// Refreshes queued on the preparation lock behind a 401 refresh that is then refused
7372    /// with invalid_grant (a request's due preparation and a background 401 job) find
7373    /// the rejection under the lock and send nothing.
7374    #[tokio::test]
7375    async fn refreshes_queued_behind_a_refused_one_send_nothing() {
7376        let name = "claude-queued";
7377        let id = format!("{name}.json");
7378        let refused = (400, r#"{"error":"invalid_grant"}"#);
7379        let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7380        let gate = Gate::new();
7381        let (rt, dir, calls) = claude_gated(name, file, refused, Some(gate.clone())).await;
7382        let cfg = rt.config();
7383        let credential = rt.store.get(&id).unwrap();
7384        let first = {
7385            let (rt, c, cfg) = (rt.clone(), credential.clone(), cfg.clone());
7386            tokio::spawn(async move { rt.refresh_after_unauthorized(&c, &cfg).await })
7387        };
7388        gate.entered.notified().await;
7389        let queued = {
7390            let (rt, id, cfg) = (rt.clone(), id.clone(), cfg.clone());
7391            tokio::spawn(async move { rt.prepare_credential(&id, &cfg, None).await })
7392        };
7393        let job = tokio::spawn(run_refresh_job(
7394            rt.clone(),
7395            cfg.clone(),
7396            Arc::new(tokio::sync::Semaphore::new(1)),
7397            credential.clone(),
7398            true,
7399        ));
7400        // Both reach the preparation lock while the exchange is held.
7401        tokio::time::sleep(Duration::from_millis(300)).await;
7402        gate.release.add_permits(3);
7403        assert!(first.await.unwrap().is_none());
7404        assert_eq!(queued.await.unwrap().unwrap_err().status, 401);
7405        job.await.unwrap();
7406        assert_eq!(
7407            calls.load(Ordering::SeqCst),
7408            1,
7409            "a queued refresh sent another exchange"
7410        );
7411        assert!(rt.store.tokens_rejected(&rt.store.get(&id).unwrap()));
7412        std::fs::remove_dir_all(dir).unwrap();
7413    }
7414
7415    /// Go `refreshAuthForRequestAtEpoch` from the scheduled 401 refresh: when the refresh
7416    /// right after the 401 failed for another reason, the background job that retries it
7417    /// records the rejection on invalid_grant.
7418    #[tokio::test]
7419    async fn a_background_401_refresh_records_the_rejection() {
7420        let name = "claude-revoked-later";
7421        let id = format!("{name}.json");
7422        let refused = (400, r#"{"error":"invalid_grant"}"#);
7423        let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7424        let (rt, dir, calls) = claude_refusing(name, file, refused).await;
7425        let credential = rt.store.get(&id).unwrap();
7426        let workers = Arc::new(tokio::sync::Semaphore::new(1));
7427        run_refresh_job(rt.clone(), rt.config(), workers, credential, true).await;
7428        assert_eq!(calls.load(Ordering::SeqCst), 1);
7429        assert!(
7430            rt.store.tokens_rejected(&rt.store.get(&id).unwrap()),
7431            "the scheduled 401 refresh recorded nothing"
7432        );
7433        std::fs::remove_dir_all(dir).unwrap();
7434    }
7435
7436    /// A refusal that lands after its credential was removed and imported again with the
7437    /// same tokens rejects nothing: the import is a new registration (Go
7438    /// `RegistrationEpoch`).
7439    #[tokio::test]
7440    async fn a_late_refusal_never_blocks_a_reimported_credential() {
7441        let name = "claude-reimported";
7442        let id = format!("{name}.json");
7443        let refused = (400, r#"{"error":"invalid_grant"}"#);
7444        let file = claude_file("sk-ant-oat-old-fake", chrono::Duration::hours(1), false);
7445        let gate = Gate::new();
7446        let (rt, dir, calls) = claude_gated(name, file, refused, Some(gate.clone())).await;
7447        let cfg = rt.config();
7448        let claude = || Selection {
7449            provider: "claude".into(),
7450            ..Selection::default()
7451        };
7452        let credential = rt.store.get(&id).unwrap();
7453        let job = {
7454            let (rt, c, cfg) = (rt.clone(), credential.clone(), cfg.clone());
7455            tokio::spawn(async move { rt.refresh_after_unauthorized(&c, &cfg).await })
7456        };
7457        gate.entered.notified().await;
7458        let reload = |rt: &Runtime| {
7459            rt.store.reconcile(
7460                cpa_core::config::credentials::from_auth_dir(&cfg)
7461                    .into_iter()
7462                    .map(crate::testing::local)
7463                    .collect(),
7464            )
7465        };
7466        let bytes = std::fs::read(dir.join(&id)).unwrap();
7467        std::fs::remove_file(dir.join(&id)).unwrap();
7468        reload(&rt);
7469        assert!(rt.store.get(&id).is_none());
7470        std::fs::write(dir.join(&id), &bytes).unwrap();
7471        reload(&rt);
7472        gate.release.add_permits(1);
7473        assert!(job.await.unwrap().is_none());
7474        assert_eq!(calls.load(Ordering::SeqCst), 1);
7475        let imported = rt.store.get(&id).unwrap();
7476        assert!(
7477            !rt.store.tokens_rejected(&imported),
7478            "the old registration's refusal blocked the import"
7479        );
7480        assert_eq!(rt.store.select(claude()).unwrap().credential.id, id);
7481        std::fs::remove_dir_all(dir).unwrap();
7482    }
7483
7484    /// A rejection is recorded only for the tokens the store still holds: a failure for
7485    /// tokens A that lands after a rotation to B neither blocks B nor lifts B's own
7486    /// rejection; an empty access token is never rejected (Go `accessTokenRejected`).
7487    #[tokio::test]
7488    async fn a_stale_rejection_never_blocks_or_unblocks_rotated_tokens() {
7489        let name = "claude-rotated";
7490        let id = format!("{name}.json");
7491        let file = claude_file("sk-ant-oat-a-fake", chrono::Duration::hours(1), false);
7492        let (rt, dir, _) = claude_refusing(name, file, (500, "{}")).await;
7493        let cfg = rt.config();
7494        let invalid = ExecError::local(400, FailureScope::Request, r#"{"error":"invalid_grant"}"#);
7495        let a = rt.store.get(&id).unwrap();
7496        let registration = rt.store.get_registered(&id).unwrap().1.registration;
7497        let other = ExecError::local(400, FailureScope::Request, r#"{"error":"server_error"}"#);
7498        assert!(!rt.store.reject_refused(&a, registration, &other), "not invalid_grant");
7499        std::fs::write(
7500            dir.join(&id),
7501            claude_file("sk-ant-oat-b-fake", chrono::Duration::hours(1), false).to_string(),
7502        )
7503        .unwrap();
7504        rt.store.reconcile(
7505            cpa_core::config::credentials::from_auth_dir(&cfg)
7506                .into_iter()
7507                .map(crate::testing::local)
7508                .collect(),
7509        );
7510        let b = rt.store.get(&id).unwrap();
7511        assert!(!rt.store.reject_refused(&a, registration, &invalid));
7512        assert!(!rt.store.tokens_rejected(&b), "a stale failure blocked the new tokens");
7513        assert!(rt.store.reject_refused(&b, registration, &invalid));
7514        assert!(!rt.store.reject_refused(&a, registration, &invalid));
7515        assert!(
7516            rt.store.tokens_rejected(&b),
7517            "a stale failure lifted the new tokens' rejection"
7518        );
7519        // A file whose access token is blank: nothing to compare, so never rejected.
7520        std::fs::write(
7521            dir.join(&id),
7522            claude_file(" ", chrono::Duration::hours(1), false).to_string(),
7523        )
7524        .unwrap();
7525        rt.store.reconcile(
7526            cpa_core::config::credentials::from_auth_dir(&cfg)
7527                .into_iter()
7528                .map(crate::testing::local)
7529                .collect(),
7530        );
7531        let empty = rt.store.get(&id).unwrap();
7532        assert_eq!(empty.str("access_token"), Some(" "));
7533        assert!(
7534            !rt.store.reject_refused(&empty, registration, &invalid),
7535            "empty access token"
7536        );
7537        assert!(!rt.store.tokens_rejected(&empty));
7538        std::fs::remove_dir_all(dir).unwrap();
7539    }
7540
7541    /// A due refresh of a disabled credential is skipped with one log line per
7542    /// revision, and nothing is sent to the token endpoint.
7543    #[tokio::test]
7544    async fn a_skipped_refresh_of_a_disabled_credential_is_logged_once() {
7545        let name = "claude-off";
7546        // Claude refreshes 4 h before expiry: due 3 s after the loop's first scan, which
7547        // must wake for it on its own.
7548        let expires = chrono::Duration::hours(4) + chrono::Duration::seconds(3);
7549        let file = claude_file("sk-ant-oat-old-fake", expires, true);
7550        let (rt, dir, calls) = claude_refusing(name, file, (500, "{}")).await;
7551        let logs = Logs::default();
7552        let _guard = logs.install();
7553        let skipped = "credential refresh for claude (claude-off.json) is due but the credential is disabled";
7554        rt.start_auto_refresh();
7555        logs.wait_for(skipped, 1).await;
7556        for _ in 0..2 {
7557            rt.config_changed.send_replace(());
7558            tokio::time::sleep(Duration::from_millis(1500)).await;
7559        }
7560        assert_eq!(logs.text().matches(skipped).count(), 1, "{}", logs.text());
7561        assert_eq!(calls.load(Ordering::SeqCst), 0);
7562        rt.stop_auto_refresh();
7563        std::fs::remove_dir_all(dir).unwrap();
7564    }
7565
7566    /// Claude credential files `ids` due for a refresh within the hour (each with its own
7567    /// refresh token, `fake-rt-<id>`), with a token endpoint answering each request body
7568    /// by `answer` on loopback, and their runtime under config `yaml`. Test hooks are keyed by credential ID, so each test uses its own IDs.
7569    async fn claude_refresh_fixture<F, Fut>(ids: &[&str], yaml: &str, answer: F) -> (Arc<Runtime>, std::path::PathBuf)
7570    where
7571        F: Fn(String) -> Fut + Clone + Send + Sync + 'static,
7572        Fut: std::future::Future<Output = (axum::http::StatusCode, &'static str)> + Send + 'static,
7573    {
7574        let app = axum::Router::new().route("/token", axum::routing::post(move |body: String| answer(body)));
7575        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7576        let base = format!("http://{}", listener.local_addr().unwrap());
7577        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7578        let dir = std::env::temp_dir().join(format!("cpa-refresh-{}", uuid::Uuid::new_v4()));
7579        std::fs::create_dir_all(&dir).unwrap();
7580        for id in ids {
7581            std::fs::write(
7582                dir.join(id),
7583                serde_json::json!({"type": "claude", "access_token": "sk-ant-oat-old-fake",
7584                    "refresh_token": format!("fake-rt-{id}"),
7585                    "expired": (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc3339(),
7586                    "account_uuid": "acct-fake", "claude_device_ids": ["b".repeat(64)]})
7587                .to_string(),
7588            )
7589            .unwrap();
7590        }
7591        let mut cfg = Config::parse(yaml).unwrap();
7592        cfg.auth_dir = dir.clone();
7593        let (token, profile) = (format!("{base}/token"), format!("{base}/profile"));
7594        let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), &token, &profile, &profile);
7595        let rt = Arc::new(crate::testing::runtime(
7596            cfg.clone(),
7597            cpa_core::config::credentials::from_auth_dir(&cfg)
7598                .into_iter()
7599                .map(crate::testing::local)
7600                .collect(),
7601            Executors {
7602                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7603                    .unwrap()
7604                    .with_oauth(oauth),
7605                codex: Default::default(),
7606                devices: Default::default(),
7607                openai: Default::default(),
7608                google: Default::default(),
7609            },
7610        ));
7611        (rt, dir)
7612    }
7613
7614    /// Waits until credential `id` holds the refreshed access token.
7615    async fn refreshed(rt: &Runtime, id: &str) -> Arc<Credential> {
7616        tokio::time::timeout(Duration::from_secs(10), async {
7617            loop {
7618                let c = rt.store.get(id).unwrap();
7619                if c.str("access_token") == Some("sk-ant-oat-new-fake") {
7620                    break c;
7621                }
7622                tokio::time::sleep(Duration::from_millis(20)).await;
7623            }
7624        })
7625        .await
7626        .unwrap_or_else(|_| panic!("{id} was not refreshed"))
7627    }
7628
7629    /// Go `markRefreshPending`: a job past [`REFRESH_JOB_BOUND`] (2 s under test, counted
7630    /// from when it got a worker) keeps its credential reserved, so no second job starts
7631    /// while it runs, even after the backoff ends, but gives up its worker: with one
7632    /// worker, another credential that comes due meanwhile refreshes while the first's
7633    /// token call is still held. The held job's own result, a failure, is recorded when
7634    /// it ends, and that wakes the loop: with no event after it, the rejected credential
7635    /// (whose scan sets no other timer) is retried once its backoff ends, and commits.
7636    #[tokio::test]
7637    async fn a_job_past_its_bound_stays_the_only_one_and_wakes_the_loop_when_it_ends() {
7638        let (slow, quick) = ("claude-slow.json", "claude-quick.json");
7639        let release = Arc::new(tokio::sync::Notify::new());
7640        let slow_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7641        let answer = {
7642            let (release, slow_calls) = (release.clone(), slow_calls.clone());
7643            move |body: String| {
7644                let (release, slow_calls) = (release.clone(), slow_calls.clone());
7645                async move {
7646                    if body.contains(&format!("fake-rt-{slow}")) && slow_calls.fetch_add(1, Ordering::SeqCst) == 0 {
7647                        release.notified().await;
7648                        return (axum::http::StatusCode::BAD_REQUEST, r#"{"error":"invalid_request"}"#);
7649                    }
7650                    let ok = r#"{"access_token":"sk-ant-oat-new-fake","refresh_token":"fake-rt-2","expires_in":28800}"#;
7651                    (axum::http::StatusCode::OK, ok)
7652                }
7653            }
7654        };
7655        let workers = "oauth:\n  auth-auto-refresh-workers: 1\n";
7656        let (rt, dir) = claude_refresh_fixture(&[slow, quick], workers, answer).await;
7657        // Neither is due on its own: an upstream 401 is what schedules the slow one.
7658        let set_expiry = |id: &str, ahead: chrono::Duration| {
7659            let mut patch = MetadataPatch::default();
7660            patch
7661                .set
7662                .insert("expired".into(), (chrono::Utc::now() + ahead).to_rfc3339().into());
7663            let c = rt.store.get(id).unwrap();
7664            rt.store.apply_patch(id, c.revision, &patch).unwrap()
7665        };
7666        set_expiry(quick, chrono::Duration::days(9));
7667        let c = set_expiry(slow, chrono::Duration::days(9));
7668        rt.store
7669            .unauthorized
7670            .lock()
7671            .unwrap()
7672            .pending
7673            .insert(slow.into(), (c.revision, Instant::now()));
7674        crate::refresh::override_backoff(slow, Duration::from_secs(1));
7675        let logs = Logs::default();
7676        let _guard = logs.install();
7677        rt.start_auto_refresh();
7678        logs.wait_for(
7679            &format!("credential refresh for claude ({slow}) still running after 2s"),
7680            1,
7681        )
7682        .await;
7683        // The only worker is free again: a credential that comes due now refreshes.
7684        set_expiry(quick, chrono::Duration::hours(1));
7685        refreshed(&rt, quick).await;
7686        // The backoff ends twice while the job runs; neither forced scan starts another.
7687        for _ in 0..2 {
7688            let scans = tests::reached("scan", slow);
7689            rt.refresh_state.lock().unwrap().expire(&c);
7690            rt.config_changed.send_replace(());
7691            tokio::time::timeout(Duration::from_secs(10), async {
7692                while tests::reached("scan", slow) == scans {
7693                    tokio::time::sleep(Duration::from_millis(20)).await;
7694                }
7695            })
7696            .await
7697            .expect("a forced scan");
7698        }
7699        assert_eq!(slow_calls.load(Ordering::SeqCst), 1);
7700        assert_eq!(tests::reached("prepare", slow), 1, "one job while it runs");
7701        release.notify_one();
7702        logs.wait_for(
7703            &format!("credential refresh failed for claude ({slow}) after an upstream 401: status 400"),
7704            1,
7705        )
7706        .await;
7707        // No event from here on: only the ended job can wake the loop.
7708        let committed = refreshed(&rt, slow).await;
7709        assert_eq!(slow_calls.load(Ordering::SeqCst), 2);
7710        // The job commits before it ends its reservation.
7711        tokio::time::timeout(Duration::from_secs(5), async {
7712            while rt.refresh_state.lock().unwrap().running(&committed) {
7713                tokio::time::sleep(Duration::from_millis(10)).await;
7714            }
7715        })
7716        .await
7717        .expect("the job ends its reservation");
7718        let text = logs.text();
7719        assert_eq!(text.matches("still running after").count(), 1, "{text}");
7720        assert_eq!(
7721            text.matches("its previous refresh is still running").count(),
7722            1,
7723            "{text}"
7724        );
7725        assert_eq!(text.matches("credential refresh failed").count(), 1, "{text}");
7726        rt.stop_auto_refresh();
7727        std::fs::remove_dir_all(dir).unwrap();
7728    }
7729
7730    /// Every reservation belongs to a spawned job from the moment it is made, and the job
7731    /// ends it on every path: a scan that panics after reserving a credential restarts
7732    /// the loop without stranding that credential, whose job then panics in preparation
7733    /// (after the loop that started it is gone), backs off and is retried.
7734    #[tokio::test]
7735    async fn no_reservation_is_stranded_by_a_panicking_scan_or_job() {
7736        let ids = ["claude-strand-a.json", "claude-strand-b.json"];
7737        let answer = |_| async {
7738            let ok = r#"{"access_token":"sk-ant-oat-new-fake","refresh_token":"fake-rt-2","expires_in":28800}"#;
7739            (axum::http::StatusCode::OK, ok)
7740        };
7741        let (rt, dir) = claude_refresh_fixture(&ids, "", answer).await;
7742        let order: Vec<String> = rt.store.snapshot().iter().map(|c| c.id.clone()).collect();
7743        let (first, second) = (order[0].as_str(), order[1].as_str());
7744        crate::refresh::override_backoff(first, Duration::from_secs(1));
7745        tests::arm_panic("scan", second);
7746        tests::arm_panic("prepare", first);
7747        let logs = Logs::default();
7748        let _guard = logs.install();
7749        rt.start_auto_refresh();
7750        refreshed(&rt, first).await;
7751        refreshed(&rt, second).await;
7752        assert_eq!(tests::reached("prepare", first), 2, "the panicked job, then its retry");
7753        assert!(
7754            logs.text()
7755                .contains("credential auto-refresh loop panicked (injected scan panic)")
7756        );
7757        assert!(logs.text().contains(&format!(
7758            "credential refresh failed for claude ({first}): status 500: refresh panicked"
7759        )));
7760        rt.stop_auto_refresh();
7761        std::fs::remove_dir_all(dir).unwrap();
7762    }
7763
7764    /// A plugin-routed refresh job that panics records no deadline of the plugin's, so
7765    /// the refresh state backs it off like any failure instead of starting it again on
7766    /// the next scan.
7767    #[cfg(unix)]
7768    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7769    #[cfg_attr(feature = "media-relay", ignore = "needs the Go plugin toolchain")]
7770    async fn a_panicking_plugin_refresh_backs_off() {
7771        let tmp = std::env::temp_dir();
7772        let Some(built) = cpa_plugin::testing::built_plugins(&tmp, "a_panicking_plugin_refresh") else {
7773            return;
7774        };
7775        let dir = cpa_plugin::testing::scratch(&tmp, "plugin-refresh-panic");
7776        let (auths, plugins, records) = (dir.join("auths"), dir.join("plugins"), dir.join("records"));
7777        for d in [&auths, &plugins, &records] {
7778            std::fs::create_dir_all(d).unwrap();
7779        }
7780        std::fs::copy(built.join("recorder.so"), plugins.join("acme.so")).unwrap();
7781        std::fs::write(
7782            auths.join("p.json"),
7783            r#"{"type":"acme","access_token":"p","refresh_interval":"1h"}"#,
7784        )
7785        .unwrap();
7786        let yaml = format!(
7787            "auth-dir: {}\nplugins:\n  enabled: true\n  dir: {}\n  configs:\n    acme:\n      enabled: true\n      label: acme\n      record: {}\n      caps: auth_provider\n",
7788            auths.display(),
7789            plugins.display(),
7790            records.display()
7791        );
7792        let cfg = Config::parse(&yaml).unwrap();
7793        let credentials = cpa_core::config::credentials::load(&cfg);
7794        let rt = Arc::new(crate::testing::runtime(
7795            cfg,
7796            credentials,
7797            Executors {
7798                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
7799                codex: Default::default(),
7800                devices: Default::default(),
7801                openai: Default::default(),
7802                google: Default::default(),
7803            },
7804        ));
7805        crate::plugins::start(&rt).await;
7806        // As after the first plugin-aware load.
7807        rt.store.end_bootstrap();
7808        let c = rt.store.get("p.json").unwrap();
7809        assert!(
7810            crate::plugins::refresh::route(&rt, &c, &rt.config()).is_some(),
7811            "plugin-routed"
7812        );
7813        tests::arm_panic("prepare", "p.json");
7814        rt.start_auto_refresh();
7815        let backed_off = tokio::time::timeout(Duration::from_secs(10), async {
7816            loop {
7817                let c = rt.store.get("p.json").unwrap();
7818                let retry = rt.refresh_state.lock().unwrap().retry_at(&c);
7819                if retry.is_some_and(|at| at > Instant::now() + Duration::from_secs(250)) {
7820                    break;
7821                }
7822                tokio::time::sleep(Duration::from_millis(20)).await;
7823            }
7824        });
7825        if backed_off.await.is_err() {
7826            let c = rt.store.get("p.json").unwrap();
7827            panic!(
7828                "no backoff: reached {}, retry {:?}, due {:?}, schedule {:?}",
7829                tests::reached("prepare", "p.json"),
7830                rt.refresh_state
7831                    .lock()
7832                    .unwrap()
7833                    .retry_at(&c)
7834                    .map(|at| at.saturating_duration_since(Instant::now())),
7835                crate::plugins::refresh::due_at(&rt, &c, &rt.config()),
7836                rt.store.refresh_schedules().lock().unwrap().get("p.json").copied()
7837            );
7838        }
7839        tokio::time::sleep(Duration::from_millis(1500)).await;
7840        assert_eq!(
7841            tests::reached("prepare", "p.json"),
7842            1,
7843            "no second attempt before the backoff"
7844        );
7845        rt.stop_auto_refresh();
7846        std::fs::remove_dir_all(dir).unwrap();
7847    }
7848
7849    /// Of a plugin's refresh failure only the status and a known error code are logged:
7850    /// its free message may hold anything, an unlabelled token included, which the
7851    /// pattern-based sanitizer cannot see.
7852    #[test]
7853    fn a_plugin_refresh_failure_logs_no_plugin_message() {
7854        let logs = Logs::default();
7855        let _guard = logs.install();
7856        let c = cred("p.json", "acme", false);
7857        for code in ["invalid_grant", "rejected"] {
7858            let error = crate::plugins::refresh::call_error(cpa_plugin::rpc::CallError::Rpc {
7859                code: code.into(),
7860                message: "rejected abc123xyz456opaque".into(),
7861                status: 401,
7862            });
7863            warn_refresh_failed(&c, "", &error, None);
7864        }
7865        let text = logs.text();
7866        assert!(
7867            text.contains(
7868                "credential refresh failed for acme (p.json): status 401: plugin refresh failed: invalid_grant;"
7869            ),
7870            "{text}"
7871        );
7872        assert!(
7873            text.contains("status 401: plugin refresh failed; next attempt"),
7874            "an unknown code, which could be a token, is left out: {text}"
7875        );
7876        assert!(!text.contains("abc123"), "{text}");
7877    }
7878
7879    /// The loop sets no timer while nothing can come due, and a credential added later
7880    /// is tried after the store change, not on a schedule.
7881    #[tokio::test]
7882    async fn refresh_loop_sleeps_until_the_store_changes() {
7883        let unreachable = "http://127.0.0.1:1/token";
7884        let oauth = cpa_exec::oauth::OAuth::with_endpoints(wreq::Client::new(), unreachable, unreachable, unreachable);
7885        let rt = Arc::new(crate::testing::runtime(
7886            Config::parse("").unwrap(),
7887            Vec::new(),
7888            Executors {
7889                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1")
7890                    .unwrap()
7891                    .with_oauth(oauth),
7892                codex: Default::default(),
7893                devices: Default::default(),
7894                openai: Default::default(),
7895                google: Default::default(),
7896            },
7897        ));
7898        rt.start_auto_refresh();
7899        tokio::time::sleep(Duration::from_millis(50)).await;
7900        let mut due = cred("a.json", "claude", false);
7901        due.metadata.insert("refresh_token".into(), "fake-refresh".into());
7902        due.metadata.insert(
7903            "expired".into(),
7904            (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339().into(),
7905        );
7906        rt.store.reconcile(vec![due]);
7907        let added = rt.store.get("a.json").unwrap();
7908        // The refresh fails (no token endpoint), which backs off for five minutes.
7909        tokio::time::timeout(Duration::from_secs(20), async {
7910            loop {
7911                let retry = rt.refresh_state.lock().unwrap().retry_at(&added);
7912                if retry.is_some_and(|at| at > Instant::now() + Duration::from_secs(200)) {
7913                    break;
7914                }
7915                tokio::time::sleep(Duration::from_millis(20)).await;
7916            }
7917        })
7918        .await
7919        .expect("the added credential was tried and backed off");
7920        rt.stop_auto_refresh();
7921    }
7922
7923    #[tokio::test]
7924    async fn every_lease_reports_exactly_once() {
7925        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
7926        let stats = |s, f, c| AttemptStats {
7927            success: s,
7928            failure: f,
7929            cancelled: c,
7930        };
7931
7932        // Dropped without completing (client gone mid-execute): cancelled.
7933        drop(store.select(sel("claude")).unwrap());
7934        assert_eq!(store.stats(), stats(0, 0, 1));
7935        // Completed explicitly: the drop that follows does not report again.
7936        store.select(sel("claude")).unwrap().complete(Outcome::Success);
7937        assert_eq!(store.stats(), stats(1, 0, 1));
7938
7939        // A stream that errors and would then hang forever: the wrapper stops after Err.
7940        let err = ExecError::local(502, FailureScope::Transport, "boom");
7941        let inner = futures_util::stream::iter(vec![Ok(Bytes::from_static(b"a")), Err(err)])
7942            .chain(futures_util::stream::pending())
7943            .boxed();
7944        let mut s = Completing::new(inner, store.select(sel("claude")).unwrap());
7945        assert!(s.next().await.unwrap().is_ok());
7946        assert!(s.next().await.unwrap().is_err());
7947        assert!(s.next().await.is_none(), "fused after the first error");
7948        drop(s);
7949        assert_eq!(store.stats(), stats(1, 1, 1));
7950
7951        let mut done = Completing::new(
7952            futures_util::stream::empty().boxed(),
7953            store.select(sel("claude")).unwrap(),
7954        );
7955        assert!(done.next().await.is_none());
7956        drop(done);
7957        assert_eq!(store.stats(), stats(2, 1, 1));
7958
7959        // Dropped mid-stream: cancelled.
7960        let mut mid = Completing::new(
7961            futures_util::stream::iter(vec![Ok(Bytes::new())])
7962                .chain(futures_util::stream::pending())
7963                .boxed(),
7964            store.select(sel("claude")).unwrap(),
7965        );
7966        assert!(mid.next().await.is_some());
7967        drop(mid);
7968        assert_eq!(store.stats(), stats(2, 1, 2));
7969    }
7970
7971    /// A remote lease ends once, never through the local scheduler, and only after the
7972    /// upstream body it was streaming has been dropped.
7973    #[tokio::test]
7974    async fn remote_leases_end_after_the_upstream_body_closes() {
7975        use std::sync::atomic::AtomicBool;
7976        struct Probe(Arc<AtomicBool>);
7977        impl Drop for Probe {
7978            fn drop(&mut self) {
7979                self.0.store(true, Ordering::SeqCst);
7980            }
7981        }
7982        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
7983        let dropped = Arc::new(AtomicBool::new(false));
7984        let observed = Arc::new(Mutex::new(Vec::new()));
7985        let mut lease = store.select(sel("claude")).unwrap();
7986        let (seen, probe) = (observed.clone(), dropped.clone());
7987        lease.remote = Some(crate::remote::RemoteEnd {
7988            end: Some(Box::new(move || {
7989                seen.lock().unwrap().push(probe.load(Ordering::SeqCst));
7990                None
7991            })),
7992            releases: Default::default(),
7993            cancel: None,
7994            private: Default::default(),
7995        });
7996        let guard = Probe(dropped.clone());
7997        let err = ExecError::local(502, FailureScope::Transport, "boom");
7998        let inner = futures_util::stream::iter(vec![Ok(Bytes::from_static(b"a")), Err(err)])
7999            .chain(futures_util::stream::pending())
8000            .map(move |item| {
8001                let _ = &guard;
8002                item
8003            })
8004            .boxed();
8005        let mut stream = Completing::new(inner, lease);
8006        assert!(stream.next().await.unwrap().is_ok());
8007        assert!(observed.lock().unwrap().is_empty(), "still streaming");
8008        assert!(stream.next().await.unwrap().is_err());
8009        assert_eq!(
8010            *observed.lock().unwrap(),
8011            vec![true],
8012            "body dropped first, then one end"
8013        );
8014        drop(stream);
8015        assert_eq!(observed.lock().unwrap().len(), 1);
8016        assert_eq!(
8017            store.stats(),
8018            AttemptStats::default(),
8019            "the local scheduler saw nothing"
8020        );
8021    }
8022    /// Go binds the attempt's cancellation to the Home selection: a drain closes the
8023    /// upstream body and ends the lease while the client has stopped reading
8024    /// (backpressure), and the reader then sees the cancellation once.
8025    #[tokio::test]
8026    async fn a_drain_ends_a_remote_stream_the_client_stopped_reading() {
8027        use std::sync::atomic::AtomicBool;
8028        struct Probe(Arc<AtomicBool>);
8029        impl Drop for Probe {
8030            fn drop(&mut self) {
8031                self.0.store(true, Ordering::SeqCst);
8032            }
8033        }
8034        let store = CredentialStore::new(vec![cred("a.json", "claude", false)]);
8035        let dropped = Arc::new(AtomicBool::new(false));
8036        let observed = Arc::new(Mutex::new(Vec::new()));
8037        let mut lease = store.select(sel("claude")).unwrap();
8038        let (seen, probe) = (observed.clone(), dropped.clone());
8039        let (cancel_tx, cancel) = tokio::sync::watch::channel(false);
8040        lease.remote = Some(crate::remote::RemoteEnd {
8041            end: Some(Box::new(move || {
8042                seen.lock().unwrap().push(probe.load(Ordering::SeqCst));
8043                None
8044            })),
8045            releases: Default::default(),
8046            cancel: Some(cancel),
8047            private: Default::default(),
8048        });
8049        let guard = Probe(dropped.clone());
8050        let inner = futures_util::stream::iter(vec![Ok(Bytes::from_static(b"a"))])
8051            .chain(futures_util::stream::pending())
8052            .map(move |item| {
8053                let _ = &guard;
8054                item
8055            })
8056            .boxed();
8057        let mut stream = Completing::new(inner, lease);
8058        assert!(stream.next().await.unwrap().is_ok());
8059        // The client stops polling; the dispatcher drains.
8060        cancel_tx.send(true).unwrap();
8061        for _ in 0..100 {
8062            if !observed.lock().unwrap().is_empty() {
8063                break;
8064            }
8065            tokio::time::sleep(Duration::from_millis(10)).await;
8066        }
8067        assert_eq!(
8068            *observed.lock().unwrap(),
8069            vec![true],
8070            "body dropped first, then one end, without a poll"
8071        );
8072        let error = stream.next().await.unwrap().unwrap_err();
8073        assert_eq!(String::from_utf8_lossy(&error.body), "context canceled");
8074        assert!(stream.next().await.is_none());
8075        drop(stream);
8076        assert_eq!(observed.lock().unwrap().len(), 1);
8077    }
8078}