Skip to main content

cpa_server/
cooldown_store.rs

1//! `routing.cooldown.save-cooldown-status`: cooldowns persisted as one `.cds` file per
2//! credential under `auth-dir` (Go sdk/cliproxy/auth/cooldown_state.go). The files use
3//! Go's layout and field names, so state survives a switch between CLIProxyAPI and
4//! cliproxy-rs in either direction.
5
6use std::path::{Component, Path, PathBuf};
7use std::time::{Duration, SystemTime};
8
9use chrono::{DateTime, SecondsFormat, Utc};
10use serde::{Deserialize, Serialize};
11
12/// Go `CooldownStateRecord`. An empty `model` is the credential-level record.
13#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
14pub struct Record {
15    #[serde(default, skip_serializing_if = "String::is_empty")]
16    pub provider: String,
17    #[serde(default)]
18    pub auth_id: String,
19    #[serde(default, skip_serializing_if = "String::is_empty")]
20    pub model: String,
21    #[serde(default, skip_serializing_if = "String::is_empty")]
22    pub status: String,
23    #[serde(default, with = "go_time")]
24    pub next_retry_after: Option<SystemTime>,
25    #[serde(default, skip_serializing_if = "String::is_empty")]
26    pub reason: String,
27    #[serde(default)]
28    pub quota: Quota,
29    #[serde(default, skip_serializing_if = "Option::is_none")]
30    pub last_error: Option<LastError>,
31    #[serde(default, with = "go_time")]
32    pub updated_at: Option<SystemTime>,
33    /// The credential file (Go `AuthFile`, `json:"-"`): decides the `.cds` path.
34    #[serde(skip)]
35    pub auth_file: Option<PathBuf>,
36}
37
38/// Go `QuotaState` cooldown fields (`cooldownFieldsOf`).
39#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
40pub struct Quota {
41    #[serde(default)]
42    pub exceeded: bool,
43    #[serde(default, skip_serializing_if = "String::is_empty")]
44    pub reason: String,
45    #[serde(default, with = "go_time")]
46    pub next_recover_at: Option<SystemTime>,
47    #[serde(default, skip_serializing_if = "is_zero")]
48    pub backoff_level: u32,
49    #[serde(default, with = "go_time")]
50    pub observed_at: Option<SystemTime>,
51    /// cliproxy-rs only (`max-trusted-cooldown`): bounded windows spent, 0 for none.
52    /// Written on every quota record cliproxy-rs saves, so its presence marks a record
53    /// whose deadline restores as it is. Go's decoder ignores it; records without it
54    /// (Go's, or older) get the first bound.
55    #[serde(default, skip_serializing_if = "Option::is_none")]
56    pub trust_windows: Option<u32>,
57}
58
59/// Go `Error`.
60#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
61pub struct LastError {
62    #[serde(default, skip_serializing_if = "String::is_empty")]
63    pub code: String,
64    #[serde(default)]
65    pub message: String,
66    #[serde(default)]
67    pub retryable: bool,
68    #[serde(default, skip_serializing_if = "is_zero")]
69    pub http_status: u32,
70}
71
72fn is_zero(n: &u32) -> bool {
73    *n == 0
74}
75
76#[derive(Serialize, Deserialize)]
77struct Envelope {
78    #[serde(default)]
79    version: u32,
80    #[serde(default, skip_serializing_if = "String::is_empty")]
81    auth_id: String,
82    #[serde(default, skip_serializing_if = "String::is_empty")]
83    provider: String,
84    #[serde(default, with = "go_time")]
85    updated_at: Option<SystemTime>,
86    #[serde(default)]
87    records: Vec<Record>,
88}
89
90/// Go `time.Time` JSON: RFC 3339 with trimmed nanoseconds; the zero time is
91/// `0001-01-01T00:00:00Z` (`None` here).
92mod go_time {
93    use super::*;
94
95    const ZERO: &str = "0001-01-01T00:00:00Z";
96
97    pub fn serialize<S: serde::Serializer>(t: &Option<SystemTime>, s: S) -> Result<S::Ok, S::Error> {
98        match t {
99            Some(t) => s.serialize_str(&rfc3339_nano(*t)),
100            None => s.serialize_str(ZERO),
101        }
102    }
103
104    /// Go `time.RFC3339Nano`: nanoseconds with trailing zeros trimmed, no dot when zero.
105    pub fn rfc3339_nano(t: SystemTime) -> String {
106        let text = DateTime::<Utc>::from(t).to_rfc3339_opts(SecondsFormat::Nanos, true);
107        let (head, _) = text.split_once('Z').unwrap_or((&text, ""));
108        let head = head.trim_end_matches('0').trim_end_matches('.');
109        format!("{head}Z")
110    }
111
112    pub fn deserialize<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Option<SystemTime>, D::Error> {
113        let raw = Option::<String>::deserialize(d)?.unwrap_or_default();
114        if raw.is_empty() {
115            return Ok(None);
116        }
117        let parsed = DateTime::parse_from_rfc3339(&raw).map_err(serde::de::Error::custom)?;
118        // Go's zero time and anything before the epoch mean "no time".
119        Ok(u64::try_from(parsed.timestamp())
120            .ok()
121            .map(|secs| SystemTime::UNIX_EPOCH + Duration::new(secs, parsed.timestamp_subsec_nanos())))
122    }
123}
124
125/// Go `stateRelativePath`: the credential file's path below `dir` with a `.cds`
126/// extension, else a sanitized file name.
127fn relative_path(dir: &Path, record: &Record) -> Option<PathBuf> {
128    match &record.auth_file {
129        Some(file) if file.is_absolute() => match file.strip_prefix(dir) {
130            Ok(rel) if !rel.as_os_str().is_empty() => cds_path_for_rel(rel),
131            _ => file
132                .file_name()
133                .and_then(|n| sanitize(&n.to_string_lossy()))
134                .map(PathBuf::from),
135        },
136        Some(file) => cds_path_for_rel(file),
137        None => sanitize(record.auth_id.trim()).map(PathBuf::from),
138    }
139}
140
141fn cds_path_for_rel(rel: &Path) -> Option<PathBuf> {
142    if rel
143        .components()
144        .any(|c| !matches!(c, Component::Normal(_) | Component::CurDir))
145    {
146        return None;
147    }
148    let base = sanitize(&rel.file_name()?.to_string_lossy())?;
149    Some(match rel.parent().filter(|p| !p.as_os_str().is_empty()) {
150        Some(parent) => parent.join(base),
151        None => PathBuf::from(base),
152    })
153}
154
155/// Go `sanitizeCooldownFileName`: extension dropped, runs outside `[A-Za-z0-9._-]`
156/// become `_`, edges trimmed of `._-`, then `.cds`.
157fn sanitize(name: &str) -> Option<String> {
158    let name = name.trim();
159    let stem = name.rfind('.').map_or(name, |dot| &name[..dot]);
160    let mut out = String::with_capacity(stem.len());
161    let mut in_run = false;
162    for c in stem.chars() {
163        if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') {
164            out.push(c);
165            in_run = false;
166        } else if !in_run {
167            out.push('_');
168            in_run = true;
169        }
170    }
171    let out = out.trim_matches(|c| matches!(c, '.' | '_' | '-'));
172    (!out.is_empty()).then(|| format!("{out}.cds"))
173}
174
175/// Go `json.MarshalIndent(v, "", "  ")` plus a newline: serde's pretty layout matches;
176/// Go additionally escapes `<`, `>`, `&`, U+2028 and U+2029, which only occur in strings.
177fn marshal_indent(envelope: &Envelope) -> Vec<u8> {
178    let text = serde_json::to_string_pretty(envelope).expect("cooldown records serialize");
179    let mut out = String::with_capacity(text.len() + 1);
180    for c in text.chars() {
181        match c {
182            '<' => out.push_str("\\u003c"),
183            '>' => out.push_str("\\u003e"),
184            '&' => out.push_str("\\u0026"),
185            '\u{2028}' => out.push_str("\\u2028"),
186            '\u{2029}' => out.push_str("\\u2029"),
187            c => out.push(c),
188        }
189    }
190    out.push('\n');
191    out.into_bytes()
192}
193
194fn cds_files(dir: &Path) -> Vec<PathBuf> {
195    let mut out = Vec::new();
196    let mut stack = vec![dir.to_path_buf()];
197    while let Some(dir) = stack.pop() {
198        let Ok(entries) = std::fs::read_dir(&dir) else { continue };
199        for entry in entries.flatten() {
200            let path = entry.path();
201            match entry.file_type() {
202                Ok(t) if t.is_dir() => stack.push(path),
203                Ok(_) if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("cds")) => out.push(path),
204                _ => {}
205            }
206        }
207    }
208    out.sort();
209    out
210}
211
212/// A cooldown store other than `.cds` files (Go `CooldownStateStore` from a token store
213/// that is a `CooldownStateStoreProvider`: PGSTORE's `cooldown_store` table). Used
214/// while `save-cooldown-status` is on, in place of the files.
215pub trait Backend: Send + Sync {
216    /// Go `Load`.
217    fn load(&self) -> Result<Vec<Record>, String>;
218    /// Go `Save`: the complete live set; records no longer present are cleared.
219    fn save(&self, records: Vec<Record>, now: SystemTime) -> Result<(), String>;
220}
221
222/// Go `FileCooldownStateStore.Load`: every `.cds` file below `dir`. A missing directory
223/// is empty state; an unreadable file is an error.
224pub fn load(dir: &Path) -> Result<Vec<Record>, String> {
225    let mut records = Vec::new();
226    for path in cds_files(dir) {
227        let data = match std::fs::read(&path) {
228            Ok(data) => data,
229            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
230            Err(e) => return Err(format!("read cooldown state {}: {e}", path.display())),
231        };
232        if data.trim_ascii().is_empty() {
233            continue;
234        }
235        let envelope: Envelope =
236            serde_json::from_slice(&data).map_err(|e| format!("parse cooldown state {}: {e}", path.display()))?;
237        records.extend(envelope.records);
238    }
239    Ok(records)
240}
241
242/// Go `FileCooldownStateStore.Save`: one atomically replaced file per credential, records
243/// sorted by model; `.cds` files no longer backed by a record are removed.
244pub fn save(dir: &Path, records: Vec<Record>, now: SystemTime) -> Result<(), String> {
245    let mut groups: std::collections::BTreeMap<PathBuf, Vec<Record>> = Default::default();
246    for record in records {
247        if record.auth_id.trim().is_empty() {
248            continue;
249        }
250        let rel = relative_path(dir, &record).ok_or("cooldown state path: missing auth identity")?;
251        groups.entry(dir.join(rel)).or_default().push(record);
252    }
253    for (path, mut records) in groups.clone() {
254        records.sort_by(|a, b| a.model.cmp(&b.model));
255        let envelope = Envelope {
256            version: 1,
257            auth_id: records[0].auth_id.clone(),
258            provider: records[0].provider.clone(),
259            updated_at: Some(now),
260            records,
261        };
262        create_private_dir(path.parent().unwrap_or(dir))?;
263        // Go `os.CreateTemp` + rename: an exclusively created 0600 temporary file.
264        crate::runtime::write_bytes_atomic(&path, &marshal_indent(&envelope))
265            .map_err(|e| format!("replace cooldown state file: {e}"))?;
266    }
267    for path in cds_files(dir) {
268        if !groups.contains_key(&path)
269            && let Err(e) = std::fs::remove_file(&path)
270            && e.kind() != std::io::ErrorKind::NotFound
271        {
272            return Err(format!("remove stale cooldown state {}: {e}", path.display()));
273        }
274    }
275    Ok(())
276}
277
278fn create_private_dir(dir: &Path) -> Result<(), String> {
279    let mut builder = std::fs::DirBuilder::new();
280    builder.recursive(true);
281    #[cfg(unix)]
282    std::os::unix::fs::DirBuilderExt::mode(&mut builder, 0o700);
283    builder
284        .create(dir)
285        .map_err(|e| format!("create cooldown state directory: {e}"))
286}
287
288#[cfg(test)]
289mod tests {
290    use super::*;
291
292    #[test]
293    fn file_names_follow_go_sanitizing() {
294        let dir = Path::new("/auth");
295        let rec = |file: Option<&str>, id: &str| Record {
296            auth_id: id.into(),
297            auth_file: file.map(PathBuf::from),
298            ..Default::default()
299        };
300        let path = |r: &Record| relative_path(dir, r).map(|p| p.to_string_lossy().into_owned());
301        assert_eq!(
302            path(&rec(Some("/auth/claude-a.json"), "x")).as_deref(),
303            Some("claude-a.cds")
304        );
305        assert_eq!(
306            path(&rec(Some("/auth/sub/team b.json"), "x")).as_deref(),
307            Some("sub/team_b.cds")
308        );
309        assert_eq!(path(&rec(Some("/elsewhere/k!!.json"), "x")).as_deref(), Some("k.cds"));
310        assert_eq!(path(&rec(Some("rel/a b.json"), "x")).as_deref(), Some("rel/a_b.cds"));
311        assert_eq!(path(&rec(Some("../escape.json"), "x")), None);
312        assert_eq!(path(&rec(None, "cfg:key/0")).as_deref(), Some("cfg_key_0.cds"));
313        assert_eq!(path(&rec(None, "...")), None);
314    }
315
316    /// Go `FileCooldownStateStore` output for the same `MarkResult` sequence
317    /// (tests/reference/server/main.go `cooldown_files`), compared with timestamps
318    /// masked. Go also writes an aggregate credential-level record derived from its model
319    /// states (`updateAggregatedAvailability`); Go recomputes it on restore, so Rust
320    /// writes only the credential-wide quota record and the comparison skips the rest.
321    #[test]
322    fn files_match_go_store() {
323        use crate::scheduler::{Policy, Scheduler};
324        use cpa_core::credential::{Credential, Source};
325        use cpa_core::exec::{ExecError, FailureScope};
326        use std::time::Instant;
327
328        let fixture: serde_json::Value =
329            serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
330        let go = fixture["cooldown_files"].as_object().unwrap();
331        let dir = std::env::temp_dir().join(format!(
332            "cds-{}-{}",
333            std::process::id(),
334            SystemTime::now()
335                .duration_since(SystemTime::UNIX_EPOCH)
336                .unwrap()
337                .as_nanos()
338        ));
339        let file = |id: &str, disabled: bool| {
340            let metadata = serde_json::json!({"type": "claude"}).as_object().unwrap().clone();
341            let mut c = Credential::from_file(&dir, &dir.join(id), metadata).unwrap();
342            c.disabled = disabled;
343            c
344        };
345        let mut config = file("x.json", false);
346        config.id = "cfg:key/0".into();
347        config.source = Source::Config {
348            section: "claude-api-key".into(),
349            index: 0,
350        };
351        let creds = [
352            file("claude-a.json", false),
353            file("sub/team b.json", false),
354            config,
355            file("off.json", true),
356        ];
357        let policy = Policy::default();
358        let mut s = Scheduler::default();
359        let now = Instant::now();
360        let fail =
361            |s: &mut Scheduler, i: usize, model: &str, status: u16, message: &str, hint: u64, credential: bool| {
362                let scope = match (credential, status) {
363                    (true, _) => FailureScope::Credential,
364                    (_, 429) => FailureScope::Model,
365                    (_, 401..=403 | 500..) => FailureScope::Credential,
366                    _ => FailureScope::Request,
367                };
368                let mut e = ExecError::local(status, scope, message);
369                e.headers.insert("content-type", "text/plain".parse().unwrap());
370                e.retry_after = (hint > 0).then(|| Duration::from_secs(hint));
371                s.record(&creds[i], model, &crate::runtime::Outcome::Failure(e), &policy, now);
372            };
373        fail(&mut s, 0, "m1", 429, "rate limited", 30, false);
374        fail(&mut s, 0, "m2", 401, "unauthorized", 0, false);
375        fail(&mut s, 1, "m2", 500, "boom", 0, false);
376        fail(&mut s, 1, "m1", 429, "credential quota", 60, true);
377        fail(&mut s, 2, "m3", 403, "challenge-platform", 0, false);
378        fail(&mut s, 3, "m1", 500, "boom", 0, false);
379        let wall = SystemTime::now();
380        let records: Vec<Record> = creds
381            .iter()
382            .filter(|c| !c.disabled)
383            .flat_map(|c| s.records(c, now, wall))
384            .collect();
385        save(&dir, records, wall).unwrap();
386
387        let mask = |v: &mut serde_json::Value| {
388            fn walk(v: &mut serde_json::Value) {
389                match v {
390                    serde_json::Value::Object(map) => {
391                        for (k, v) in map.iter_mut() {
392                            if matches!(k.as_str(), "updated_at" | "next_retry_after" | "next_recover_at")
393                                && v.as_str() != Some("0001-01-01T00:00:00Z")
394                            {
395                                *v = "<time>".into();
396                            } else {
397                                walk(v);
398                            }
399                        }
400                    }
401                    serde_json::Value::Array(items) => items.iter_mut().for_each(walk),
402                    _ => {}
403                }
404            }
405            walk(v);
406        };
407        let mut rust_files: Vec<String> = cds_files(&dir)
408            .iter()
409            .map(|p| p.strip_prefix(&dir).unwrap().to_string_lossy().into_owned())
410            .collect();
411        rust_files.sort();
412        let mut go_files: Vec<String> = go.keys().cloned().collect();
413        go_files.sort();
414        assert_eq!(rust_files, go_files, "same files, none for the disabled credential");
415        for (name, text) in go {
416            let mut expected: serde_json::Value = serde_json::from_str(text.as_str().unwrap()).unwrap();
417            expected["records"]
418                .as_array_mut()
419                .unwrap()
420                .retain(|r| r["model"].as_str().is_some() || r["quota"]["reason"] == "credential_quota");
421            let raw = std::fs::read_to_string(dir.join(name)).unwrap();
422            assert!(raw.ends_with("}\n"), "{name}: MarshalIndent plus newline");
423            let mut actual: serde_json::Value = serde_json::from_str(&raw).unwrap();
424            mask(&mut actual);
425            // The one deliberate difference (docs/DIFFERENCES-FROM-GO.md): every quota
426            // record cliproxy-rs writes carries `trust_windows`, which Go's decoder skips.
427            for record in actual["records"].as_array_mut().unwrap() {
428                if record["quota"]["exceeded"] == true {
429                    let removed = record["quota"].as_object_mut().unwrap().remove("trust_windows");
430                    assert!(
431                        removed.is_some_and(|v| v.is_u64()),
432                        "{name}: trust_windows on quota records"
433                    );
434                }
435            }
436            assert_eq!(actual, expected, "{name}");
437        }
438        let _ = std::fs::remove_dir_all(&dir);
439    }
440
441    #[test]
442    fn times_round_trip_go_layout() {
443        let t = SystemTime::UNIX_EPOCH + Duration::new(1_790_000_000, 120_000_000);
444        let quota = Quota {
445            next_recover_at: Some(t),
446            ..Default::default()
447        };
448        let json = serde_json::to_string(&quota).unwrap();
449        assert_eq!(
450            json,
451            r#"{"exceeded":false,"next_recover_at":"2026-09-21T14:13:20.12Z","observed_at":"0001-01-01T00:00:00Z"}"#
452        );
453        let back: Quota = serde_json::from_str(&json).unwrap();
454        assert_eq!(back, quota);
455        // Go writes local time with an offset; any offset parses.
456        let offset: Quota = serde_json::from_str(r#"{"next_recover_at":"2026-09-21T16:13:20.12+02:00"}"#).unwrap();
457        assert_eq!(offset.next_recover_at, Some(t));
458        let whole = SystemTime::UNIX_EPOCH + Duration::from_secs(1_790_000_000);
459        assert_eq!(go_time::rfc3339_nano(whole), "2026-09-21T14:13:20Z");
460    }
461}