Skip to main content

cpa_server/
remote.rs

1//! Credentials dispatched by a control plane instead of the local scheduler (Go Home
2//! mode: `pickHomeDispatchSelection` / `executeHomeOnce`). While a dispatcher is
3//! installed every request routes to provider `home`, each pick asks the dispatcher for
4//! one credential, and the credential's lease ends with a release the dispatcher
5//! reports back. The dispatcher owns cooldowns and retry limits; nothing is recorded in
6//! the local scheduler.
7
8use cpa_core::gostr::GoStr;
9use std::sync::{Arc, Mutex, PoisonError};
10use std::time::Duration;
11
12use axum::response::IntoResponse;
13use cpa_core::credential::Credential;
14use cpa_core::exec::ExecError;
15use futures_util::future::BoxFuture;
16
17/// Attribute carrying the upstream model the dispatcher chose (Go
18/// `homeUpstreamModelAttributeKey`).
19pub const UPSTREAM_MODEL: &str = "home_upstream_model";
20/// Set to `true` when responses report the route model instead (Go
21/// `homeForceMappingAttributeKey`).
22pub const FORCE_MAPPING: &str = "home_force_mapping";
23/// The alias Home mapped from (Go `homeOriginalAliasAttributeKey`).
24pub const ORIGINAL_ALIAS: &str = "home_original_alias";
25/// The dispatched auth's own `provider` field (Go `Auth.Provider`; the selection
26/// provider is its lower-case form). The credential's provider is the executor key.
27pub const PROVIDER: &str = cpa_core::config::credentials::HOME_PROVIDER;
28/// Home's definition of the dispatched model, as sent (Go `homeDispatchModelInfo`):
29/// [`crate::capabilities::bind_home`] makes it the attempt's resolved model.
30pub const MODEL_INFO: &str = "home_model_info";
31/// The session the dispatcher sent for this pick (Go
32/// `HomeDispatchSelection.CanonicalSessionID`); usage records report it.
33pub const SESSION: &str = "home_session_id";
34/// Its parent (Go `HomeDispatchSelection.ParentSessionID`).
35pub const PARENT_SESSION: &str = "home_parent_session_id";
36/// What a Home pick carries that Go keeps outside `Auth.Attributes`: the auth's
37/// runtime state, and the attribute names the dispatcher set itself (plugins never see
38/// those; attributes Home sent stay).
39#[derive(Debug, Default, Clone)]
40pub struct Private {
41    pub availability: Arc<Availability>,
42    pub attributes: Vec<String>,
43}
44
45/// The runtime state Go keeps on a dispatched `coreauth.Auth` (`status`, `unavailable`,
46/// `next_retry_after`, `model_states`) and the Responses WebSocket pin check reads,
47/// decoded as Go's typed JSON decoding does.
48#[derive(Debug, Default, PartialEq)]
49pub struct Availability {
50    state: State,
51    /// A `None` entry is Go's nil state (`null`): present, but never consulted.
52    model_states: std::collections::BTreeMap<String, Option<State>>,
53}
54
55#[derive(Debug, Default, PartialEq)]
56struct State {
57    status: String,
58    unavailable: bool,
59    /// Unix seconds and nanoseconds; `None` is Go's zero time.
60    retry: Option<(i64, u32)>,
61}
62
63/// An object's members as written. ponytail: a repeated key keeps its last value, where
64/// Go applies each in turn and skips a `null`; Go also matches names case-insensitively.
65type RawObject<'a> = std::collections::HashMap<String, &'a serde_json::value::RawValue>;
66
67impl State {
68    fn decode(object: &RawObject<'_>) -> Result<Self, ()> {
69        use serde_json::Value;
70        let field = |key: &str| object.get(key).map(|raw| raw.get()).filter(|raw| *raw != "null");
71        let value = |raw: &str| serde_json::from_str::<Value>(raw).map_err(|_| ());
72        Ok(Self {
73            status: match field("status").map(value).transpose()? {
74                None => String::new(),
75                Some(Value::String(s)) => s,
76                Some(_) => return Err(()),
77            },
78            unavailable: match field("unavailable").map(value).transpose()? {
79                None => false,
80                Some(v) => v.as_bool().ok_or(())?,
81            },
82            // Go's `Time.UnmarshalJSON` parses the quoted bytes as written, escapes and
83            // all: a backslash never parses.
84            retry: match field("next_retry_after") {
85                None => None,
86                Some(raw) => {
87                    let quoted = raw.strip_prefix('"').and_then(|r| r.strip_suffix('"')).ok_or(())?;
88                    go_time(quoted).ok_or(())?
89                }
90            },
91        })
92    }
93
94    fn disabled(&self) -> bool {
95        self.status == "disabled"
96    }
97
98    /// `Unavailable && !NextRetryAfter.IsZero() && NextRetryAfter.After(now)`.
99    fn cooling(&self, now: (i64, u32)) -> bool {
100        self.unavailable && self.retry.is_some_and(|at| at > now)
101    }
102}
103
104impl Availability {
105    /// Go's `json.Unmarshal` of an auth object (as written) into `coreauth.Auth`, for
106    /// these fields: `null` or absent keeps the default, any other mistyped value or
107    /// unparsable time fails the whole reply (Home's `invalid_auth`, with this message).
108    pub fn decode(auth: &str) -> Result<Self, &'static str> {
109        Self::decode_fields(auth).map_err(|()| "home returned invalid auth payload")
110    }
111
112    fn decode_fields(auth: &str) -> Result<Self, ()> {
113        let object: RawObject<'_> = serde_json::from_str(auth).map_err(|_| ())?;
114        let state = State::decode(&object)?;
115        let model_states = match object.get("model_states").map(|raw| raw.get()) {
116            None | Some("null") => Default::default(),
117            Some(raw) => serde_json::from_str::<RawObject<'_>>(raw)
118                .map_err(|_| ())?
119                .into_iter()
120                .map(|(model, state)| match state.get() {
121                    "null" => Ok((model, None)),
122                    raw => {
123                        let state: RawObject<'_> = serde_json::from_str(raw).map_err(|_| ())?;
124                        Ok((model, Some(State::decode(&state)?)))
125                    }
126                })
127                .collect::<Result<_, ()>>()?,
128        };
129        Ok(Self { state, model_states })
130    }
131}
132
133/// Go's `time.Time` JSON decoding (`parseStrictRFC3339`, whose strict checks are off,
134/// so `time.Parse(time.RFC3339, ...)`): a one- or two-digit hour, a fraction after `.`
135/// or `,` (nanoseconds, the rest ignored), `Z` or a `±hh:mm` offset up to 24:60.
136/// `Some(None)` is Go's zero time (`IsZero`), `None` a value Go rejects.
137fn go_time(raw: &str) -> Option<Option<(i64, u32)>> {
138    let b = raw.as_bytes();
139    let digits = |from: usize, to: usize| -> Option<u32> {
140        let part = b.get(from..to)?;
141        part.iter()
142            .all(u8::is_ascii_digit)
143            .then(|| part.iter().fold(0, |n, d| n * 10 + u32::from(d - b'0')))
144    };
145    let at = |i: usize, c: u8| b.get(i) == Some(&c);
146    // Go's `stdLongYear` wants a digit first, so no sign.
147    let year = digits(0, 4)? as i32;
148    let (month, day) = (digits(5, 7)?, digits(8, 10)?);
149    if !(at(4, b'-') && at(7, b'-') && at(10, b'T')) {
150        return None;
151    }
152    let mut i = if b.get(12).is_some_and(u8::is_ascii_digit) {
153        13
154    } else {
155        12
156    };
157    let hour = digits(11, i)?;
158    if !(at(i, b':') && at(i + 3, b':')) {
159        return None;
160    }
161    let (minute, second) = (digits(i + 1, i + 3)?, digits(i + 4, i + 6)?);
162    i += 6;
163    let mut nanos = 0;
164    if matches!(b.get(i), Some(b'.' | b',')) && b.get(i + 1).is_some_and(u8::is_ascii_digit) {
165        let start = i + 1;
166        i = start;
167        while b.get(i).is_some_and(u8::is_ascii_digit) {
168            i += 1;
169        }
170        let fraction = &b[start..i];
171        for k in 0..9 {
172            nanos = nanos * 10 + fraction.get(k).map_or(0, |d| u32::from(d - b'0'));
173        }
174    }
175    let offset = match b.get(i)? {
176        b'Z' => {
177            i += 1;
178            0
179        }
180        sign @ (b'+' | b'-') => {
181            let (hh, mm) = (digits(i + 1, i + 3)?, digits(i + 4, i + 6)?);
182            if !at(i + 3, b':') || hh > 24 || mm > 60 {
183                return None;
184            }
185            i += 6;
186            let offset = i64::from((hh * 60 + mm) * 60);
187            if *sign == b'-' { -offset } else { offset }
188        }
189        _ => return None,
190    };
191    if i != b.len() || !(1..=12).contains(&month) || hour > 23 || minute > 59 || second > 59 {
192        return None;
193    }
194    let local = chrono::NaiveDate::from_ymd_opt(year, month, day)?.and_hms_opt(hour, minute, second)?;
195    let seconds = local.and_utc().timestamp() - offset;
196    // Go's zero time: January 1, year 1, 00:00:00 UTC.
197    const ZERO: i64 = -62_135_596_800;
198    Some((seconds != ZERO || nanos != 0).then_some((seconds, nanos)))
199}
200
201/// Go `responsesWebsocketAuthAvailableForModel` on a dispatched auth's snapshot:
202/// enabled (`disabled` and `status`), then the model's state when the snapshot has one
203/// (its status and cooldown decide), else the auth-wide cooldown. A cooldown counts only
204/// while its retry time is set and in the future.
205pub fn available_for_model(
206    credential: &Credential,
207    snapshot: &Availability,
208    model: &str,
209    now: std::time::SystemTime,
210) -> bool {
211    if credential.disabled {
212        return false;
213    }
214    let now = match now.duration_since(std::time::UNIX_EPOCH) {
215        Ok(since) => (since.as_secs() as i64, since.subsec_nanos()),
216        Err(_) => (0, 0),
217    };
218    if snapshot.state.disabled() {
219        return false;
220    }
221    if !model.is_empty() && !snapshot.model_states.is_empty() {
222        let base = cpa_common::thinking::parse_suffix(model).model_name;
223        let base = base.trim();
224        let state = match snapshot.model_states.get(model) {
225            Some(Some(state)) => Some(state),
226            _ if !base.is_empty() && base != model => snapshot.model_states.get(base).and_then(Option::as_ref),
227            _ => None,
228        };
229        if let Some(state) = state {
230            return !state.disabled() && !state.cooling(now);
231        }
232    }
233    !snapshot.state.cooling(now)
234}
235
236/// Go `HomeDispatchSelection.Provider`: the dispatched auth's provider, lower-cased.
237pub fn selection_provider(credential: &Credential) -> String {
238    credential
239        .attributes
240        .get(PROVIDER)
241        .map_or(credential.provider.as_str(), String::as_str)
242        .trim()
243        .go_lower()
244}
245
246/// One credential request (Go's RPOP request fields).
247#[derive(Debug, Clone, Default, PartialEq, Eq)]
248pub struct RemoteRequest {
249    /// The route model as the client sent it (Home mode resolves nothing locally).
250    pub model: String,
251    pub session_id: String,
252    pub parent_session_id: String,
253    /// Downstream request headers; the dispatcher authenticates the client with them.
254    pub headers: Vec<(String, String)>,
255    /// 1 for the first pick of a retry round, then 2, 3, ... (Go `homeAuthCount`).
256    pub count: i64,
257    pub retry_round: i64,
258    /// Credentials already tried this round.
259    pub excluded: Vec<String>,
260    pub pinned: String,
261    pub request_id: String,
262    /// `http`, `stream` or `websocket`.
263    pub kind: &'static str,
264    /// Go `credentialPolicyFromContext`: narrows Home's choice (`codex_alpha_search_v1`),
265    /// empty for inference.
266    pub credential_policy: String,
267}
268
269/// Completes once the dispatcher acknowledged a release (bounded by the dispatcher).
270pub type ReleaseWait = BoxFuture<'static, Result<(), String>>;
271
272/// Ends a dispatched credential's lease exactly once; returns the acknowledgement to
273/// wait for before the next pick of the same request, when there is one.
274pub type EndLease = Box<dyn FnOnce() -> Option<ReleaseWait> + Send>;
275
276/// Set to `true` when the dispatcher cancels the execution (Go: a draining registry
277/// closes the selection's bound resources, cancelling its attempt contexts).
278pub type CancelSignal = tokio::sync::watch::Receiver<bool>;
279
280/// Resolves once `signal` is set; never when its sender is gone without setting it.
281pub(crate) fn cancelled(mut signal: CancelSignal) -> BoxFuture<'static, ()> {
282    Box::pin(async move {
283        if signal.wait_for(|cancelled| *cancelled).await.is_err() {
284            std::future::pending::<()>().await;
285        }
286    })
287}
288
289/// The error an execution cancelled by the dispatcher ends with (Go `context.Canceled`).
290pub(crate) fn cancelled_error() -> ExecError {
291    ExecError::local(502, cpa_core::exec::FailureScope::Transport, "context canceled")
292}
293
294/// Go `clienterror.StatusClientClosedRequest`: what `HTTPStatusFromErrorOr` makes of a
295/// cancelled context, so of a request Home's drain stopped.
296pub(crate) const CLIENT_CLOSED: u16 = 499;
297
298/// The error Go's `http.Client` returns for a request its cancelled context stopped
299/// (`*url.Error`): `Post "<url>": context canceled`.
300pub(crate) fn cancelled_request(method: &str, url: &str) -> String {
301    let mut op = method.to_ascii_lowercase();
302    if let Some(first) = op.get_mut(..1) {
303        first.make_ascii_uppercase();
304    }
305    format!("{op} {}: context canceled", cpa_common::gostr::quote(url))
306}
307
308/// A dispatched credential and the end of its lease.
309pub struct RemoteGrant {
310    pub credential: Credential,
311    pub end: EndLease,
312    /// Cancels the execution running on this credential.
313    pub cancel: Option<CancelSignal>,
314    /// The request-retry limit Home set for this request (Go `selection.requestRetry`).
315    pub request_retry: Option<i64>,
316    /// The client key Home authenticated (Go `dispatch.UserAPIKey`), trimmed; empty
317    /// when Home sent none. Private request context, like `Caller::principal`.
318    pub user_api_key: String,
319    pub private: Private,
320}
321
322/// How a failed pick takes part in retries (Go's typed Home errors).
323#[derive(Debug, Clone, PartialEq, Eq)]
324pub enum RemoteErrorKind {
325    Plain,
326    /// Go `homeDispatchRetryAfterError` (`model_cooldown`).
327    Cooldown {
328        retry_after: Option<Duration>,
329        request_retry: Option<i64>,
330    },
331    /// Go `HomeConcurrencyBusyError`: never retried; `header` is its safe `Retry-After`.
332    Busy {
333        header: Option<u64>,
334    },
335}
336
337/// A failed pick: the client-facing error (status and Go's `code: message`), its code
338/// and its retry behaviour.
339#[derive(Debug, Clone)]
340pub struct RemoteError {
341    pub error: ExecError,
342    /// Boxed to keep the error small (it travels in `Result`s).
343    pub code: Box<str>,
344    pub kind: RemoteErrorKind,
345}
346
347impl RemoteError {
348    pub fn plain(error: ExecError, code: &str) -> Self {
349        Self {
350            error,
351            code: code.into(),
352            kind: RemoteErrorKind::Plain,
353        }
354    }
355}
356
357/// Why the control plane's model catalog is not available.
358#[derive(Debug, Clone, PartialEq, Eq)]
359pub enum ModelsError {
360    /// No live control-plane client (Go `home.Current() == nil`).
361    Unavailable,
362    /// The request failed; the text is Go's error.
363    Failed(String),
364}
365
366pub trait RemoteDispatch: Send + Sync + 'static {
367    /// Go `HeartbeatOK`: requests are refused while the control plane is unreachable.
368    fn available(&self) -> bool;
369    /// One pick.
370    fn dispatch(&self, request: RemoteRequest) -> BoxFuture<'_, Result<RemoteGrant, RemoteError>>;
371    /// Go `GetModels`: the catalog for the client these request headers and query
372    /// parameters authenticate (raw Home payload).
373    fn models(
374        &self,
375        headers: Vec<(String, String)>,
376        query: Vec<(String, String)>,
377    ) -> BoxFuture<'_, Result<Vec<u8>, ModelsError>>;
378
379    /// Go RPushRequestLog. Unavailable Home lifetimes discard the log, never
380    /// fall back to writing it locally. Test/internal dispatchers may ignore it.
381    fn request_log(&self, payload: Vec<u8>) -> BoxFuture<'_, Result<(), String>> {
382        let _ = payload;
383        Box::pin(async { Ok(()) })
384    }
385}
386
387/// Releases ended during one request, awaited before its next pick (Go
388/// `endHomeSelectionBeforeRedispatch`).
389#[derive(Clone, Default)]
390pub struct PendingReleases(Arc<Mutex<Vec<ReleaseWait>>>);
391
392impl PendingReleases {
393    pub(crate) fn push(&self, wait: ReleaseWait) {
394        self.0.lock().unwrap_or_else(PoisonError::into_inner).push(wait);
395    }
396
397    /// Waits for every release ended so far; the first failure is returned.
398    pub(crate) async fn settle(&self) -> Result<(), String> {
399        let waits = std::mem::take(&mut *self.0.lock().unwrap_or_else(PoisonError::into_inner));
400        for wait in waits {
401            wait.await?;
402        }
403        Ok(())
404    }
405}
406
407/// A Home selection a handler could not make, as Go's handlers write it:
408/// `HTTPStatusFromErrorOr(err, 503)`, `err.Error()` and the safe `Retry-After` seconds.
409#[derive(Debug, Clone, PartialEq, Eq)]
410pub(crate) struct Refusal {
411    pub status: u16,
412    pub text: String,
413    pub retry_after: Option<u64>,
414}
415
416impl Refusal {
417    fn not_found(message: &str) -> Self {
418        Self {
419            status: 503,
420            text: format!("auth_not_found: {message}"),
421            retry_after: None,
422        }
423    }
424}
425
426/// Go `SelectHomeAuthByKind` and `SelectHomeAuthWithCredentialPolicy`: one Home pick for
427/// a handler outside the dispatch loop. A pick `accept` refuses is released, and the
428/// release acknowledged, before the next pick, which excludes it; refusing the same
429/// credential twice ends the selection. Also returns the client key Home authenticated
430/// (Go `setHomeUserAPIKeyOnGinContext`: the last non-empty one of these picks).
431pub(crate) async fn select(
432    rt: &crate::Runtime,
433    remote: &dyn RemoteDispatch,
434    selection: crate::runtime::Selection,
435    request: RemoteRequest,
436    accept: impl Fn(&Credential) -> bool,
437) -> Result<(crate::runtime::Lease, String), Refusal> {
438    let releases = PendingReleases::default();
439    let mut tried: Vec<String> = Vec::new();
440    let mut home_key = String::new();
441    let mut count = request.count.max(1);
442    loop {
443        let pick = RemoteRequest {
444            count,
445            excluded: tried.clone(),
446            ..request.clone()
447        };
448        let (lease, _, user_key) = rt
449            .acquire_remote(remote, selection.clone(), pick, &releases)
450            .await
451            .map_err(|failed| Refusal {
452                status: failed.error.status,
453                text: String::from_utf8_lossy(&failed.error.body).into_owned(),
454                retry_after: match failed.kind {
455                    RemoteErrorKind::Cooldown { retry_after, .. } => {
456                        retry_after.map(|d| d.as_secs() + u64::from(d.subsec_nanos() > 0))
457                    }
458                    RemoteErrorKind::Busy { header } => header,
459                    RemoteErrorKind::Plain => None,
460                },
461            })?;
462        if !user_key.trim().is_empty() {
463            home_key = user_key.trim().to_owned();
464        }
465        let credential = lease.credential.clone();
466        if accept(&credential) {
467            return Ok((lease, home_key));
468        }
469        // Go `endHomeSelectionBeforeRedispatch`.
470        drop(lease);
471        if let Err(message) = releases.settle().await {
472            return Err(Refusal {
473                status: 503,
474                text: format!("home_unavailable: Home did not acknowledge credential release: {message}"),
475                retry_after: None,
476            });
477        }
478        let id = credential.id.trim().to_owned();
479        if id.is_empty() {
480            return Err(Refusal::not_found("selected auth has no ID"));
481        }
482        if tried.contains(&id) {
483            return Err(Refusal::not_found("selector repeatedly returned an ineligible auth"));
484        }
485        tried.push(id);
486        count += 1;
487    }
488}
489
490/// The lease side of a dispatched credential: ends it on the first report.
491pub(crate) struct RemoteEnd {
492    pub end: Option<EndLease>,
493    pub releases: PendingReleases,
494    pub cancel: Option<CancelSignal>,
495    pub private: Private,
496}
497
498impl RemoteEnd {
499    pub(crate) fn finish(&mut self) {
500        if let Some(end) = self.end.take()
501            && let Some(wait) = end()
502        {
503            self.releases.push(wait);
504        }
505    }
506}
507
508tokio::task_local! {
509    /// The `?key=` (else `?auth_token=`) value of the request being served (Go
510    /// `homeQueryCredentialFromContext` reads gin's query).
511    static QUERY_CREDENTIAL: String;
512}
513
514/// `fut` with this request's query credential, for dispatch work that outlives the
515/// gate (a keep-alive moves the unfinished call into the response body).
516pub(crate) fn keep_query_credential<F: std::future::Future>(fut: F) -> impl std::future::Future<Output = F::Output> {
517    let credential = QUERY_CREDENTIAL.try_with(Clone::clone).unwrap_or_default();
518    QUERY_CREDENTIAL.scope(credential, fut)
519}
520
521/// Go `homeDispatchHeaders`: the request headers Home authenticates the client with;
522/// a query credential is added as X-Goog-Api-Key when no credential header was sent:
523/// the request's `?key=`/`?auth_token=`, else the query key `caller` authenticated with
524/// (Go's `accessMetadata` branch; it also serves WebSocket sessions, which run after
525/// their upgrade request).
526pub(crate) fn home_headers(
527    headers: &axum::http::HeaderMap,
528    caller: Option<&cpa_core::exec::Caller>,
529) -> Vec<(String, String)> {
530    let mut out: Vec<(String, String)> = headers
531        .iter()
532        .map(|(name, value)| {
533            (
534                name.as_str().to_owned(),
535                String::from_utf8_lossy(value.as_bytes()).into_owned(),
536            )
537        })
538        .collect();
539    let mut credential = QUERY_CREDENTIAL.try_with(Clone::clone).unwrap_or_default();
540    if credential.is_empty()
541        && let Some(caller) = caller.filter(|c| matches!(c.source, "query-key" | "query-auth-token"))
542    {
543        credential = caller.principal.trim().to_owned();
544    }
545    let has_header = ["authorization", "x-goog-api-key", "x-api-key"]
546        .iter()
547        .any(|name| headers.get(*name).is_some_and(|v| !v.is_empty()));
548    if !credential.is_empty() && !has_header {
549        out.push(("x-goog-api-key".to_owned(), credential));
550    }
551    out
552}
553
554/// The query credential Go reads for Home: `key`, else `auth_token`, trimmed.
555fn query_credential(query: &str) -> String {
556    ["key", "auth_token"]
557        .iter()
558        .map(|name| {
559            crate::access::query_get(query, name)
560                .map(|v| String::from_utf8_lossy(&v).trim().to_owned())
561                .unwrap_or_default()
562        })
563        .find(|v| !v.is_empty())
564        .unwrap_or_default()
565}
566
567/// Go `homeHeartbeatMiddleware`: while a dispatcher is installed and its control plane
568/// is unreachable, every route but management answers 503 with an empty body.
569/// Inside, the request's query credential is available to `home_headers`.
570pub async fn gate(
571    axum::extract::State(rt): axum::extract::State<Arc<crate::Runtime>>,
572    request: axum::extract::Request,
573    next: axum::middleware::Next,
574) -> axum::response::Response {
575    let path = request.uri().path();
576    let exempt = ["/v0/management", "/v8/management"]
577        .iter()
578        .any(|base| path == *base || path.strip_prefix(base).is_some_and(|rest| rest.starts_with('/')))
579        || path.starts_with("/v0/resource/plugins/")
580        || path == "/management.html";
581    if !exempt
582        && let Some(remote) = rt.remote_dispatch()
583        && !remote.available()
584    {
585        return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response();
586    }
587    let credential = query_credential(request.uri().query().unwrap_or_default());
588    QUERY_CREDENTIAL.scope(credential, next.run(request)).await
589}
590
591#[cfg(test)]
592mod tests {
593    use super::*;
594
595    /// Go `time.Time` JSON decoding, expected values from Go 1.27: a comma fraction and
596    /// a 24:60 offset are accepted, extra digits are cut at nanoseconds, a one-digit
597    /// hour is accepted, and Go's zero time (whatever the offset) is `IsZero`.
598    #[test]
599    fn home_times_decode_as_go_does() {
600        const Y2999: i64 = 32_472_144_000;
601        for (raw, want) in [
602            ("2999-01-01T00:00:00Z", Some(Some((Y2999, 0)))),
603            ("2999-01-01T00:00:00,1Z", Some(Some((Y2999, 100_000_000)))),
604            ("2999-01-01T00:00:00.1234567891234Z", Some(Some((Y2999, 123_456_789)))),
605            ("2999-01-01T00:00:00+24:60", Some(Some((Y2999 - 25 * 3600, 0)))),
606            ("2999-01-01T00:00:00-00:00", Some(Some((Y2999, 0)))),
607            ("2000-01-01T0:00:00Z", Some(Some((946_684_800, 0)))),
608            ("0001-01-01T00:00:00Z", Some(None)),
609            ("0001-01-01T01:00:00+01:00", Some(None)),
610            ("2999-01-01T00:00:00+25:00", None),
611            ("2999-01-01T00:00:00+00:61", None),
612            ("2999-01-01T00:00:00z", None),
613            ("2999-01-01t00:00:00Z", None),
614            ("2999-01-01T00:00:60Z", None),
615            ("2999-01-01T24:00:00Z", None),
616            ("2999-02-30T00:00:00Z", None),
617            ("2999-01-01T00:00:00,Z", None),
618            ("2999-01-01T00:00:00+0100", None),
619            ("2999-01-01T00:00:00Z ", None),
620            ("+001-01-01T00:00:00Z", None),
621            ("-001-01-01T00:00:00Z", None),
622            ("2999-01-01T00:00:00", None),
623            ("", None),
624        ] {
625            assert_eq!(go_time(raw), want, "{raw}");
626        }
627    }
628
629    /// Go decodes the auth's runtime state into typed fields: `null` keeps a default,
630    /// any other mistyped value fails the reply.
631    #[test]
632    fn home_availability_rejects_what_go_cannot_decode() {
633        let decode = Availability::decode;
634        for valid in [
635            r#"{}"#,
636            r#"{"status":null,"unavailable":null,"next_retry_after":null,"model_states":null}"#,
637            r#"{"model_states":{"m":null,"n":{"status":"disabled"}}}"#,
638        ] {
639            assert!(decode(valid).is_ok(), "{valid}");
640        }
641        for invalid in [
642            r#"{"status":1}"#,
643            r#"{"unavailable":"true"}"#,
644            r#"{"next_retry_after":""}"#,
645            r#"{"next_retry_after":0}"#,
646            r#"{"model_states":[]}"#,
647            r#"{"model_states":{"m":1}}"#,
648            r#"{"model_states":{"m":{"next_retry_after":""}}}"#,
649            r#"{"model_states":{"m":{"unavailable":1}}}"#,
650            // Go parses the quoted bytes as written: an escaped `Z` is not `Z`.
651            r#"{"next_retry_after":"2999-01-01T00:00:00\u005a"}"#,
652            r#"{"model_states":{"m":{"next_retry_after":"2999-01-01T00:00:00\u005a"}}}"#,
653        ] {
654            assert!(decode(invalid).is_err(), "{invalid}");
655        }
656    }
657
658    /// `responsesWebsocketAuthAvailableForModel` on decoded snapshots, at 2026-01-01.
659    #[test]
660    fn home_availability_follows_go() {
661        let now = std::time::UNIX_EPOCH + Duration::from_secs(1_767_225_600);
662        let available = |json: &str, model: &str| {
663            let snapshot = Availability::decode(json).unwrap();
664            available_for_model(&Credential::relay_session("home-a"), &snapshot, model, now)
665        };
666        let future = "2999-01-01T00:00:00,1Z";
667        for (json, model, want) in [
668            (r#"{}"#.to_owned(), "m", true),
669            (r#"{"status":"disabled"}"#.into(), "m", false),
670            (
671                format!(r#"{{"unavailable":true,"next_retry_after":"{future}"}}"#),
672                "m",
673                false,
674            ),
675            (
676                r#"{"unavailable":true,"next_retry_after":"2001-01-01T00:00:00Z"}"#.into(),
677                "m",
678                true,
679            ),
680            (
681                r#"{"unavailable":true,"next_retry_after":"0001-01-01T00:00:00Z"}"#.into(),
682                "m",
683                true,
684            ),
685            (r#"{"unavailable":true}"#.into(), "m", true),
686            (
687                format!(r#"{{"model_states":{{"m":{{"unavailable":true,"next_retry_after":"{future}"}}}}}}"#),
688                "m",
689                false,
690            ),
691            // The suffix-free base model's state, and the model's state over the auth's.
692            (
693                r#"{"model_states":{"m":{"status":"disabled"}}}"#.into(),
694                "m(high)",
695                false,
696            ),
697            (
698                format!(r#"{{"unavailable":true,"next_retry_after":"{future}","model_states":{{"m":{{}}}}}}"#),
699                "m",
700                true,
701            ),
702            // A nil state is never consulted: the auth-wide cooldown decides.
703            (
704                format!(r#"{{"unavailable":true,"next_retry_after":"{future}","model_states":{{"m":null}}}}"#),
705                "m",
706                false,
707            ),
708            (r#"{"model_states":{"other":{"status":"disabled"}}}"#.into(), "m", true),
709        ] {
710            assert_eq!(available(&json, model), want, "{json} {model}");
711        }
712    }
713
714    /// Go `homeDispatchHeaders` and `homeQueryCredentialFromContext`: `?key=` (else
715    /// `?auth_token=`) reaches Home as X-Goog-Api-Key unless a credential header was
716    /// sent; outside a request nothing is added. Covers Go
717    /// `TestHomeDispatchHeadersAddsQueryKeyCredential`,
718    /// `TestHomeDispatchHeadersAddsQueryCredentialFromAccessMetadata` (the query-key
719    /// caller), `TestHomeDispatchHeadersKeepsExistingCredentialHeader` and
720    /// `TestHomeDispatchHeadersIgnoresHeaderCredentialSource` (the header caller).
721    #[tokio::test]
722    async fn query_credentials_reach_home_as_x_goog_api_key() {
723        let headers = |pairs: &[(&str, &str)]| {
724            let mut map = axum::http::HeaderMap::new();
725            for (name, value) in pairs {
726                map.insert(
727                    axum::http::HeaderName::from_bytes(name.as_bytes()).unwrap(),
728                    value.parse().unwrap(),
729                );
730            }
731            map
732        };
733        let goog = |query: &str, pairs: &[(&str, &str)]| {
734            let map = headers(pairs);
735            let credential = query_credential(query);
736            async move {
737                QUERY_CREDENTIAL
738                    .scope(credential, async {
739                        home_headers(&map, None)
740                            .into_iter()
741                            .filter(|(name, _)| name == "x-goog-api-key")
742                            .map(|(_, value)| value)
743                            .collect::<Vec<_>>()
744                    })
745                    .await
746            }
747        };
748        assert_eq!(goog("key=k1&alt=sse", &[]).await, ["k1"]);
749        assert_eq!(goog("auth_token=t1", &[("user-agent", "x")]).await, ["t1"]);
750        assert_eq!(goog("auth_token=t1&key=%20k2%20", &[]).await, ["k2"]);
751        assert!(goog("key=k1", &[("authorization", "Bearer other")]).await.is_empty());
752        assert_eq!(
753            goog("key=k1", &[("x-goog-api-key", "header-key")]).await,
754            ["header-key"]
755        );
756        assert!(goog("key=k1", &[("x-api-key", "k")]).await.is_empty());
757        assert!(goog("alt=sse", &[]).await.is_empty());
758        assert!(home_headers(&headers(&[]), None).is_empty());
759        // Outside a request, a query-key caller still identifies itself.
760        let caller = cpa_core::exec::Caller {
761            principal: "local-key".into(),
762            source: "query-key",
763            provider: String::new(),
764        };
765        let outside: Vec<_> = home_headers(&headers(&[]), Some(&caller));
766        assert_eq!(outside, [("x-goog-api-key".to_owned(), "local-key".to_owned())]);
767        let header_caller = cpa_core::exec::Caller {
768            principal: "local-key".into(),
769            source: "authorization",
770            provider: String::new(),
771        };
772        assert!(home_headers(&headers(&[]), Some(&header_caller)).is_empty());
773    }
774}