Skip to main content

session_core/
request_keys.rs

1//! The key each model request carries, resolved when the request is sent (`ModelRuntime.getAuth` per request, so a
2//! `/login`, a changed auth.json or a refreshed OAuth token applies to the next request), and the request stack the
3//! session's side lanes (frames, in-memory subagents, the predict adapter) send through.
4//!
5//! - [`Keys`]: `--api-key` is a runtime key of the provider it was given for (session-worker.ts
6//!   `setRuntimeApiKey(resolved.model.provider, apiKey)`); every other provider resolves its own credentials.
7//!   Ambient AWS or Google credentials (`<authenticated>`) authenticate a request without a key, so none is sent.
8//! - [`Keyed`]: a stream whose requests carry the key of their model's provider unless the caller set one.
9//! - [`Retrying`]: the harness retry policy (`settings.retry`, read per request) over a stream, as a TypeScript
10//!   harness lane retries a transient provider error.
11
12use agent::StreamFn;
13use ai::http::BoxFuture;
14use ai::retry::RetryPolicy;
15use ai::{AssistantMessage, EventSink, Model, SimpleStreamOptions, TranscriptContext};
16use std::pin::pin;
17use std::sync::Arc;
18use std::task::Poll;
19
20/// Resolves a provider's own credentials (auth.json, models.json, the environment) now.
21pub type ProviderCredentials = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
22
23#[derive(Clone)]
24pub struct Keys {
25    /// `--api-key` and the provider it was given for.
26    runtime: Option<(String, String)>,
27    resolve: ProviderCredentials,
28}
29
30impl Keys {
31    /// `runtime`: the `--api-key` of `(provider, key)`; every provider else resolves through `resolve`.
32    pub fn new(runtime: Option<(String, String)>, resolve: ProviderCredentials) -> Keys {
33        Keys { runtime, resolve }
34    }
35
36    /// The session's keys: `api_key` (`--api-key`) for `provider` (the starting model's), the profile's credentials
37    /// of `env` for every provider.
38    pub fn session(api_key: Option<String>, provider: &str, env: &crate::RunEnv) -> Keys {
39        let env = env.clone();
40        Keys::new(
41            api_key.map(|key| (provider.to_string(), key)),
42            Arc::new(move |provider: &str| crate::lane::api_key(&crate::ModelFlags::default(), &env, provider)),
43        )
44    }
45
46    /// Whether `provider` has a key (or ambient credentials): what a prompt's preflight checks.
47    pub fn key(&self, provider: &str) -> Option<String> {
48        match &self.runtime {
49            Some((runtime, key)) if runtime == provider => Some(key.clone()),
50            _ => (self.resolve)(provider),
51        }
52    }
53
54    /// The API key a request of `provider` sends: [`Keys::key`] without the ambient-credentials marker.
55    pub fn request_key(&self, provider: &str) -> Option<String> {
56        self.key(provider).filter(|key| key != ai::env::AMBIENT_CREDENTIALS)
57    }
58
59    /// [`Keys::request_key`] as a provider-key function.
60    pub fn request_keys(&self) -> ProviderCredentials {
61        let keys = self.clone();
62        Arc::new(move |provider: &str| keys.request_key(provider))
63    }
64}
65
66/// A stream whose requests carry the key of their model's provider, resolved per request, unless the caller set one.
67pub struct Keyed<S> {
68    pub inner: S,
69    pub keys: Keys,
70}
71
72impl<S: StreamFn> StreamFn for Keyed<S> {
73    fn stream<'a>(
74        &'a self,
75        model: &'a Model,
76        context: &'a TranscriptContext,
77        options: &'a SimpleStreamOptions,
78        sink: &'a mut EventSink<'_>,
79    ) -> BoxFuture<'a, AssistantMessage> {
80        Box::pin(async move {
81            if options.base.api_key.is_some() {
82                return self.inner.stream(model, context, options, sink).await;
83            }
84            let mut keyed = options.clone();
85            keyed.base.api_key = self.keys.request_key(&model.provider);
86            self.inner.stream(model, context, &keyed, sink).await
87        })
88    }
89}
90
91/// Reads the retry policy as the settings name it now.
92pub type RetrySource = Arc<dyn Fn() -> RetryPolicy + Send + Sync>;
93
94/// `settings.retry` of a settings manager (`getRetrySettings`): what the durable runtime retries a lane's
95/// generation with.
96pub fn retry_policy(settings: &config::settings::SettingsManager) -> RetryPolicy {
97    let retry = settings
98        .settings()
99        .get("retry")
100        .cloned()
101        .unwrap_or(serde_json::Value::Null);
102    RetryPolicy::from_settings(&retry, true)
103}
104
105/// A stream that asks again after a transient provider error (`isRetryableAssistantError`), up to the policy's
106/// retries with its backoff; an aborted request is never retried and the wait ends with the request's signal.
107pub struct Retrying<S> {
108    pub inner: S,
109    pub policy: RetrySource,
110}
111
112impl<S: StreamFn> StreamFn for Retrying<S> {
113    fn stream<'a>(
114        &'a self,
115        model: &'a Model,
116        context: &'a TranscriptContext,
117        options: &'a SimpleStreamOptions,
118        sink: &'a mut EventSink<'_>,
119    ) -> BoxFuture<'a, AssistantMessage> {
120        Box::pin(async move {
121            let policy = (self.policy)();
122            let mut attempt = 0;
123            loop {
124                let message = self.inner.stream(model, context, options, sink).await;
125                let aborted = options.base.signal.as_ref().is_some_and(ai::AbortSignal::aborted);
126                if aborted || !policy.enabled || attempt >= policy.max_retries || !ai::retry::is_retryable(&message) {
127                    return message;
128                }
129                attempt += 1;
130                let delay = std::time::Duration::from_millis(policy.delay_ms(attempt));
131                if !wait(delay, options.base.signal.as_ref()).await {
132                    return message;
133                }
134            }
135        })
136    }
137}
138
139/// Waits `delay`; false when `signal` aborts first.
140async fn wait(delay: std::time::Duration, signal: Option<&ai::AbortSignal>) -> bool {
141    let Some(signal) = signal else {
142        tokio::time::sleep(delay).await;
143        return true;
144    };
145    let mut sleep = pin!(tokio::time::sleep(delay));
146    let mut cancelled = pin!(signal.cancelled());
147    std::future::poll_fn(|cx| {
148        if cancelled.as_mut().poll(cx).is_ready() {
149            return Poll::Ready(false);
150        }
151        match sleep.as_mut().poll(cx) {
152            Poll::Ready(()) => Poll::Ready(true),
153            Poll::Pending => Poll::Pending,
154        }
155    })
156    .await
157}
158
159/// The stream a side lane of the session requests through as a TypeScript harness lane does: the live request
160/// settings and idle timeout (`streamOptions`, `httpIdleTimeoutMs`), the key of each request's provider resolved per
161/// request, and the harness retry policy.
162pub fn lane_stream(settings: crate::lane::SettingsReader, keys: Keys) -> Arc<dyn StreamFn> {
163    let reader = settings.clone();
164    Arc::new(Retrying {
165        inner: Keyed {
166            inner: crate::lane::Provider::new().with_live_settings(settings),
167            keys,
168        },
169        policy: Arc::new(move || retry_policy(&reader())),
170    })
171}