Skip to main content

harness/chat/
models.rs

1//! What generation needs from the provider layer, and the settings it reads.
2//!
3//! [`Models`] is upstream `HarnessOptions.models` reduced to what the generation and compaction tasks call:
4//! resolve a model, stream one response under the invocation's signal, and poll or cancel a deferred response. It
5//! blocks the calling task thread. The product implementation is `session_core::turns::TurnModels`; tests use
6//! `faux::Faux` (feature `test-support`).
7
8use std::sync::Arc;
9
10pub use ai::overflow::is_context_overflow;
11use ai::{AssistantMessage, AssistantMessageEvent, DeferredHandle, Model, TranscriptContext};
12use serde::{Deserialize, Serialize};
13use serde_json::{Map, Value};
14
15use super::inbox::{QueueMode, QueueModes};
16use crate::scheduler::{Cancel, HarnessError, Result};
17
18/// A model by provider and ID, as `ultron.agent` and generation checkpoints pin it.
19#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase")]
21pub struct ModelRef {
22    pub provider: String,
23    pub model_id: String,
24}
25
26impl ModelRef {
27    pub fn new(provider: &str, model_id: &str) -> ModelRef {
28        ModelRef {
29            provider: provider.to_owned(),
30            model_id: model_id.to_owned(),
31        }
32    }
33}
34
35/// One request's options: the pinned stream options (`ConversationStreamOptions` as JSON, e.g. `maxTokens`,
36/// `temperature`), the provider session ID, and the reasoning level (`None` for thinking off).
37#[derive(Clone, Debug, Default, PartialEq)]
38pub struct StreamRequest {
39    pub options: Map<String, Value>,
40    pub session_id: Option<String>,
41    pub reasoning: Option<String>,
42    /// The conversation the request is for (e.g. to charge its usage to the work that owns it).
43    pub conversation_id: Option<durable::ConversationId>,
44    /// Set on a run's final request ([`super::gate::RequestGate::Final`]): the note its context ends with. The
45    /// request's options carry tool choice `none`; [`super::gate::apply_final`] applies both to what is sent.
46    pub final_note: Option<String>,
47}
48
49impl StreamRequest {
50    /// The request as `ai::SimpleStreamOptions` (the signal is set by the caller).
51    pub fn simple_options(&self) -> ai::SimpleStreamOptions {
52        let mut options: ai::SimpleStreamOptions =
53            serde_json::from_value(Value::Object(self.options.clone())).unwrap_or_default();
54        options.base.session_id = self.session_id.clone();
55        options.reasoning = self.reasoning.clone();
56        options
57    }
58}
59
60/// The provider layer as generation uses it. Every call blocks; `signal` is the task invocation's.
61pub trait Models: Send + Sync {
62    fn get_model(&self, provider: &str, model_id: &str) -> Option<Model>;
63
64    /// Stream one response. Events go to `on_event` as they arrive; provider failures are messages with stop reason
65    /// `error`; `Err` is a transport bug. An aborted signal ends the stream with an `aborted` message or `Err`.
66    fn stream(
67        &self,
68        model: &Model,
69        context: &TranscriptContext,
70        request: &StreamRequest,
71        signal: &Cancel,
72        on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
73    ) -> Result<AssistantMessage>;
74
75    /// Poll a deferred response; a still deferred result keeps stop reason `deferred`.
76    fn fetch_deferred(&self, model: &Model, handle: &DeferredHandle, signal: &Cancel) -> Result<AssistantMessage> {
77        let _ = (model, handle, signal);
78        Err(HarnessError::msg("This provider has no deferred responses"))
79    }
80
81    fn cancel_deferred(&self, model: &Model, handle: &DeferredHandle, signal: &Cancel) -> Result<()> {
82        let _ = (model, handle, signal);
83        Ok(())
84    }
85}
86
87/// `settings.compaction` (spec ยง8.7).
88#[derive(Clone, Copy, Debug, PartialEq)]
89pub struct CompactionPolicy {
90    pub enabled: bool,
91    pub reserve_tokens: u64,
92    pub keep_recent_tokens: u64,
93    pub background_tokens: u64,
94}
95
96impl Default for CompactionPolicy {
97    fn default() -> Self {
98        CompactionPolicy {
99            enabled: true,
100            reserve_tokens: 16_384,
101            keep_recent_tokens: 20_000,
102            background_tokens: 32_768,
103        }
104    }
105}
106
107/// Whether a round's cells run at once or one after another.
108#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
109pub enum ToolExecution {
110    #[default]
111    Parallel,
112    Sequential,
113}
114
115/// Harness settings, read at each decision so changes apply to the next one.
116#[derive(Clone, Debug, PartialEq)]
117pub struct Settings {
118    /// Request options pinned at preparation (`ConversationStreamOptions`).
119    pub stream: Map<String, Value>,
120    pub retry: ai::retry::RetryPolicy,
121    pub compaction: CompactionPolicy,
122    /// Throttle of committed generation partials.
123    pub partial_interval_ms: u64,
124    /// Throttle of committed cell output.
125    pub output_interval_ms: u64,
126    pub tool_execution: ToolExecution,
127    pub steering_mode: QueueMode,
128    pub follow_up_mode: QueueMode,
129    /// A final boundary that places steering messages leaves follow-ups for the next one (the TypeScript agent loop:
130    /// steering continues the run, follow-ups are polled only when it would stop), so a steer and a follow-up
131    /// queued at a final answer run as two turns. Off: the spec's boundary places both at once.
132    pub follow_up_after_steer: bool,
133}
134
135impl Default for Settings {
136    fn default() -> Self {
137        Settings {
138            stream: Map::new(),
139            retry: ai::retry::RetryPolicy::default(),
140            compaction: CompactionPolicy::default(),
141            partial_interval_ms: 100,
142            output_interval_ms: 100,
143            tool_execution: ToolExecution::Parallel,
144            steering_mode: QueueMode::OneAtATime,
145            follow_up_mode: QueueMode::OneAtATime,
146            follow_up_after_steer: false,
147        }
148    }
149}
150
151impl Settings {
152    pub(crate) fn queue_modes(&self) -> QueueModes {
153        QueueModes {
154            steering: self.steering_mode,
155            follow_up: self.follow_up_mode,
156            follow_up_after_steer: self.follow_up_after_steer,
157        }
158    }
159}
160
161/// Where settings are read from: a fixed value or a live source.
162pub type SettingsSource = Arc<dyn Fn() -> Settings + Send + Sync>;