1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
//! Cache-warm pings: one capped request that re-reads the session's cached
//! prompt prefix so the provider keeps it alive while the model idles on a
//! `process start` that asked for `keep_warm`.
//!
//! The TUI owns scheduling and stop policy (`tui/cache_warm.rs`); this module
//! only builds a request whose prefix matches what the next real turn sends:
//! session history (system prompt included), host request context, the
//! advertised tool specs, reasoning level, and prompt-cache key. A placeholder
//! user message ends it because some models reject assistant prefill.
use std::{num::NonZeroU32, sync::Arc, time::Duration};
use rho_sdk::{
model::{Message, ModelUsage},
provider::{ModelProvider, ModelRequestOptions},
CancellationToken, ProviderRequestUsageContext, ProviderRequestUsageRecording,
SessionRequestPrefix,
};
use super::InteractiveRuntime;
/// Output cap for a ping. Receipt: the Responses API rejects
/// `max_output_tokens: 0` while streaming; 1 returned one output token, zero
/// reasoning tokens, and a full cache read (`/tmp/spike_resp.log`).
pub(crate) const PING_OUTPUT_TOKENS: NonZeroU32 = NonZeroU32::MIN;
/// Trailing user turn. Its text is never part of a real request, so only the
/// prefix before it matters for cache reads.
const PING_TEXT: &str = "(prompt cache keep-alive; no reply needed)";
/// Usage-ledger purpose for ping spend, separate from turns and compaction.
const PING_PURPOSE: &str = "cache_warm";
/// A ready-to-send ping, owned so it can run on a background task.
pub(crate) struct CacheWarmPing {
provider: Arc<dyn ModelProvider>,
prefix: SessionRequestPrefix,
context: ProviderRequestUsageContext,
recording: ProviderRequestUsageRecording,
}
/// What one ping cost and whether the provider failed it.
///
/// A capped ping normally ends as an `incomplete` response error after
/// reporting usage, so `error` only matters when no usage arrived.
#[derive(Debug, Default)]
pub(crate) struct CacheWarmOutcome {
pub(crate) usage: ModelUsage,
pub(crate) error: Option<String>,
}
impl CacheWarmPing {
/// Sends the ping and records every attempt under the `cache_warm` purpose.
pub(crate) async fn send(self) -> CacheWarmOutcome {
let mut next_attempt_index = 0;
let mut usage = ModelUsage::default();
// No service tier: a priority multiplier would raise the ping price,
// and a keep-alive needs no latency class.
let options = ModelRequestOptions::default().with_max_output_tokens(PING_OUTPUT_TOKENS);
let result = crate::usage::send_recorded_with(
self.provider.as_ref(),
crate::usage::RecordedRequest {
request: self.prefix.as_request(CancellationToken::new()),
options,
context: self.context,
recording: self.recording,
},
&mut next_attempt_index,
&mut usage,
|_| {},
)
.await;
CacheWarmOutcome {
usage,
error: result.err().map(|error| error.to_string()),
}
}
}
impl InteractiveRuntime {
/// True while a live process holds the prompt cache warm.
pub(crate) fn has_keep_warm_hold(&self) -> bool {
self.processes()
.is_some_and(crate::tools::process::ProcessManager::has_keep_warm_hold)
}
/// Whether the current model route can send a capped ping.
pub(crate) fn supports_cache_warm(&self) -> bool {
self.provider.supports_cache_warm()
}
/// Prompt cache TTL for the current provider, with its default resolved once.
pub(crate) fn prompt_cache_ttl(&self) -> Duration {
self.config
.providers
.resolved_prompt_cache_ttl(&self.provider.provider().identity().provider)
}
/// Builds a ping for the current session prefix.
pub(crate) fn cache_warm_ping(&self) -> Result<CacheWarmPing, rho_sdk::Error> {
let provider = Arc::clone(self.provider.provider());
let session_id = self.session_id().clone();
let prefix = self
.sessions
.session()
.request_prefix()?
.with_trailing_message(Message::user_text(PING_TEXT));
let context = ProviderRequestUsageContext::for_purpose(provider.identity(), PING_PURPOSE)
.with_session_id(session_id.clone());
Ok(CacheWarmPing {
prefix,
context,
recording: self.usage_recording.clone(),
provider,
})
}
}